diff --git a/apps/docs/content.manifest.ts b/apps/docs/content.manifest.ts index 9075738a..c3adccb3 100644 --- a/apps/docs/content.manifest.ts +++ b/apps/docs/content.manifest.ts @@ -72,6 +72,7 @@ export const sections: Section[] = [ }, { path: 'getting-started', title: 'Getting started', icon: 'Rocket' }, { path: 'recipes', title: 'Recipes', icon: 'ChefHat' }, + { path: 'scenarios', title: 'Scenarios', icon: 'Map' }, { path: 'concepts', title: 'Concepts', icon: 'Lightbulb' }, { path: 'guides', title: 'Guides', icon: 'BookOpen' }, { path: 'guides/authoring', title: 'Authoring & composition' }, @@ -261,6 +262,169 @@ export const pages: Page[] = [ kind: 'guide', }, + // ── Scenarios ─────────────────────────────────────────────────────────── + { + path: 'scenarios/index', + title: 'Scenarios', + description: + 'Real integration problems that have no one-line answer anywhere — what the usual fixes cost, what StitchAPI changes, and what it leaves to you.', + kind: 'landing', + }, + { + path: 'scenarios/oauth2-refresh-token-rotation', + title: 'OAuth2 refresh tokens that rotate', + description: + 'A single-use refresh token plus two concurrent workers revokes the whole account. What the usual fixes cost, and what a custom auth strategy buys you.', + kind: 'guide', + }, + { + path: 'scenarios/cost-based-rate-limits', + title: 'Rate limits priced in query cost', + description: + 'Shopify bills per query cost, answers 200 OK when you overspend, and puts the wait in the body. Why status-code retry and rate-per-second both miss, and what does work.', + kind: 'guide', + }, + { + path: 'scenarios/batch-partial-failure', + title: 'Batch writes that fail one item at a time', + description: + 'A bulk endpoint returns 200 and reports that 7 of your 100 items did not land. Retrying the request re-writes the 93 that did — so the retry unit has to be the body, not the call.', + kind: 'guide', + }, + { + path: 'scenarios/async-job-polling', + title: 'Submit, poll, download — the async job triangle', + description: + 'A 202 with a Location header, a status endpoint that reports failure at HTTP 200, and a single-use result URL. Three endpoints and a loop, and you have to pick which guarantee you keep.', + kind: 'guide', + }, + { + path: 'scenarios/mid-stream-failure', + title: 'A stream that fails after 800 tokens', + description: + 'The 200 was spent on the first token, so the failure arrives in-band or not at all. When a stream can be resumed this is one flag; when it cannot — every LLM API — that same flag replays the whole answer.', + kind: 'guide', + }, + { + path: 'scenarios/conditional-requests-304', + title: 'The free poll — ETag revalidation and the bodyless 304', + description: + 'A 304 means "use what you have", carries no body, and is not a 2xx. Turning it back into the resource takes one seam — and the cache primitive cannot help.', + kind: 'guide', + }, + { + path: 'scenarios/multipart-upload', + title: 'The upload you must clean up after', + description: + 'Multipart upload is four steps, and the fourth — abort on failure — is the one no HTTP client models. Skip it and the parts bill forever, invisibly.', + kind: 'guide', + }, + { + path: 'scenarios/webhook-receipt', + title: 'Receiving a signed webhook', + description: + 'StitchAPI does not receive webhooks — that is your server. Here is exactly where the line falls, measured, and what the library does own on the far side of it.', + kind: 'guide', + }, + { + path: 'scenarios/multi-tenant-blast-radius', + title: "One customer's revoked token, everyone's outage", + description: + 'Tokens and caches isolate per tenant automatically. Rate budgets and circuit breakers do not — they isolate only by a string you have to remember to write.', + kind: 'guide', + }, + { + path: 'scenarios/provider-failover', + title: 'Failing over to the backup provider', + description: + 'Everything per-provider is free and declarative. The routing between them is entirely yours — and the combinator named for this job bills you twice on every successful call.', + kind: 'guide', + }, + { + path: 'scenarios/unstable-pagination', + title: 'The page that moved while you were reading it', + description: + 'Offset pagination over a live collection silently returns wrong lists. A client cannot fix that — but it should not report a clean run over data it lost.', + kind: 'guide', + }, + { + path: 'scenarios/intermittent-drift', + title: 'The vendor changed the shape for 5% of responses', + description: + 'Leveled drift catches a canary rollout precisely and refuses to invent a value. What it cannot do is tell a harmless coercion from a destructive one.', + kind: 'guide', + }, + { + path: 'scenarios/large-response-memory', + title: 'The export that eats the heap', + description: + 'The NDJSON decoder is genuinely O(1) — and the engine retains every chunk one line later, so neither await nor .stream() is memory-bounded.', + kind: 'guide', + }, + { + path: 'scenarios/expiring-signatures', + title: 'The signature that expired in your own queue', + description: + 'A rate-limited queue cannot age a SigV4 signature here — the wait happens before signing, by construction. Clock drift still needs 26 lines.', + kind: 'guide', + }, + { + path: 'scenarios/unconfirmed-write', + title: "The charge you can't confirm", + description: + 'A timeout tells you nothing about the server. idempotency.keyOf fixes the restart and the race in configuration alone — the default key does not, and it double-charged a re-driven job.', + kind: 'guide', + }, + { + path: 'scenarios/n-plus-one-fanout', + title: 'One list, a hundred follow-up calls', + description: + 'cache.coalesce collapses in-flight duplicates — 100 concurrent calls over 30 ids made 30 requests. A coalesced failure is not shared, and that is where it loses.', + kind: 'guide', + }, + { + path: 'scenarios/deprecation-headers', + title: 'The vendor told you for six months, in a header', + description: + 'Deprecation and Sunset arrive on responses that succeeded, so nothing fails and nothing retries. Response headers are reachable in exactly three places — here is the table.', + kind: 'guide', + }, + { + path: 'scenarios/agent-holds-the-tool', + title: 'The agent picks the arguments', + description: + 'Exposing a vendor API to an LLM over MCP. The credential boundary held under 30 payload scans — the argument boundary is yours, and a query parameter pinned in the path is only a default.', + kind: 'guide', + }, + { + path: 'scenarios/stale-fixture', + title: 'The mock that passed for six months', + description: + 'Your fake goes stale and the suite keeps saying green. Resilience and streams test perfectly offline — here is the definitive table of which time-driven features manualClock actually drives.', + kind: 'guide', + }, + { + path: 'scenarios/precision-loss', + title: 'The ID that changed on the way in', + description: + 'JSON.parse turns a 64-bit snowflake into a different number, silently. wire.response text plus transform recovers the exact digits in 16 lines — and a bigint in params vanishes.', + kind: 'guide', + }, + { + path: 'scenarios/pii-in-the-logs', + title: "The customer data you didn't mean to log", + description: + 'Response bodies reach 13 destinations and metadata reaches 11 — with nothing in between. An output allowlist takes it to zero; sensitive: true does not, and only gates the cache.', + kind: 'guide', + }, + { + path: 'scenarios/dual-run-migration', + title: 'The migration you have to run twice', + description: + 'Dual-running a vendor v1 and v2 when you own neither endpoint. One of five isolation channels is safe by default, and the combinator that looks built for this broadcasts one input to both.', + kind: 'guide', + }, + // ── Concepts ──────────────────────────────────────────────────────────── { path: 'concepts/the-stitch', diff --git a/apps/docs/content/docs/meta.json b/apps/docs/content/docs/meta.json index 2c8ba2a7..41e5c147 100644 --- a/apps/docs/content/docs/meta.json +++ b/apps/docs/content/docs/meta.json @@ -4,6 +4,7 @@ "pages": [ "getting-started", "recipes", + "scenarios", "concepts", "guides", "surfaces", diff --git a/apps/docs/content/docs/scenarios/agent-holds-the-tool.mdx b/apps/docs/content/docs/scenarios/agent-holds-the-tool.mdx new file mode 100644 index 00000000..95eeab10 --- /dev/null +++ b/apps/docs/content/docs/scenarios/agent-holds-the-tool.mdx @@ -0,0 +1,161 @@ +--- +title: 'The agent picks the arguments' +description: 'Exposing a vendor API to an LLM over MCP. The credential boundary held under 30 payload scans — the argument boundary is yours, and a query parameter pinned in the path is only a default.' +prerequisites: ['/docs/surfaces/mcp', '/docs/reference/auth-strategies'] +--- + +## The problem + +You already call a vendor API from your code. Now an agent needs to call it too, over MCP. The +model picks **which** call and **what arguments** — from a prompt that may contain text you did +not write. + +That inverts the usual trust story. An agent tool is an API exposed to an untrusted caller, but +it is almost always written as if it were an internal function. Three hazards, all measured in +the wild: + +- **The credential must never reach the model.** A survey of over 10,000 public MCP servers + found credentials, keys and PII leaking at rates + [exceeding 10%](https://checkmarx.com/learn/mcp-security-risks-real-world-incidents-and-security-controls/). + A token in a schema, an argument, a result or an error message is a token in the model's + context — and therefore in its output, its logs, and any downstream tool it calls. +- **The arguments are attacker-influenced.** A scan of popular MCP servers found 43% with + command-injection flaws, 22% allowing path traversal and 30% exploitable via SSRF. The input + schema stops being ergonomics and becomes the security boundary. +- **The loop is the cost.** One agent scanning a network reached a + [$6,531 bill](https://www.nexgismo.com/blog/ai-agent-budget-guards-stop-runaway-api-costs) in + days with no hard limits. + +## The common solutions + +| Approach | What it is | Where it breaks | +| -------------------------------- | ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------- | +| **One MCP server per vendor** | Hand-write a server wrapping the API. | Full control — and you write the auth, validation and limits yourself. That is what the >10% leak rate is measuring. | +| **One tool per endpoint** | Narrow, typed tools the model picks between. | The safest shape: the schema _is_ the allow-list. Costs a tool definition per endpoint, and a lot of context. | +| **One generic "run it" tool** | The model names the call and passes arguments. | Compact, and far more dangerous: the argument object becomes the attack surface. | +| **Gateway in front** | Policy, quotas and egress rules outside the app. | The enterprise answer. Another hop, and it cannot see intent. | +| **Human confirmation on writes** | Ask before anything irreversible. | The one control that survives prompt injection. Needs a place to hook it. | + +## What StitchAPI does + +`stitch mcp` is **code-mode**: three generic tools — `run_stitch`, `list_stitches`, +`describe_stitch` — rather than one tool per endpoint. That is the compact-and-dangerous row of +the table above, so the two boundaries are worth separating, because they landed differently. + +### The credential boundary held + +This is the product's central promise and it survived the sharpest test available. Across **34 +JSON-RPC exchanges and 30 payload scans (14,529 bytes)** — `initialize`, `tools/list`, +`describe_stitch` on all ten stitches, successful calls on `bearer`, `apiKey` in header, query +and cookie form, `cookieSession`, a vendor 401 whose **body contained a credential-shaped +string**, a validation failure, an unknown stitch, an unknown tool, a malformed JSON-RPC method, +and the same run over stdio — **not one of the five held credentials appeared, by value, +anywhere.** + +The controls confirm the calls were real: the same exchanges put `Bearer sk_live_…`, +`X-API-Key: ak_live_…`, `api_key=ak_live_…` and `Cookie: SESSION=sess_live_…` on the wire, and +the vendor authenticated every one. + +**And the model cannot forge a header at all.** `sanitizeAgentInput` deletes `input.headers` +unless the stitch explicitly declares an `input.headers` schema: six model-supplied headers +including `authorization`, `cookie` and `host` reached the vendor as **zero** headers. Even +where an operator opts in, the credential header specifically is unforgeable, because `auth` is +applied to a clone _after_ the merge — a model-set `authorization` was overwritten with the real +token on every attempt. + + + This refuted our own starting hypothesis. `engine.ts` does merge input + headers over config headers — but on the MCP path the agent's headers are + removed before that merge ever runs. + + +### The argument boundary is yours + +Everything else in `input` reaches the request. Five levers, measured on an **ordinary stitch**: + +| Lever | Measured | +| ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | +| **A query parameter pinned in the path is a _default_** | `path: '/v1/orders?tenant=acme'` → the model sent `tenant=globex`, **and the vendor returned the other tenant's data** | +| **The whole request body of a write**, when no `body` schema | a 999,999 refund | +| **Reserved expansion traverses endpoints** | `{+id}` reached `/v1/api-keys` **with the bearer token attached**; ordinary `{id}` correctly encoded it to `..%2F..%2F` | +| **A templated endpoint reaches anywhere** | `url: '{+endpoint}'` → `https://metadata.internal/latest` | +| **`cookieSession` joins where `apiKey` replaces** | with headers opted in: `SESSION=attacker; SESSION=sess_live_…` — a vendor reading the first pair runs as the model's session | + +The first row is the sharpest, because `?tenant=acme` in a configured path reads like an +operator invariant and is spelled like one. It is `{ ...predefined, ...input.query }`. + +**A declared input schema does not close it**, and this is the finding most likely to surprise: +`validateInput` throws on failure but **discards the parsed value**, so a schema that strips +unknown keys — the default in Zod, Valibot and ArkType alike — does not strip them from the +request. Measured: a `query` validator that returned `{ limit: 10 }` still put +`?tenant=globex&limit=10` on the wire. A schema here is a **check, not a filter**. + +### The safe exposure is 47 lines across 3 seams + +No fork, no config key. Replayed against the same vendor, the naive exposure sent +`?tenant=globex`, `?include=internal_notes` and a 999,999 refund; the safe one sent +`?tenant=acme&limit=5` and nothing else, refused the POST with a reason the model can read, and +still authenticated every read. + +```ts +const server = createMcpServer( + expose({ getOrder: only(getOrder, { params: ['id'], query: ['limit'] }) }), + { adapter: readsOnly(fetchAdapter) }, +); +``` + +- **`expose`** — the registry object you hand `createMcpServer` _is_ the allow-list. It also has + to reject a stitch whose configured `name` differs from its key, because `selectStitch` falls + back to that name: a renamed stitch stays callable while vanishing from `list_stitches`. +- **`only`** — a `Proxy` apply-trap that **rebuilds** the input from an explicit key list before + the engine sees it. This is what the discarded-parsed-value finding forces. +- **`readsOnly`** — an `Adapter` wrapper, the last seam before the transport, refusing non-GET. + +`throttle` and `circuit` both apply on the MCP path, because `run_stitch` calls the stitch and +the stitch _is_ the engine: `throttle: '50/s'` paced ten tool calls; `circuit: { failures: 3 }` +turned twenty tool calls into three vendor requests and seventeen fast-fails. + +## What StitchAPI does not solve + +1. **An error message can carry a credential out.** `run_stitch` renders `(e as Error).message` + unfiltered, so any text the _transport_ writes reaches the model verbatim. With + `apiKey({ in: 'query' })` on the default adapter, a DNS failure put + `…/v1/metrics?api_key=ak_live_…` into the model's context — **from zero lines of user code**. + The fix is `apiKey({ in: 'header' })`; the [auth guide](/docs/reference/auth-strategies) already + warns that a key in a URL leaks wherever URLs go, and the model's context is one more place + URLs go. +2. **There is no confirmation seam, in either direction.** The server advertises only + `capabilities: { tools }` and cannot originate a message, so it cannot ask. The tool + descriptors carry no `annotations`, so `readOnlyHint`/`destructiveHint` are absent and the + host cannot decide to prompt — and because code-mode puts every endpoint behind one tool + name, reading an order and issuing a 25,000 refund arrive at the host as the same + `run_stitch` call. User code can **refuse** (`hooks.onRequest` throws; the vendor got zero + requests, and `retry: { attempts: 3 }` asked the gate exactly once) but never **ask**. +3. **One tool call is not one request.** `retry: { attempts: 5 }` made five; `paginate` made + twelve, with a default ceiling of 50 — and neither is signalled in the tool result. A host + that budgets 20 tool calls has budgeted up to 1,000 vendor requests. +4. **No bound expresses cost.** Every limiter is a count (`throttle.rate`, `retry.attempts`, + `paginate.pages`, `circuit.failures`) or a duration (`timeout`, `circuit.cooldown`). Nothing + expresses tokens, bytes or money — the axis the runaway-bill incidents actually ran along. +5. **`describe_stitch` is a map.** Per stitch it discloses the full internal endpoint URL, the + surface, the auth _scheme_, which resilience features are on, and a Mermaid diagram — about + 1 KB. Not the credential, not your configured request headers, not the env var name. Whether + that is disclosure or documentation depends on who is connected. +6. **`stitch mcp --module ./stitches.ts` exposes everything.** `collectStitches` sweeps up every + exported stitch — in our fixture, a write and a login stitch alongside the intended read. The + registry filter is a real seam, but the documented starter does not use it. + + + Nothing in `StitchConfig`'s 44 top-level slots excludes a stitch from MCP. + The nearest-looking key, `sensitive: true`, is a **cache** opt-out — a + stitch carrying it was still listed and still ran. + + +## See also + +- [The MCP surface](/docs/surfaces/mcp) — the three tools and the registry +- [Auth strategies](/docs/reference/auth-strategies) — why `{ in: 'header' }` beats `{ in: 'query' }` +- [Scenario: the webhook you cannot verify](/docs/scenarios/webhook-receipt) — the other surface + where the caller is untrusted +- [Scenario: one tenant takes the others down](/docs/scenarios/multi-tenant-blast-radius) — + keying limiters, which is the containment half of this problem diff --git a/apps/docs/content/docs/scenarios/async-job-polling.mdx b/apps/docs/content/docs/scenarios/async-job-polling.mdx new file mode 100644 index 00000000..ac6aab82 --- /dev/null +++ b/apps/docs/content/docs/scenarios/async-job-polling.mdx @@ -0,0 +1,154 @@ +--- +title: 'Submit, poll, download — the async job triangle' +description: 'A 202 with a Location header, a status endpoint that reports failure at HTTP 200, and a single-use result URL. Three endpoints and a loop, and you have to pick which guarantee you keep.' +prerequisites: ['/docs/reference/surfaces', '/docs/guides/resilience/timeout'] +--- + +## The problem + +You ask for something slow — a Salesforce Bulk export, a report render, a transcode. The API +answers **`202 Accepted`** with a `Location`, and you come back later. That is the +[asynchronous request–reply pattern](https://learn.microsoft.com/en-us/azure/architecture/patterns/asynchronous-request-reply), +and it is not one call but three endpoints with a loop between them: + +1. `POST /jobs` → `202`, `Location: /jobs/{id}`, often `Retry-After` +2. `GET /jobs/{id}` → repeatedly, until the state is terminal +3. `GET ` → the payload, often a pre-signed link that expires or is single-use + +Each step brings its own difficulty. **The next URL is in a response header**, not the body. +**Terminal state is in-band** — Salesforce runs `InProgress → JobComplete | Failed`, all at +HTTP 200, so failure arrives as a successful response. **The wait is the server's to set**, +via `Retry-After`, over minutes to hours. **The budget spans the whole triangle**, not one +call. And **a restart loses the job**, which is still running server-side — resubmitting +duplicates hours of work. + + + The tell that this has no easy answer: in + [jsforce#298](https://github.com/jsforce/jsforce/issues/298) the poll + timeout is **hardcoded**, and the documented workaround is to turn the + helper off and write the loop yourself. Same request in + [go-salesforce#139](https://github.com/k-capehart/go-salesforce/issues/139) + and + [salesforcer#13](https://github.com/StevenMMortimer/salesforcer/issues/13). + + +## The common solutions + +| Approach | Where it breaks | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | +| The SDK's built-in waiter | Hardcoded timeout. Fine until the job is big, then unfixable without abandoning the helper. | +| Hand-rolled `while` + `sleep` | Correct and universal. Timeout, circuit breaking and tracing see three unrelated calls, not one operation. | +| Fixed-interval polling | Hammers the API for hour-long jobs and ignores `Retry-After`. | +| Exponential backoff | The right default with no `Retry-After` — but uncapped, the last gap overshoots the finish by minutes. | +| Webhook callback | Strictly better where offered, and _additional_ work: you still need a fallback poll for missed deliveries. | +| Queue + separate worker | The production answer for hour-long jobs, and the only one that survives a restart. Costs infrastructure. | + +## What StitchAPI does + +There is no poll primitive. Polling is spelled as **retry, where the failure is "not done +yet"** — a custom [surface](/docs/reference/surfaces) whose `interpret` reads the in-band +state: + +```ts +export function jobPollSurface(clock: Clock): Surface { + return { + id: 'job-poll', + interpret: (res, cfg) => { + // verdictOf FIRST, or a 404 comes back as a successful poll. + const failed = verdictOf(res, cfg); + if (failed) return failed; + + const state = stateOf(res.body); + if (state === 'InProgress') { + const after = retryAfterMs(res.headers['retry-after'], clock); + return after === undefined + ? { ok: false, retry: true, message: 'InProgress' } // capped expo fallback + : { ok: false, retry: true, message: 'InProgress', after }; + } + if (state === 'Failed') + return { + ok: false, + message: `job failed: ${errorMessageOf(res.body)}`, + }; + return { ok: true, data: res.body }; + }, + }; +} +``` + +Three stitches then run under [`linked`](/docs/reference/helpers), which chains them into one +trace, with a caller-owned `AbortSignal` as the operation deadline. + +**Measured:** 1 submit → 5 polls at the server's own 300 s pacing → 1 download, 20 virtual +minutes, `Failed` terminating on the first terminal body with 17 of 20 poll attempts unspent. +`linked` produced **one traceId across three spans**, each parented to the last. A crash +mid-poll resumed after 3 polls with **1 submit total**. + +## The trade you have to make + +This is the part worth knowing before you start. There are two constructions, and **you can +have one deadline over the whole triangle, or per-hop retry policies — not both.** + +| | One stitch (hook rewrites the URL) | Three stitches under `linked` | +| -------------------------- | ----------------------------------------------------- | -------------------------------------------------- | +| Deadline over the triangle | **`timeout.total`** — measured 253 ms | caller-owned `AbortSignal` on every `input.signal` | +| Per-hop retry policy | **no** — one stitch is one `retry` block | **yes** — 20 poll attempts, 1 download attempt | +| Single-use result URL | burned all 8 shared attempts on the dead link | download retried exactly once | +| Trace | 1 span, `attempts: 5` — the three endpoints invisible | 3 spans, `submit → poll → download` | +| Concurrency | **unsafe** — see below | safe | + +The one-stitch form buys the config-level deadline and loses the per-hop split; `linked` keeps +both of those and replaces the deadline with a signal you own. + +## StitchAPI vs the common solution + +The hand-rolled `while` loop produces a **byte-identical** request sequence and pacing, in +**49 lines against 110**. What the extra lines buy was measured, not asserted: one `start` and +one `done` per hop with the polls folded in as `attempts: 3` rather than three unrelated calls, +one traceId chaining `job-submit → job-poll → job-download`, and a per-hop retry policy. All of +the _semantics_ — the state machine, the pacing, the resume — are still yours either way. + +If you don't need the trace or the per-hop policies, the `while` loop is the honest answer. + +## What StitchAPI does not solve here + +1. **No poll or until primitive.** Nothing in the config vocabulary waits for a state. +2. **`retry.respect` does not reach the body-driven path.** With the server asking for 30 s, + the measured gaps were **7 ms** — the computed backoff. The engine reads `Retry-After` only + on the status-driven path; a surface must pick the header up itself. +3. **`parseRetryAfter` is not exported.** The HTTP-date form has to be re-implemented by every + surface author, and if you don't, the server's pacing is discarded silently. +4. **No operation-scoped deadline field.** `linked` takes a body and nothing else. The budget + is an `AbortSignal` you build and thread through every `input.signal`. +5. **No operation-level span.** `linked` emits nothing of its own, so a mid-operation failure + is attributable to the _step_, never to "the export failed". +6. **Nothing persists the job id.** The store is engine state — throttle, auth, cache. Resume + is entirely yours. +7. **Every failure flattens to `StitchError` plus a string.** "Deadline fired", "poll budget + exhausted" and "the job failed" are indistinguishable by type, and `status` is `undefined` + on all three. +8. **`paginate` cannot do this** — and not for the reason you'd guess. It _does_ loop (the + default `items` wraps a non-array body as one item, so the empty-page break never fires), + but it cannot **wait**: measured gaps of `0, 0, 0`, with no delay field. A paginated poll + also cannot fail — `Failed` is aggregated as just another value. + + + **Two concurrency traps, both measured.** A stitch that rewrites its own URL + in a hook is **not safe to call twice at once**: two concurrent calls + submitted two jobs, polled the second one twice, and handed *both* callers + the second job's result — the first job ran to completion, unread. And a + poll surface without a `hooks.onRequest` guard **re-submits**: 4 POSTs, 4 + duplicate jobs, under `attempts: 4`. Build one stitch per in-flight job. + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/batch-partial-failure.mdx b/apps/docs/content/docs/scenarios/batch-partial-failure.mdx new file mode 100644 index 00000000..c2215277 --- /dev/null +++ b/apps/docs/content/docs/scenarios/batch-partial-failure.mdx @@ -0,0 +1,143 @@ +--- +title: 'Batch writes that fail one item at a time' +description: 'A bulk endpoint returns 200 and reports that 7 of your 100 items did not land. Retrying the request re-writes the 93 that did — so the retry unit has to be the body, not the call.' +prerequisites: ['/docs/guides/resilience/retry', '/docs/reference/surfaces'] +--- + +## The problem + +You write in bulk — DynamoDB `BatchWriteItem`, Elasticsearch `_bulk`, SQS `SendMessageBatch`, +Salesforce sObject Collections. One request carries 25 or 1,000 items, the API answers **HTTP +200**, and inside the body it says some of them didn't land. + +**The retry unit is smaller than the request.** Every HTTP client retries by replaying the +identical request, which here re-applies the writes that already succeeded. The correct +behaviour is to rewrite the body to the failed subset, resend, wait longer each round, and +report whatever never landed. + +Three things make it worse than it sounds: backoff is **mandatory** (AWS is explicit — the +cause is capacity, so an immediate resend throttles again), failures are **not uniform** +(a `429` item should be resent, a `400` mapping error must not be), and the loop needs to +hand back **the residue** — the caller needs the items, not just an error. + + + This is the failure mode behind + [`elastic/logstash#1631`](https://github.com/elastic/logstash/issues/1631) — + "rejected docs in bulk indexing partial failure are **silently lost**" — and + [`elasticsearch-py#1004`](https://github.com/elastic/elasticsearch-py/issues/1004), + where errors are aggregated *without their data*, so you cannot tell which + items to resend. + + +## The common solutions + +| Approach | Where it breaks | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| The client's built-in retry | Replays **all** items. Fixes 7 by re-writing 93 — duplicate side effects on a non-idempotent endpoint. | +| Hand-rolled `while` loop | Correct, and what most teams write. Lives outside the client, so timeout, circuit breaking and tracing stop seeing the real call. | +| Vendor SDK helper | Only where an SDK exists, and the policy is theirs — `streaming_bulk` retries `429` only, and drops the failed items' data. | +| Check the status, move on | The Logstash bug. Silent data loss, found later by absence. | +| One request per item | Trivially retryable, at 100× the requests the batch endpoint existed to avoid. | + +## What StitchAPI does + + + **Not `paginate`, however much it looks like the answer.** `paginate.next` + genuinely expresses the residue resend — measured: 3 requests for 6 items, + **zero** duplicate writes. Then it loses your data. A round in which + *nothing* lands aggregates zero items, and the loop treats that as the end: + measured **4 of 6 rows never written, `ok: true`, no error**. That is the + ordinary response from a table that is out of capacity. Hitting the `pages` + cap also returns `ok: true`, so "finished" and "gave up" are the same value + — and a residue ledger built inside `next` is stale by one round, naming an + item that already landed. + + +The seam that works is `Surface.interpret` — which reads the 200 body and asks for another +attempt with a wait that grows — paired with **`hooks.onRequest`, the only place in the library +that can change a request between attempts**: + +```ts +const kind: Surface = { + id: 'batch-residue', + interpret: (res, cfg): SurfaceOutcome => { + // A 500 is a transport failure before it is a batch envelope — and must still + // open the circuit. Let the declarative verdict compose first. + const failed = verdictOf(res, cfg); + if (failed) return failed; + + ledger.rounds += 1; + ledger.landed.push(...landedOf(res.body)); + ledger.terminal.push(...terminalOf(res.body)); // 400s — never resent + ledger.residue = residueOf(res.body); // 429s / UnprocessedItems + + if (ledger.residue.length === 0) return { ok: true, data: ledger }; + if (ledger.rounds >= rounds) { + // Out of rounds. Resolve SUCCESSFULLY with the residue in the payload — + // an error would throw the landed items away, and dropping it is the Logstash bug. + ledger.gaveUp = true; + return { ok: true, data: ledger }; + } + return { ok: false, retry: true, after: backoff(ledger.rounds) }; + }, +}; + +const hooks = { + onRequest: (ctx) => { + if (!ctx.req || ctx.attempt === 1) return; + // Assign, never mutate: each attempt's request is a shallow clone of one baseReq, + // so an in-place edit of `body` rewrites the caller's own array too. + ctx.req.body = bodyOf(ledger.residue); + }, +}; +``` + +**Measured** against a capacity-limited table: 4 rounds at t = 0, 1000, 3000, 7000 — +`abcdef → cdef → def → f` — **zero duplicate writes**, every item landed, and the wait is +the engine's own sleep rather than a hidden one. It survives three consecutive zero-progress +rounds, which is precisely the case `paginate` drops. + +## StitchAPI vs the common solution + +**It is more code, not less** — 50 lines against 28 for the hand-rolled `while` loop. The +trade is what the hand-rolled loop gives up, and this was measured rather than assumed: + +| | hand-rolled loop | on the surface seam | +| --------------------------------- | --------------------- | ------------------------------------------------- | +| `start` events for one logical op | 3 | **1** | +| reported `attempts` | 1, three times over | **3** | +| `retry` events | 0 | **2**, with detail | +| circuit breaker | never sees the rounds | **opens after 2 × 500** | +| `timeout: { total: 100 }` | bounds each round | **bounds the operation** — cut at round 2, 101 ms | + +If you don't need any of that, the `while` loop is honestly the smaller answer. Reach for this +when the batch call has to behave like one call to everything else in your system. + +## What StitchAPI does not solve here + +1. **There is no batch-residue concept.** Nothing in the config vocabulary expresses "the retry + unit is smaller than the request". Everything above is assembled. +2. **`retry` is blind to per-item failure, and harmful when forced.** The status is 200, and + `retry.on` receives only the status. `retry.on: 200` typechecks, fires — and made **10 + duplicate writes on a batch that never failed at all**. +3. **`paginate` has no wait and no field to declare one.** Six rounds fired at t=0. `throttle` + paces them, but as one fixed ratio with no curve — and it paces every other call through + that stitch (`pool: 'host'` pushes unrelated reads too). +4. **A growing backoff is user code the engine can't see.** A sleep in `onRequest` works, but + 2.5 s of real waiting produced **0 `throttled` events** and no `waited` in the run report. +5. **`SurfaceOutcome` cannot see the request or the attempt number**, so a surface can neither + rewrite what it retries nor know it is on its last round. The count has to be kept by hand. +6. **The engine owns no residue channel.** Not the result, the error, `.inspect()`, `.report()`, + the event stream, or a trace sink. If you don't capture it yourself, it is gone. +7. **The ledger is per-stitch, not per-call.** Two concurrent calls through one such stitch + corrupt each other — measured: both callers told everything landed, two rows written by + nobody. Build one stitch per in-flight batch, or add your own scoping. + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/conditional-requests-304.mdx b/apps/docs/content/docs/scenarios/conditional-requests-304.mdx new file mode 100644 index 00000000..f7cce776 --- /dev/null +++ b/apps/docs/content/docs/scenarios/conditional-requests-304.mdx @@ -0,0 +1,149 @@ +--- +title: 'The free poll — ETag revalidation and the bodyless 304' +description: 'A 304 means "use what you have", carries no body, and is not a 2xx. Turning it back into the resource takes one seam — and the cache primitive cannot help.' +prerequisites: + ['/docs/reference/surfaces', '/docs/guides/state/pluggable-store'] +--- + +## The problem + +You poll for changes and nothing has changed. HTTP has a way to make that free: keep the +`ETag`, send it back as `If-None-Match`, and the server answers **`304 Not Modified`** with no +body. On GitHub a 304 **doesn't count against your primary rate limit** — 600 polls, 90% +unchanged, cost 60 requests. + +The awkwardness is that a 304 is a _status_ meaning "use what you have", and both obvious +readings are wrong. Treat it as a failure and every unchanged poll is an error; treat it as a +success and the caller gets `undefined` where the resource should be. The only correct +behaviour is to **substitute the previously cached body**, which means the cache and the +request path have to know about each other. + +Four constraints most clients get wrong: + +- **ETags are per-credential.** GitHub caches them per token. A store keyed only by URL will + replay one principal's validator for another. +- **ETags are per-page, not per-collection.** A 304 on page 1 of 5 says nothing about 2–5. +- **Weak validators compare weakly.** `W/"abc"` and `"abc"` are not interchangeable, and the + comparison is the _server's_ to make — so the client must not normalize the tag. +- **Some servers never match.** Apache's default ETag embeds the file inode, so behind a load + balancer revalidation never succeeds and the feature silently does nothing. + +## The common solutions + +| Approach | Where it breaks | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| TTL cache only | Never revalidates. Every refresh is billed **and** the data is stale up to the TTL — you pay full price for staleness. | +| Hand-rolled ETag store | Correct, and what most teams write. Easy to key wrong; the swap must happen below parsing and validation. | +| An HTTP caching proxy | The most standards-correct answer. Adds a dependency or a hop. | +| Treat 304 as an error and retry | Actively wrong — turns the success case into an error and re-sends the same validator. | +| Ignore conditional requests | What most integrations do. On GitHub it costs 10× the rate-limit budget for identical data. | + +## What StitchAPI does + +There is no conditional-request feature — `If-None-Match` and `304` appear nowhere in the core. +What there is, is the one seam that owns a request _and its own response_ in a single function: +**`Surface.execute`**. + +```ts +execute: async (req, ctx) => { + const key = `${req.method} ${req.url}|${credentialOf(req)}`; // per-credential, see below + const cached = store.get(key); + if (cached) req.headers['if-none-match'] = cached.etag; // byte-exact, never normalized + else clearValidator(req); // a re-attempt must be able to DROP it + + const res = await ctx.adapter(req); + if (res.status === 304 && cached) return { ...res, body: cached.body }; + + const etag = res.headers['etag']; + if (etag) store.set(key, { etag, body: res.body }); + return res; +}, +``` + +It needs **no custom `interpret`**. The substituted body rides back on a response whose status +is still 304, and `classifyStatus` only fails at `>= 400` — so `.inspect().status` honestly +reports 304 while `.data` is the resource. + +**Measured**, ten polls at one-minute intervals with the resource changing once before poll 6: + +| | billed | versions seen | +| ------------------ | ------ | ----------------------------------------------------------- | +| no caching | 10/10 | correct, never stale | +| TTL cache (30 min) | 1/10 | `[1,1,1,1,1,1,1,1,1,1]` — **never sees the change** | +| revalidation | 2/10 | `[1,1,1,1,1,2,2,2,2,2]` — picked up on the poll it happened | + +Eight of ten polls became free with **zero** staleness. That middle row is the argument: a TTL +cache is cheaper _and_ wrong, serving a superseded version on 5 of 10 polls. + + + `interpret` **does** run on a 304 — measured with a counter across `[200, + 304, 404]`. The engine interprets every response, not just 2xx. (The + "`interpret` never runs" finding on [the streaming + page](/docs/scenarios/mid-stream-failure) is specific to streams.) So + `Surface.interpret` + `hooks.onRequest` is a valid second route; `execute` + is just fewer moving parts. + + +## StitchAPI vs the common solution + +**The StitchAPI version is longer — 87 lines against 79** for a feature-matched hand-rolled +twin, and both produce identical versions, statuses and billed counts on all four shapes +(quiet, changed, weak validators, never-matching). So the extra lines are not buying +behaviour. + +They attribute exactly: `credentialOf` and `clearValidator`, two helpers that exist only +because the engine hands a surface a shared header record it never case-folds. What the +hand-rolled twin lacks is everything that stayed _configuration_ on the stitch — `auth`, +`output`, `retry`, `timeout`, `seam.as()` and the trace spine. Every one of those would have to +be written into the hand-rolled file to match. + +## What StitchAPI does not solve here + +1. **`cache` cannot revalidate — at all.** It is a _value_ store, not a response store: the + entry is `{ v, s, vary }` and what gets written is the post-`interpret`, post-validation + value. No response header, and therefore no ETag, can reach it. +2. **`revalidateOnHit` is a false friend.** It re-checks the stored value against the `output` + **schema**, never the network. +3. **A cache hit short-circuits everything below it.** Measured over 3 calls: hooks fired once, + `interpret` ran once, one request. The hit spine is `[start, cache:hit, result, done]` — no + `request` phase at all — so revalidation can't run under a hit. +4. **The cache cannot store a 304 either.** Forced into its own key, three conditional calls + measured `[undefined, undefined, undefined]` — a stored `undefined` reads as a permanent miss. +5. **The one workaround turns caching off.** Folding the ETag into the value via `transform` + makes the stitch un-fingerprintable, so it fails closed and refuses to cache. +6. **A surface cannot see the bound principal.** `ResolvedStitchConfig` has no `principal`, and + `buildRequest` runs _before_ `auth.apply` — so it can't even read the credential header. Only + `hooks.onRequest` and `Surface.execute` are downstream of auth. +7. **`buildRequest` runs once per run, not per attempt.** A validator set there is baked into + every retry — measured 3 identical validators across 3 attempts and a failed run. +8. **Adding an `output` schema to a bare conditional poll breaks it.** The empty 304 body fails + the contract: `ok: false`, `contract violation (drift)`. With substitution in place the + schema is handed the real body and never sees `undefined`. +9. **`cache.ttl` ignores an injected `clock`** — it reads `Date.now()`. Measured: one request + after advancing a `manualClock` by a virtual hour against a 1-second TTL. + + + **The cross-principal leak, and why it hides.** Against a server with content-derived + validators, an ETag store keyed on `METHOD URL` alone leaked: one store entry, and **bob + received alice's data** (`viewer: tok-alice`). The reason it survives review is that the + rate-limit metrics *improve* while it happens — a 50% 304 rate is exactly what a working + revalidator looks like. `cache.tenancy` does not protect a store you wrote yourself. Put + the credential in the key, as above. + + **And the silent no-op.** Against a server minting a fresh validator per response (the + load-balancer inode case), 10 polls billed 10, got **zero** 304s, and raised nothing. + Assert on it: `revalidated === 0` while `stored === 10` is the only signal you get. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/cost-based-rate-limits.mdx b/apps/docs/content/docs/scenarios/cost-based-rate-limits.mdx new file mode 100644 index 00000000..3e712571 --- /dev/null +++ b/apps/docs/content/docs/scenarios/cost-based-rate-limits.mdx @@ -0,0 +1,135 @@ +--- +title: 'Rate limits priced in query cost' +description: 'Shopify bills per query cost, answers 200 OK when you overspend, and puts the wait in the body. Why status-code retry and rate-per-second both miss, and what does work.' +prerequisites: ['/docs/guides/data/graphql', '/docs/guides/resilience/retry'] +--- + +## The problem + +Shopify's GraphQL Admin API meters a **1,000-point bucket refilling at 50 points/second**, +and each query has its own price — 11 points for one, 900 for another. Four properties each +defeat a different standard tool: + +1. **The unit is cost, not requests.** No single requests-per-second is correct for both an + 11-point query and a 900-point one. +2. **Overspending answers `200 OK`.** The failure arrives in the body as a `THROTTLED` entry + in `errors[]` — never a `429`. Retry policies keyed on status see success. +3. **The wait is arithmetic, not a guess.** Every response carries + `extensions.cost.throttleStatus`; the correct wait is + `(requestedQueryCost − currentlyAvailable) / restoreRate`. A backoff curve over-waits when + the bucket is full and under-waits when it is empty. +4. **The bucket belongs to the shop, not to you.** Another app draining it moves your + headroom between two of your own requests + ([shopify-api-js#602](https://github.com/Shopify/shopify-api-js/issues/602)), so a + local ledger can never be authoritative — it must be overwritten from every response. + +## The common solutions + +| Approach | Where it breaks | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| Retry on 429 + exponential backoff | Never fires. The response is a `200`, and the THROTTLED envelope is returned as data. | +| Body-sniffing retry, then exponential backoff | Fires correctly, then ignores the arithmetic the server already supplied. | +| Compute the wait from `throttleStatus` | Correct — but needs `extensions`, which GraphQL clients discard when they unwrap `data`. | +| Local cost ledger, pause below a threshold | Paces your own traffic; blind to other apps on the same shop. | +| Fixed rate limiter (`N/sec`) | Wrong unit. Sized for the worst query it wastes the quota; sized for the average it throttles. | +| Single-worker global queue | Correct and common. Costs concurrency and a piece of infrastructure. | + +## What StitchAPI does + +Not `retry`, and not `throttle` — measured, both miss, and the section below says exactly how. +The seam that fits is a **custom [surface](/docs/reference/surfaces)**: `interpret` sees every +response body before the engine decides anything, and the `SurfaceOutcome` it returns can ask +for a retry **after a wait you computed**. + +```ts +import { graphqlSurface, verdictOf } from 'stitchapi'; +import type { Surface, SurfaceOutcome } from 'stitchapi'; + +export function shopifyCostSurface(ledger: CostLedger): Surface { + return { + id: 'graphql', + buildRequest: graphqlSurface.buildRequest, + interpret: (res, cfg): SurfaceOutcome => { + const failure = verdictOf(res, cfg); + if (failure) return failure; + + // EVERY response updates the budget — successes carry throttleStatus too, + // and the server's number is authoritative because the bucket is shared. + const cost = costOfBody(res.body); + if (cost) ledger.record(cost); + + // The 200-with-THROTTLED, and the wait the server's own arithmetic dictates. + if (isThrottled(res.body) && cost) + return { + ok: false, + retry: true, + message: `THROTTLED — need ${cost.requestedQueryCost}`, + after: deficitWaitMs(cost), // ← (requested − available) / restoreRate + }; + + return ( + graphqlSurface.interpret?.(res, cfg) ?? { + ok: true, + data: res.body, + } + ); + }, + }; +} +``` + +Pair it with an `onRequest` hook that pauses while the ledger says the next query is +unaffordable, and the reactive half only handles what the proactive half cannot predict — +another app draining the shop. + +**Measured:** the computed wait was honored exactly (**6000 ms**, succeeding on attempt 2, +where the built-in curve waited 100 ms and failed all three attempts). Against a neighbour +emptying the bucket before every single call, **8/8 queries succeeded**, absorbing 8 +throttles. The whole thing is **73 lines**. + +## StitchAPI vs the common solution + +The retry loop stays the engine's. Because the cost logic lives in `interpret` rather than in +a wrapper around the call, `timeout.total` still bounds the whole thing, the circuit breaker +still counts failures, and every wait shows up as a `retry` progress event on the +[event stream](/docs/concepts/event-stream). The obvious alternative — wrapping the adapter — +was measured going blind: a call that made 2 requests and slept 6 seconds reported +`attempts: 1` and emitted **zero** retry events. + +## What StitchAPI does not solve here + +1. **`retry.on` cannot see the body.** The predicate receives exactly one argument, the status + number. A 200-with-THROTTLED is invisible to every built-in retry policy. +2. **`retry.on: 200` retries your successes.** The status matcher runs before `interpret` and + cannot tell the two apart — measured 3× the requests and 3× the points for one result. +3. **`backoff` has no function form.** A wait computed from the payload cannot enter through + it. Cast past the type error and it is **silently ignored**, degrading to the default curve. +4. **`retry.respect` reads a `Retry-After` header** that Shopify never sends. +5. **`throttle.rate` cannot express cost.** Requests-per-interval only, minimum spacing, no + burst. Approximating the 1,000-point bucket took **18 s** for work the bucket absorbs + instantly — and with mixed costs no single spacing is correct for both. +6. **`throttle.delegate` is status-keyed.** The escape hatch the throttle docs point at for + vendor-accounted quotas does not reach a body-reported one. +7. **A rejected body never reaches the caller.** On a surface failure the `StitchError` has no + `body`, the error event has no field for one, and `.inspect().raw` is `null`. Whatever you + need from the body must be captured inside `interpret` or `hooks.onResponse`. +8. **`interpret` is synchronous.** A _distributed_ cost ledger cannot live in the seam that + otherwise solves this — single process only. + + + **Do not reach for `verdict.flag` here.** It is the one built-in that reads + the body for a verdict, so it is the natural guess — and on a throttled + Shopify response it is **inert**: the payload has no `data` key, an absent + path is "no signal", the 200 stands, and the call returns **`ok: true` with + the THROTTLED envelope as your data**. Silent, and it looks like a + successful sync. + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/deprecation-headers.mdx b/apps/docs/content/docs/scenarios/deprecation-headers.mdx new file mode 100644 index 00000000..e2afb6fe --- /dev/null +++ b/apps/docs/content/docs/scenarios/deprecation-headers.mdx @@ -0,0 +1,132 @@ +--- +title: 'The vendor told you for six months, in a header' +description: 'Deprecation and Sunset arrive on responses that succeeded, so nothing fails and nothing retries. Response headers are reachable in exactly three places — here is the table.' +prerequisites: ['/docs/reference/surfaces', '/docs/guides/authoring/hooks'] +--- + +## The problem + +A vendor is retiring the endpoint you depend on. They announced it in a blog post, sent one +email, and — if they follow the standards — have been telling you **on every single response** +for six months, via `Deprecation` (RFC 9745) and `Sunset` (RFC 8594). + +**The signal arrives on responses that succeeded.** Nothing failed, nothing retried, no status +changed — so every mechanism a client has for noticing trouble points the wrong way. As one +write-up puts it: _"The clients that broke never read the blog post — but their code reads your +HTTP responses on every single request."_ + +Three specifics make it awkward: **two headers, two formats** (`Deprecation` is a +structured-field date `@1735689600`, `Sunset` is an HTTP-date); **it is a hint, not a +guarantee**, so failing the call is wrong and ignoring it is also wrong; and **the useful unit +is the fleet** — "which of my forty endpoints are deprecated, and which sunsets first" — which +no per-call log line can answer. + +## Where a response header is reachable + +This is the table worth keeping. Measured across every accessor on a **successful** call: + +| | carries response headers | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| `adapter` | **yes** — it built the response, but knows no stitch name and can't change the result | +| `hooks.onResponse` | **yes** — the whole `AdapterResponse`, plus `ctx.name` | +| **`Surface.interpret(res, cfg)`** | **yes** — and it returns the value the call resolves to | +| `await` / `.unwrap()` / `.safe()` | no | +| `.inspect()` (5 keys) / `.report()` (9 keys) | no — `status`, but no headers | +| `StitchError` (5 keys) | no | +| `transform` | no — its one parameter is the body | +| the **entire event spine** — 4 events, 15 distinct keys | **no** | + +That last row is the one with consequences: because no event carries a header, a `TraceSink` +inherits the same hole. It can only aggregate what a `Surface` folded into the value. + +`Surface.interpret` is the only seat where a header **and** the returned value are in scope +together, which makes it the answer to this scenario and to any other "the signal is in a +header" problem. + +## What StitchAPI does + +**Not one config key knows what a `Deprecation` header is.** The answer is three seams and +about 112 lines: `Surface.interpret` reads the headers and renders a verdict, a seam-level +`trace` sink aggregates by `ctx.name`, and the injected `clock` makes the sunset crossing +deterministic. + +**The fleet view works, and the [scenario-12](/docs/scenarios/intermittent-drift) shape +transfers.** One sink configured once on a seam, 500 calls across 5 endpoints: + +``` +3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users +``` + +One row per endpoint however many calls arrive — and with traffic skewed **200:1** toward the +healthy endpoints the report was unchanged, which is exactly what a per-call log line cannot do. + +**And the tripwire is elegant.** `interpret`'s `{ ok: false }` arm plus `cfg.clock` makes the +crossing deterministic to the millisecond — `[sunset−1ms, sunset, sunset+1ms]` measured +`["ok", "FAILED", "FAILED"]`. Two things make it safe to deploy, both measured: it does **not** +burn retry attempts (5 configured, 1 request made) and it does **not** open the circuit breaker +(5 consecutive trips past `failures: 2`, every message still the real one), because +`classifyStatus` rules on the status and the transport was healthy. + +## What StitchAPI does not solve here + +1. **No config key, and no reachable parser.** `parseRetryAfter` handles delta-seconds **or** an + HTTP-date against an injectable clock and returns ms-until — precisely the `Sunset` + requirement — and it is not on any of the 17 published subpaths. The public barrel exports + `parseDuration`, `parseBytes`, `parseRate`; none parses a date. +2. **No event carries headers**, so a `TraceSink` sees `null` for a response that carried both — + unless a `Surface` folded them into the value. And then an ordinary `output` contract that + doesn't declare the folded field **deletes it again** before the `result` event fires, with no + warning. +3. **No API mints a levelled finding.** A `Validator` returns a value or _issues_, and an issue + is fatal. The one door onto the drift channel is folding a field into the value and letting an + `output` contract report it `info | undeclared` — which is non-fatal and re-levellable, but + **carries neither the header value nor the endpoint**. (Smuggling the date into the _path_ + works — `_sunset_2026-01-01` — at the cost of a new finding path per date.) +4. **No de-duplication primitive.** 600 calls produced **360 log lines carrying 3 distinct + facts**; the built-in `loggerSink` is louder at 2400. `levelOf` can drop events but cannot + answer "have I said this already" — it's a pure function of one event. A 3-line `Set` latch + fixed it; nothing on the barrel latches, samples or de-duplicates an observation. +5. **The line count goes against the library** — **132 lines against 81** hand-rolled for + identical output. Wiring alone favours it (20 vs 52); the total doesn't, because reaching + `res.headers` costs a `Surface` object and holding the per-endpoint map costs a `TraceSink` + object, where a hand-rolled client does both inline in the method that already had the + response. + + + **`hooks.onResponse` can rewrite the call, and the hooks guide says it + cannot.** The guide's "hooks never change what a stitch returns" is true + only of the *return value*. `ctx.res` is the engine's live object, read + again by `interpret` afterwards — so mutating `res.body` added a key to the + caller's value, mutating `res.status` turned the vendor's **200 into a + thrown `HTTP 503`**, and mutating `res.headers` made a surface read + `"REWRITTEN BY HOOK"` instead of the real `Sunset`. + + + + **A seam-level `kind` is a compile error that works perfectly at runtime.** `TS2353: 'kind' + does not exist in type 'SeamOptions'` — yet members inherited the surface and folded + correctly. A typed codebase writes the surface on all 40 members for a capability that + already works from one. + + **`ctx.name` defaults to the literal `"stitch"`**, so two unnamed endpoints silently merge + into one row at the sink. **A cache hit re-serves a notice captured on the one wire + response** — 9 of 10 rows were a replayed header, and a long TTL will report a passed sunset + as "in 12 days". And **`Date.parse("@1735689600")` is `NaN`**, so the naive parser reads + `Sunset` correctly and reports *no deprecation* for the format RFC 9745 actually mandates. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/dual-run-migration.mdx b/apps/docs/content/docs/scenarios/dual-run-migration.mdx new file mode 100644 index 00000000..70c57cb4 --- /dev/null +++ b/apps/docs/content/docs/scenarios/dual-run-migration.mdx @@ -0,0 +1,149 @@ +--- +title: 'The migration you have to run twice' +description: 'Dual-running a vendor v1 and v2 when you own neither endpoint. One of five isolation channels is safe by default, and the combinator that looks built for this broadcasts one input to both.' +prerequisites: + ['/docs/guides/authoring/seam', '/docs/guides/resilience/circuit-breaker'] +--- + +## The problem + +Your vendor is retiring v1. [Scenario 17](/docs/scenarios/deprecation-headers) is how you found +out; this is what you do next. You cannot flip on faith, so you run both against real traffic, +compare, and cut over when the diff goes quiet. + +**Every shadow-traffic guide in the field is written for the service owner.** The canonical +architecture mirrors at the gateway — client → proxy → primary, with a copy to the shadow and a +shadow database beside it. You own the gateway, both services and both datastores. + +As the **consumer** of a third-party API you own none of that, and every term changes: + +- **There is no proxy to mirror at.** The duplication happens in your own client code, on the + call path, which is exactly where you cannot afford it to go wrong. +- **The shadow spends the vendor's meter.** "Sample heavily" is cost control, not statistics. +- **You cannot shadow a write.** A mirrored `POST /charges` charges twice — so the technique is + read-only, and writes are the calls you are most afraid of migrating. +- **Telling a real diff from a benign one is the actual work.** A v2 that renames `created` to + `created_at`, returns ISO instants instead of epochs and orders an array differently is + **correct**, and diffs on every call. + +## The common solutions + +| Approach | What it is | Where it breaks | +| ----------------------------- | -------------------------------------------- | -------------------------------------------------------------------------------- | +| **Mirror at the gateway** | Proxy duplicates the request. | The standard answer, unavailable when the endpoint is someone else's. | +| **Dual-call in the client** | Issue both, return v1, log the diff. | Available to you — and now the shadow is inside your latency and failure budget. | +| **Offline replay** | Capture v1 traffic, replay against v2 later. | No user impact, and no live comparison. | +| **Diff in a batch job** | Log both, compare nightly. | Cheap and slow. A regression lives a day. | +| **Trust the changelog** | Read the migration guide, flip. | Free, and the reason this scenario exists. | +| **Sample a small percentage** | Shadow 1–5% of reads. | The cost control that makes it viable. Needs a spelling. | + +## What StitchAPI does + +### The safe dual-run is 69 lines across 5 seams + +Replayed against the same vendor and the same flaky v2, the naive construction and the safe one +differ **0-of-4 versus 4-of-4** user-facing calls succeeding. The naive version — +`all([v1, v2])` under one seam, both versions on `/customers` — propagated v2's 500 to the caller +twice, then fast-failed v1 on **v2's** breaker twice, and along the way put a 90 ms shadow on a +10 ms call's critical path and sent v2's parameter name to v1. + +The safe version needed no fork and no new config key: + +- **`readsOnly` on the shadow's adapter only** — an 8-line wrapper. Three shadow write attempts + (a plain POST, an `llm`-surface call, a `.with()`-bound handle) reached the wire **0 times**, + while the primary's own POST still succeeded. +- **A seam-level `throttle`** — one bucket by default, because `seamBucket` re-keys every acquire + onto the seam id, so the meter adds up across both versions. +- **Distinct `circuit.key` strings** — the shadow tripped its own breaker and the primary never + noticed. +- **A hand-written normalizer** — 7 raw diff ops per call became exactly 1, the planted + regression, reported with both values. +- **A floated `.safe()` at the call site** — the one spelling that is simultaneously off the + critical path, unable to reject, unable to cancel the primary, and eager enough to actually + run. + +### A `url` thunk moves more than the base URL + +`url` is `string | (() => string)`, it carries the **complete endpoint**, and `{param}` +interpolation still applies to a thunk-supplied URL. One flag moved `/v1/customers/{id}` to +`/v2/customers` between calls with no redeploy. + +## What StitchAPI does not solve + +### One of five isolation channels is safe by default + +| channel | safe by default? | what it takes | +| ---------------- | --------------------------------------------- | ---------------------------------------------------- | +| **retry budget** | **yes** — a per-call loop counter | — | +| **latency** | no — `all()` cost **+109 ms** on a 13 ms call | don't await the shadow; no combinator does this | +| **thrown error** | no — `all()` threw `StitchError` | `void v2.safe(input)` | +| **cancellation** | no — the primary measured `aborted: true` | never put the shadow in `all()` | +| **circuit** | no — see below | distinct `circuit.key`; never unkeyed `pool: 'host'` | + +The cancellation channel was not on our list of things to check, and it is the most dangerous: +`all()` aborts the in-flight **primary** when the shadow settles first. + +### The circuit is a key collision, not a shared default + +More precise than "resilience state is shared." Identity is +`(store) × ('circuit:' + (circuit.key ?? name ?? path ?? 'stitch'))`. Across five configurations +the primary got `1, 1, 0, 0, 1` requests — isolated standalone and under a seam with distinct +paths, **broken** with a seam and the same path, and **broken** under `pool: 'host'`. + +Both failing cases are ordinary dual-run shapes: v1 and v2 usually share a path and differ by +base URL, and they are usually on the same host. + + + **The `pool: 'host'` trap.** It is the setting that makes cost accounting + correct across two stitches — and it silently re-keys the **breaker** onto + the host too, which is exactly the configuration measured fast-failing the + primary on the shadow's breaker. Use a seam-level throttle instead: it pools + correctly and leaves the circuit keyed per path. + + +### The rest + +1. **The combinators broadcast one input.** `all`/`any` build every member's input from the one + group input, so the shadow received `/v2/customers` — **no id**. Adding the id for v2 made v1 + send `/v1/customers/cus_7Q2?customer_id=cus_7Q2`. This is + [#643](https://github.com/rejifald/StitchAPI/issues/643), and a dual-run is the case that + needs the opposite. +2. **`.with()` survives the broadcast but binds a constant.** A group built once and called twice + left the shadow pinned to `cus_7Q2` while the primary followed `cus_ZZZ`. +3. **A bare `void v2(input)` sends zero requests.** `StitchResult` extends `PromiseLike` + (`types.ts:1962`), so nothing runs until `.then`. No request, no rejection, no + `unhandledRejection` — the dual-run silently compares nothing. Same root cause as + [#660](https://github.com/rejifald/StitchAPI/issues/660); `void v2.safe(input)` is the spelling + that works. +4. **No response-vs-response comparator is reachable.** Of 33 root exports the only + comparison-shaped one is `drift()`, which takes a _schema_. The real primitives exist — + `diff(before, after)` and `classifyDiff(a, b, opts)` — and neither is exported from any of the + 17 subpaths. A hand-written replacement is 23 lines, and is **better** for this: `classifyDiff` + renders the planted regression as `"number -> number"`, a type delta with no numbers in it. +5. **`DriftOptions.ignore` is suppression, not relevancy.** Four clauses take 7 diff ops to 1 — + and measured, the clause silencing a benign tag reorder also silences a real tag change, and + the one silencing the rename also silences a v2 reporting the wrong instant. There is no + aliasing, no unordered-array comparison, no coercion hook and no tolerance anywhere in the + tree, so a filter that still catches what it should is 24 lines of user code. +6. **Sampling is not expressible.** No sample/ratio/percent slot exists on any subpath. The + shadow doubles consumption exactly — 20 logical calls became 40 vendor requests, **2.000×** — + and 5 lines of user code took that to 1.05×. +7. **No write guard exists.** An unguarded dual-run of `POST /charges` sent **2 charges** with no + config key, type error or runtime nudge objecting. The only method-shaped option is + `cache.methods`, which gates cacheability and let the POST through. A construction-time gate on + `__config.method` is also not enough: the `llm` surface reports `method === undefined`, passes + the gate, and POSTs. The guard has to sit at the `Adapter`, below every authoring surface. +8. **Cutover cannot be one flag.** A thunk moves the URL, but the input mapping is caller-side + (after the flag, a v1-shaped input against the v2 URL silently produced `/v2/customers` with no + id and nothing threw) and `output` is resolved once at construction (the same stitch went from + `ok: true` to `ok: false` against a v1-shaped schema). The real cutover is a 4-line selector + over two whole stitches. + +## See also + +- [Scenario: the vendor told you for six months, in a header](/docs/scenarios/deprecation-headers) + — how you learn the migration is coming +- [Scenario: failing over to the backup provider](/docs/scenarios/provider-failover) — where the + one-input broadcast was first measured +- [Scenario: one tenant's revoked token, everyone's outage](/docs/scenarios/multi-tenant-blast-radius) + — the keying problem this scenario runs into again diff --git a/apps/docs/content/docs/scenarios/expiring-signatures.mdx b/apps/docs/content/docs/scenarios/expiring-signatures.mdx new file mode 100644 index 00000000..f9767271 --- /dev/null +++ b/apps/docs/content/docs/scenarios/expiring-signatures.mdx @@ -0,0 +1,122 @@ +--- +title: 'The signature that expired in your own queue' +description: 'A rate-limited queue cannot age a SigV4 signature here — the wait happens before signing, by construction. Clock drift still needs 26 lines.' +prerequisites: + ['/docs/guides/resilience/throttle', '/docs/integrations/aws-sigv4'] +--- + +## The problem + +You call a service that requires **signed requests** — S3 or any AWS API over SigV4. The +signature covers a timestamp, and the server rejects anything more than **five minutes** off +its own clock. That window exists to stop replay attacks and is not negotiable. + +There are three ways to fall outside it, and only one is your clock: + +1. **The clock drifts.** Containers inherit the host's time at start and never re-sync. Retry + makes it _worse_ — the same stale clock produces the same invalid timestamp every attempt. +2. **The signature ages in a queue — yours.** Sign, then hold: behind a rate limiter, a + concurrency cap, a backoff. AWS's own answer: _"The SDK signs the request, then puts it in a + queue. If the request is pending for more than 5 minutes, the signature expires."_ The fix + filed against botocore ([#149](https://github.com/boto/botocore/issues/149)) is to generate + the timestamp per signing operation. +3. **The retry replays a stale signature**, if signing happens once per call rather than per + attempt. + +And the compounding detail: `RequestTimeTooSkewed` is a **403**, so it reads as "auth problem, +retry it" — the failure most likely to be retried is the one retry cannot fix. + +## What StitchAPI does + +**Cases 2 and 3 need no user code, because the ordering is right by construction.** Inside the +attempt loop, the throttle acquire sits _above_ the auth apply: + +``` +engine.ts:629 const { waited } = await acquireWithin(…) ← the queue wait +engine.ts:649 await cfg.auth.apply(req, …) ← signing +engine.ts:652 await cfg.hooks?.onRequest?.(…) +``` + +and each attempt gets fresh headers off the _unsigned_ base request. + +**Measured:** + +| | signature age on arrival | statuses | +| ------------------------------------------------------------- | -------------------------------------- | ------------------------ | +| 4 calls behind `rate: '1/2m'`, granted at 0/2/4/6 virtual min | **0, 0, 0, 0 ms** | `200, 200, 200, 200` | +| the same calls **pre-signed** (the botocore shape) | 0, 2, 4, **6 min** | `200, 200, 200, **403**` | +| held 6 min behind `concurrency: 1` | **0 ms** | `200` | +| 3 attempts 6 min apart, long backoff | **0, 0, 0 ms** — 3 distinct signatures | | +| a 10-minute `Retry-After` park | **0 ms** | `200` | + +**A StitchAPI throttle cannot expire a signature.** The circuit breaker doesn't either — it +fast-fails _before_ signing, so three blocked calls produced **zero** signings. + +## Clock drift is still yours — 26 lines + +Per-attempt signing re-mints the _same wrong time_: measured, 4 attempts, 4 identical +600,000 ms skews. The correction AWS SDKs implement — learn the offset from the server's `Date` +header, re-sign — is reachable through `AuthStrategy.shouldRefresh`/`refresh`: + +```ts +shouldRefresh: (res) => res.status === 403 && isSkew(res), +refresh: async () => { /* offset learned from the Date header, via a closure */ }, +``` + +**Measured:** the offset was learned (600,000 ms), the _same attempt_ re-signed, and the call +returned `200` — costing **no retry budget**, because a refresh re-runs the attempt rather than +consuming one. The offset persists, so the next call needed **one** request. + +Assembled: **4 of 4 calls succeeded through a 10-minute host drift _and_ a 6-minute rate-limited +queue**, worst signature age 0 ms, breaker never opened — 26 lines in two declarations, all of +it for the drift half. With no user code at all the same workload gave +`["403", "403", "503", "503"]`. + + + **`hooks.onRequest` runs *after* signing** (`:652` vs `:649`) — and it is + the only user-code seam that does. A hand-rolled pacing gate there + **re-creates botocore#149 inside a library that doesn't have it**: measured, + a 6-minute wait in `onRequest` aged the signature 6 minutes and got a `403`. + Pace with `throttle`, never with a sleep in a hook. + + +## What StitchAPI does not solve here + +1. **Clock drift itself.** The library re-signs faithfully with whatever clock you have. +2. **Skew correction has no config vocabulary** — it is `shouldRefresh`/`refresh` plus a closure, + because `refresh` cannot see the response that triggered it. The offset has to be smuggled out + of `shouldRefresh`. +3. **A skew 403 counts as a circuit failure.** A fault _inside your own process_ opens the + dependency's breaker, and the half-open probe then reports `RequestTimeTooSkewed` rather than + the fault that opened it. +4. **A breaker does not shed a burst already queued behind a throttle** — the circuit phase is + read before the throttle wait. Measured: 4 concurrent calls all reached the wire over 6 + minutes _after_ the breaker opened; the same 4 issued sequentially stopped after 2. +5. **SigV4 signs with `new Date()`, not the injected clock.** On real time the stamp is correct — + this is a testability defect, not a wire defect — but it means a skew test on a `manualClock` + is impossible: 600 virtual seconds moved the shipped stamp **0 seconds**, and under a default + `manualClock()` **0 of 3** calls were accepted, with ~20,670 days of apparent skew. + + + **`verdict: { accept: [403], flag: 'ok' }` swallows the skew error.** The recipe that works + for a 401 in [the multi-tenant scenario](/docs/scenarios/multi-tenant-blast-radius) does not + transfer: `verdict.flag` is three-state, an **absent** flag means "no signal", and AWS error + bodies carry no flag. Measured: `ok: true`, with `RequestTimeTooSkewed` handed to the caller + as data. Six lines of `Surface.interpret` fix both this and finding 3 above. + + Also: **`backoff.max` defaults to 10 s**, so `base: '6m'` silently waits 10 seconds. + Protective here — it cost this scenario's proofs a false negative. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/index.mdx b/apps/docs/content/docs/scenarios/index.mdx new file mode 100644 index 00000000..4406ea90 --- /dev/null +++ b/apps/docs/content/docs/scenarios/index.mdx @@ -0,0 +1,129 @@ +--- +title: 'Scenarios' +description: 'Real integration problems that have no one-line answer anywhere — what the usual fixes cost, what StitchAPI changes, and what it leaves to you.' +--- + +A scenario is a problem people actually hit against a real API, picked because the +honest answer is _not_ a one-liner in any library. Each page states the problem, +lays out the solutions the ecosystem has converged on and what each one costs, +shows the StitchAPI shape next to them — and then says plainly what StitchAPI +does **not** solve, so you can see the remaining work before you commit to it. + +Where a [recipe](/docs/recipes) shows a task with a known good answer, a scenario +shows a problem where the answer is a trade-off. + +Every claim on these pages is backed by a script that was run offline against the +real runtime, not by reading the source. + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/docs/content/docs/scenarios/intermittent-drift.mdx b/apps/docs/content/docs/scenarios/intermittent-drift.mdx new file mode 100644 index 00000000..4cf83fa5 --- /dev/null +++ b/apps/docs/content/docs/scenarios/intermittent-drift.mdx @@ -0,0 +1,135 @@ +--- +title: 'The vendor changed the shape for 5% of responses' +description: 'Leveled drift catches a canary rollout precisely and refuses to invent a value. What it cannot do is tell a harmless coercion from a destructive one.' +prerequisites: + ['/docs/guides/validation/validation', '/docs/concepts/event-stream'] +--- + +## The problem + +A vendor ships a response-shape change — but not all at once. A canary at 5% of traffic, then +25, then 50. Or not a rollout at all: the shape varies _by data_, like a geocoder that returns +`null` for `formatted_address` only on ambiguous queries. + +**Intermittent breakage is harder than total breakage.** A change that breaks every call is +found in minutes and rolled back. A change that breaks 5% looks like flakiness, sits in the +backlog for a week, and gets fixed once someone spots the pattern. + +Four change classes, and treating them alike is the mistake — addition is non-breaking by every +published policy, removal and type change are breaking, and a field becoming nullable is +warning-level and the most intermittent of all. + +The dangerous one is the type change, because a naive cast produces a **plausible** value: a +payment provider moves `transaction_id` from `12345` to `"12345"`, your code casts to int, gets +`0`, and processes a **$0 transaction**. + +## The common solutions + +| Approach | Where it breaks | +| ------------------------------- | --------------------------------------------------------------------------------- | +| Strict schema validation | Catches everything — including the added field that broke nothing. Alarm fatigue. | +| Parse loosely, cast defensively | Never alarms, and manufactures the `$0` transaction. | +| Contract tests in CI | Can't see a canary that started after your deploy. | +| Level the findings | The right model — needs a vocabulary most validators lack. | +| Log and aggregate | The only way to see 5% → 25%. Needs the finding to name the field. | +| Pin a vendor API version | The real fix where offered; useless against data-dependent nulls. | + +## What StitchAPI does + +**The default is safe, and that is the headline.** Against `z.number()`, a `transaction_id` that +arrives as `"12345"` produces `error | invalid | transaction_id | Expected number, received +string` and the **call fails** with `data: null`. Even `z.coerce.number()` on `"abc"` fails — +Zod rejects `NaN`. StitchAPI does not manufacture a $0 charge on its own. + +**The precision is excellent.** Over 100 calls where a geocoder returned `null` on 5 of them, +drift fired on **exactly** calls `[20, 40, 60, 80, 100]` — matching the vendor's own ledger, +zero false positives on the other 95, each finding naming `formatted_address` with +`null -> string`. + +**And aggregation is a real seam**, which is what makes an intermittent change actionable. A +`TraceSink` sees every event of every call, and `ctx.spanId` lets you hold per-call state +honestly: + +```ts +const charges = stitch({ + url: 'https://api.vendor.com/charges/{id}', + output: drift(StrictCharge, { severity: { undeclared: 'verbose' } }), + trace: new DriftRate({ window: '15m' }), +}); +``` + +**Measured:** `5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)`, +and on a rolling window the same field widening **5.0% → 25.0%** as the canary expanded. Across +six workloads at 100 calls each: silent on a 100% _addition_ rollout, one alert line per +breaking class at 5%, and **zero $0 charges**. The same six workloads against the soft schema +teams write for availability kept 100% of calls and produced **ten** $0 charges. + +## What StitchAPI does not solve here + +1. **A `coerced` finding cannot say whether the coercion was destructive.** `"12345" → 12345` + and `"abc" → 0` emit **byte-identical** findings — `warn | coerced | transaction_id | string +-> number` — because `detail` is `kindOf(old) -> kindOf(new)` and values never appear in a + finding. Only joining `drift` to `result` on `ctx.spanId` in a sink separates them. +2. **The nullable class is inexpressible.** "Nullable is a warning, value intact" — the fourth + industry class — has no spelling. `.nullable()` produces **nothing at all**, so the 5% + rollout is invisible; the strict schema turns it into a 5% error rate that also discards the + four fields that were fine. A hand-rolled classifier beats `DriftOptions` on exactly this row. +3. **`severity` is keyed by mechanism, not by change class.** It takes + `undeclared`/`coerced`/`defaulted` — of which only _addition_ maps 1:1 to an industry class. + Removal, type change and nullability each land on a kind decided by **your schema**, so their + loudness is a schema decision rather than a severity one. +4. **No per-path severity.** `ignore` is the only path-aware lever and it is on/off. "Coercion on + `transaction_id` pages, coercion on `description` doesn't" has no spelling. +5. **A soft finding cannot be promoted to fatal through the type.** `error` isn't in + `DriftSeverity` (a cast past it does work at runtime). +6. **Soft findings are invisible on the awaited path.** `await` and `.safe()` carry nothing — + `StitchError` has no `findings`, and a hard failure gives only a generic `contract violation +(drift)`. The trace sink for the _same_ run named the field and both types. `drift()` with + only `.safe()` does nothing for you. +7. **A schema strips what it doesn't declare.** The added field in the addition case is + `undefined` on `data`; reading it needs `.inspect().raw`, which is a fresh request. + + + **Two ordinary spellings manufacture the $0 charge.** + + `z.coerce.number()` maps **`null` → `0`** with no `.catch()` involved, because + `Number(null) === 0`. Measured: `null`, `""`, `" "`, `false` and `[]` all coerce to exactly + `0`; only `"abc"` rejects. And `.catch(0)` hands the caller `0` for *anything*. + + `.default('usd')` on a **removed** field is the same shape — it fabricates a value the vendor + never sent, at `verbose`. On money, prefer a failed call to a plausible number. + + + + + **A cache hit emits no drift, so caching divides your drift rate by the miss ratio.** + Measured: 5 calls against a vendor drifting on **100%** of responses reported **20%**. + + Two smaller counting traps: findings are not calls (2 findings on one response reads as 200% + unless you collapse on `ctx.spanId`), and `.report()`/`.inspect()` are **fresh probes** — + they cost a request, tick your denominator, and answer about a different response. `.report()` + called right after a drifting call reported **zero** findings. + + + +## StitchAPI vs the common solution + +**A wash on size — 93 lines against 92.** But the halves are not alike: detection is **9 +declarative lines against ~35**, while aggregation is ~84 lines of user code either way, because +the library counts nothing. + +What the hand-rolled 92 lack is the resilience stack, measured here as one `retry` line +absorbing 8 × `503` across the canary while the rate still counted **100 logical calls out of +108 wire requests**. + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/large-response-memory.mdx b/apps/docs/content/docs/scenarios/large-response-memory.mdx new file mode 100644 index 00000000..cb2a189c --- /dev/null +++ b/apps/docs/content/docs/scenarios/large-response-memory.mdx @@ -0,0 +1,136 @@ +--- +title: 'The export that eats the heap' +description: 'The NDJSON decoder is genuinely O(1) — and the engine retains every chunk one line later, so neither await nor .stream() is memory-bounded.' +prerequisites: ['/docs/reference/surfaces', '/docs/concepts/event-stream'] +--- + +## The problem + +You call an export endpoint. The vendor hands back one JSON array with tens of thousands of +rows. You `await` it, parse it, iterate. Then the catalog grows and the process dies. + +**Parsing costs several times the wire size**, because `JSON.parse` holds the whole string _and_ +builds the object tree. A documented sync of 22,000 products in an **84 MB response exhausted +~2.1 GB** on a 4 GB VPS; the streaming rewrite ran at **180 MB peak**. + +This failure mode is unlike every other scenario here. Nothing returns wrong data — the process +just dies, usually after a dataset crossed a threshold nobody was watching. + +## The common solutions + +| Approach | Where it breaks | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `await` the JSON | Dies above a size you can't predict, and a chunked response has no `Content-Length`. | +| Ask the vendor for NDJSON | The correct fix where offered. Most REST endpoints don't. | +| Structural streaming parser | The real answer for one giant array. A dependency, and fiddly to assemble. | +| Paginate instead of exporting | Bounded — and inherits [every pagination problem](/docs/scenarios/unstable-pagination), plus N× the requests. | +| Raise `--max-old-space-size` | Moves the cliff toward you as data grows. | +| Batch and release references | Necessary alongside streaming; useless if the parse already buffered. | + +## What StitchAPI does + +**It ships a genuinely O(1) NDJSON decoder.** Driven directly, `decode: 'ndjson'` held **0.8 MB +of retained heap for 1,000,000 rows and 214 MB of wire**, moving less than 15% across a 1000× +change in workload. That is the real thing. + +**And the engine retains every chunk one line later.** Through `runStreaming` the same decoder +is linear — 3.5 MB → 30.2 MB from 10k to 100k rows — because every delta is pushed onto an +accumulator so the terminal `result` can mirror the whole spine. + + + **`.stream()` is not a memory fix.** Measured: **30.2 MB iterating vs 33.5 + MB awaiting** — the same number twice. The accumulator is *inside* the + generator both accessors drain, and no option disables it. The engine's own + code comment recommends reading incrementally via `.stream()` for exactly + this reason; the measurement says that mitigation doesn't work. + + +So the seam that does work is `Surface.stream` — not to replace the decoding, which is already +O(1), but because the engine retains whatever that hook _yields_. Yield one small receipt per +batch instead of one row per row: + +```ts +const rows = stitch({ + url: 'https://api.vendor.com/export.ndjson', + kind: batchedExport({ size: 1_000, onBatch: writeToDb }), // yields a receipt, not rows + stream: 'ndjson', +}); +``` + +**Measured:** **1.3 MB retained for 100,000 rows against the buffered baseline's 53.8 MB — a 40× +cut — and flat**: 1.3 MB at 1,000 rows, 1.4 MB at 100,000. Under a 96 MB heap ceiling, 400,000 +rows that killed the buffered path outright processed completely at 1.3 MB. + +Fairness on the baseline: the buffered multiplier measured **2.5×**, and it is `JSON.parse`'s, +not the library's — StitchAPI's overhead over a bare `JSON.parse` of the same bytes was **0.2 +MB**. It adds no copy. It also removes none. + +## `decode: 'json'` on one array: right answer, wrong memory + +The structural decoder's **emission is correct**, and impressively so — one delta per element, +holding up under `,` `]` `}` inside string values, escaped quotes, embedded newlines, +pretty-printed multi-line records, deep nesting, and 1-character chunk boundaries. + +Its **memory is not bounded**: retained heap tracks the whole array text at **0.88× the wire**, +with 34× growth over a 100× workload, and the time is quadratic (28× for 10× the rows). + + + **So it trips its own default cap, and blames the vendor.** 37,000 rows decode; 38,000 fail + with *"a malformed or never-closing value was streamed"* — which the vendor did not do. On a + 60,000-row array the consumer receives **37,312 rows and then `error`/`done(ok: false)`**: a + silent truncation for any loop that only matches `delta`, at a threshold that moves with your + customer's data. + + The control settles where the defect is: the same 100,000 records as **concatenated top-level + values** run flat at **0.9 MB**. One branch, not a design limit. + + + +## What StitchAPI does not solve here + +1. **No memory-bounded path is reachable from config alone.** Every decoder × every accessor is + O(N) through the engine. +2. **`decode: 'json'` does not bound a single top-level array** — the one case NDJSON can't cover. +3. **No size guard, threshold, warning or event on the buffered path.** A 21 MB response emits + the same four events a 40-byte one does. When the wall arrives it is V8's: 400,000 rows under + a 96 MB ceiling gave `FATAL ERROR: … JavaScript heap out of memory`, **exit 134** — no + catchable error, no `error` event, no `finally`. +4. **`stream.buffer.chars` is a malformed-input guard, not a budget.** It bounds one line or one + un-closed value — 20,000 well-formed rows streamed cleanly through a **1,000-character** cap + while the engine accumulated all 20,000. On a buffered stitch it typechecks and is **inert**. +5. **`pick` and `transform` are silently inert on a stream** — `transform` called **zero** times + over 200 deltas. No `info`, no drift finding, no throw. A stitch that carries them and is + later switched to `kind: stream` keeps compiling and quietly stops reshaping. +6. **`output` on a stream validates without transforming.** The buffered path serves the + validated value; the streaming path keeps only the errors. The same coercing schema reshapes + your data on `await` and silently does not on `.stream()`. +7. **A failing row is a circuit breaker, not a filter** — `contract violation (drift)`, the + stream ends, and the rest is never read. +8. **No batching primitive.** There's no `paginate`-style chunking for streams; the 75 lines + above are yours. + + + **`stream({ kind: mySurface })` silently drops your surface** — `stream()` overwrites `kind` + after spreading your config. Measured: 1,000 raw rows, **zero** through the surface, no error. + Spell it `stitch({ kind })`. + + And a backlogged socket is **invisible to `heapUsed`** — a producer that ignores + `desiredSize` puts the body in the stream's internal queue, which is external memory. Watch + `arrayBuffers`. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/meta.json b/apps/docs/content/docs/scenarios/meta.json new file mode 100644 index 00000000..29500c90 --- /dev/null +++ b/apps/docs/content/docs/scenarios/meta.json @@ -0,0 +1,28 @@ +{ + "title": "Scenarios", + "icon": "Map", + "pages": [ + "oauth2-refresh-token-rotation", + "cost-based-rate-limits", + "batch-partial-failure", + "async-job-polling", + "mid-stream-failure", + "conditional-requests-304", + "multipart-upload", + "webhook-receipt", + "multi-tenant-blast-radius", + "provider-failover", + "unstable-pagination", + "intermittent-drift", + "large-response-memory", + "expiring-signatures", + "unconfirmed-write", + "n-plus-one-fanout", + "deprecation-headers", + "agent-holds-the-tool", + "stale-fixture", + "precision-loss", + "pii-in-the-logs", + "dual-run-migration" + ] +} diff --git a/apps/docs/content/docs/scenarios/mid-stream-failure.mdx b/apps/docs/content/docs/scenarios/mid-stream-failure.mdx new file mode 100644 index 00000000..80473b62 --- /dev/null +++ b/apps/docs/content/docs/scenarios/mid-stream-failure.mdx @@ -0,0 +1,148 @@ +--- +title: 'A stream that fails after 800 tokens' +description: 'The 200 was spent on the first token, so the failure arrives in-band or not at all. When a stream can be resumed this is one flag; when it cannot — every LLM API — that same flag replays the whole answer.' +prerequisites: ['/docs/reference/surfaces', '/docs/concepts/event-stream'] +--- + +## The problem + +You stream a completion to a user. Eight hundred tokens in, it stops. The request did not +fail — `200 OK` went out with the _first_ token, and the status line was spent before anything +went wrong. Every later failure has to arrive **in-band**, as an SSE frame, or as nothing at +all when the socket simply drops. + +Three consequences, all awkward: + +- **Retrying is not neutral.** Replaying re-runs the model — you pay for the first 800 tokens + _and_ their replacement — and it duplicates content the consumer has already accumulated. + The user watches the answer restart. +- **"Ended" and "ended early" are the same shape.** A complete OpenAI stream is terminated by + `[DONE]`. A truncated one just stops. Nothing else distinguishes them. +- **Resumption mostly isn't offered.** SSE's `Last-Event-ID` needs the server to put `id:` on + every frame. OpenAI-style completion chunks carry none, so the standard mechanism does not + apply to the most common streaming API there is. + +## The common solutions + +| Approach | Where it breaks | +| ----------------------------------- | ------------------------------------------------------------------------------------------ | +| Retry the whole request | Pays twice, re-runs the model, duplicates content in any accumulator. The answer restarts. | +| Never retry a stream | Safe and common. Turns every transient blip into a visible failure. | +| Resume via `Last-Event-ID` | The correct mechanism — and unavailable on LLM APIs, which emit no `id:`. | +| Continuation prompt | The pragmatic LLM answer. Costs another call, and the seam shows in the output. | +| Buffer everything, emit at `[DONE]` | Makes truncation detectable and retry safe — and discards the entire point of streaming. | +| Check the sentinel | Necessary in all of the above. Cheap, and routinely forgotten. | + +## What StitchAPI does + +**If your stream is resumable, this is one flag and it is correct.** A feed that emits `id:` +on every frame, dropped after 2 of 5 tokens, with `sse: { reconnect: true }`: + +```ts +const feed = sse({ url: 'https://api.example.com/events', reconnect: true }); +``` + +Measured: the consumer saw `ABCDE` — five deltas, **zero duplication** — and the reopened +request carried exactly the right header, `Last-Event-ID` moving `(none) → t2`, where `t2` was +the last id delivered before the drop. Server pacing is honoured too: a `retry: 9000` frame +produced 9000 ms gaps, overriding the authored `reconnect.delay`. + + + **Do not enable `reconnect` on an LLM stream.** With no `id:` to resume + from, every reopen is a request for the *whole completion*. Measured on a + stream that **completed cleanly**: 4 opens, 4 `[DONE]` sentinels, and the + text `ABCDEABCDEABCDEABCDE` delivered to the consumer as one uninterrupted + stream — ending `done(ok: true)`. `sse: true` is the same flag. See [what + does not work](#what-stitchapi-does-not-solve-here) below. + + +For the LLM case the answer is **two seams**: `Surface.execute` for connect-only retry, and +the surface's `stream` hook to require the sentinel and reject in-band error frames. + +```ts +// Retry the CONNECT phase only — the one replay that is unambiguously safe. +execute: async (req, ctx) => { + for (let attempt = 1; ; attempt++) { + try { + return await ctx.adapter(req); + } catch (err) { + if (attempt >= connectAttempts) throw err; + await ctx.sleep(backoff(attempt)); // no bytes have flowed yet + } + } +}, + +// Require the sentinel: truncation is the ABSENCE of a frame, so nothing +// per-frame can catch it. +stream: async function* (res, cfg) { + let sawDone = false; + for await (const frame of sseSurface.stream(res, cfg)) { + if (isErrorFrame(frame)) throw new Error(`provider error frame: ${messageOf(frame)}`); + if (isDone(frame)) { sawDone = true; continue; } + yield frame; + } + if (!sawDone) throw new Error('stream truncated: no [DONE] sentinel'); +}, +``` + +**Measured across six shapes:** a complete answer delivered once; a healed connect retried at +the connect phase only (3 opens, `ABCDE` once); a mid-body drop keeping `ABC`; a truncation +caught as `stream truncated: no [DONE] sentinel` with the partial kept; an in-band error frame +surfaced with the bad frame **withheld** from the consumer; and a dead server failing after 4 +connect attempts rather than resolving empty. + +## StitchAPI vs the common solution + +Unusually for this section, **the StitchAPI version is smaller** — 62 lines against 83 for the +hand-rolled twin, which has to bring its own SSE parser. Both produce byte-identical results +and identical open counts on all six shapes, so the machinery isn't buying behaviour; it's +buying the spine: one `start` / `delta`×N / `error` / `done` trace under one traceId, and +`auth`, `headers`, `throttle` and `timeout` staying configuration instead of growing the +hand-rolled file. + +## What StitchAPI does not solve here + +1. **`retry` does not run on a streaming stitch at all.** Measured: `retry: { attempts: 4 }` + against an always-503 server made **4 requests on a buffered stitch and 1 on an `sse` one**, + with `error.attempts: 1`. The safe replay — the connect phase, before any byte — is the one + case `retry` cannot cover. +2. **`retry.attempts` is inert but `retry.backoff` is live**, as the _reconnect_ curve. One + name, two fates: `retry: { attempts: 1 }` still produced 4 opens. +3. **Connect-phase and body-phase policy are not separately addressable.** One flag governs + both. `reconnect.onlyOnDrop`, `reconnect.requireToken` and `retry.phase` are all absent + (machine-checked). +4. **`interpret` and `verdict.flag` are dead code on streaming surfaces.** A custom surface's + `interpret` ran **zero** times — `runStreaming` calls only `classifyStatus`. They typecheck + and do nothing. +5. **`hooks.onError` never fires for a post-200 stream failure.** Measured hook sequence on a + mid-body drop: `[onRequest, onResponse]`, while the run failed. Any hook-based error + pipeline is blind to this entire scenario. `onResponse` also fires _before_ a single frame + is parsed — `ctx.res.body` is a live `ReadableStream`. +6. **Truncation is undetectable by any built-in.** A truncated stream and a complete one + produce the same terminal spine — `result, done(ok: true)`. `output` can't express it + either: a schema demanding the sentinel rejects delta 1 instead. +7. **The partial survives only on `.stream()`.** `.safe().data` is `null`; the thrown + `StitchError` has `body`, `data`, `partial` and `chunks` all `undefined`; and `.inspect()` — + whose whole job is "what did the server actually send?" — returns `data: null`, `raw: null`, + `status: 0`. The engine is holding the answer one line above the `return` that discards it. +8. **The reconnect loop sits above every surface hook**, so no surface can defend against the + replay in the callout above — that decision is made in `runStreaming`, before any hook runs. + + + **Default reconnect backoff is roughly 50 ms** (`expo-jitter` off base 100), so a dropped + stream replays within a tenth of a second unless you author a `retry.backoff`. And + `verdict: { accept: [503] }` combined with `reconnect` resolves a permanently-failing + server **successfully** with `data: []`. + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/multi-tenant-blast-radius.mdx b/apps/docs/content/docs/scenarios/multi-tenant-blast-radius.mdx new file mode 100644 index 00000000..47d7c67d --- /dev/null +++ b/apps/docs/content/docs/scenarios/multi-tenant-blast-radius.mdx @@ -0,0 +1,149 @@ +--- +title: "One customer's revoked token, everyone's outage" +description: 'Tokens and caches isolate per tenant automatically. Rate budgets and circuit breakers do not — they isolate only by a string you have to remember to write.' +prerequisites: + ['/docs/concepts/the-seam', '/docs/guides/resilience/circuit-breaker'] +--- + +## The problem + +You integrate a vendor API on behalf of each of your customers. Every call carries that +customer's credential. At 500 customers with 8 connections each, that's +[4,000 token lifecycles](https://truto.one/blog/how-to-architect-a-scalable-oauth-token-management-system-for-saas-integrations/). + +**The unit of failure is the tenant. The unit of protection usually isn't.** Rate budgets and +circuit breakers are scoped to a _dependency_, but what goes wrong is scoped to a _customer_ — +their token was revoked, their admin changed a permission, their batch job went rogue. When the +protection is broader than the failure, one customer's problem becomes everyone's. + + + **Measured, on a shared seam with `circuit: { failures: 3, cooldown: '30s' }`:** one customer + with a revoked token failed **9 of 9 healthy customers**, and **zero** of their requests ever + reached the vendor — they fast-failed in-process with `503 circuit open`. + + **And the outage does not end on its own.** Half-open admits exactly one trial call, and the + broken tenant is the one retrying hardest — so across four full cooldown windows the healthy + tenant measured `503, 503, 503, 503`. Recovery happens when a healthy tenant *happens* to win + the probe. That's a race, not a policy. + + + +## The common solutions + +| Approach | Where it breaks | +| -------------------------------------- | ----------------------------------------------------------------------------------------- | +| One client instance per tenant | Correct by construction — and usually assumed not to scale. | +| Partitioned limiter | The right shape, if your client offers one. Most don't. | +| Per-tenant circuit breaker | The named mitigation. Needs the breaker to accept a per-tenant key. | +| Global breaker, tuned high | Trades one failure mode for another — a real outage now takes far longer to trip. | +| Exclude auth failures from the breaker | Genuinely correct and usually forgotten. A `401` says the credential is bad, not the API. | +| Sharded workers by tenant | Real isolation, at the cost of a routing tier. | + +## What StitchAPI does + +The library already carries a tenancy axis — and it reaches exactly half of what you need. +`AuthContext.principal` is visible to the auth strategies and the cache-key builder, **and +nothing else**. So the split falls on the auth/resilience line: + +| resource | isolated by | fails | +| ------------------- | -------------------------------------------------- | -------------------------------------------- | +| **Token** | `oauth2({ tenancy: 'principal' })` + `seam.as(id)` | **closed** — errors if no principal is bound | +| **Cache** | `tenancy`, defaults to `'principal'` | **closed** | +| **Rate budget** | a per-tenant limiter **key** you write | **open, silently** | +| **Circuit breaker** | a per-tenant `circuit.key` you write | **open, silently** | + +The two whose isolation is a _security_ property fail closed. The two whose isolation is an +_availability_ property fail open. That is the whole scenario in one table. + +The working construction is one shared seam plus a per-tenant member: + +```ts +const vendor = seam({ + baseUrl: 'https://api.vendor.com', + auth: oauth2({ tenancy: 'principal' /* … */ }), + store: redisStore(/* … */), +}); + +const itemsFor = (tenant: string) => + vendor.as(tenant).stitch({ + path: '/v1/items', + name: `items:${tenant}`, // partitions the rate budget + circuit: { failures: 3, cooldown: '30s', key: `items:${tenant}` }, // and the breaker + // A 401 means the CREDENTIAL is bad, not the vendor — don't let it trip the breaker, + // but don't swallow it either. + verdict: { accept: [401], flag: 'ok' }, + }); +``` + +**Measured:** the same revoked credential that took down 9 of 9 in the callout above took down +**0 of 9** here. The broken tenant still received a real `401`. A genuine `500` still opened +**that tenant's own** breaker (`500, 500, 500, 503`) with zero effect on the others. And the +20-call burst that pushed a quiet customer from t=0 to **t=2000 ms** arrived at **t=0**. + +## Isolation is a property of the key string, never of the object graph + +This is the part worth internalising, because the intuitive constructions don't work: + +- **10 separate `.as()`-bound stitch objects** resolving to the same `path` → **1** breaker key, + 9 of 9 healthy tenants down. +- **10 separate seams** sharing one store → same result. +- A **`url`-only stitch** keys its breaker on the literal string `'stitch'` — so every such + stitch sharing a store shares one process-wide breaker, across tenants _and_ endpoints. + +Conversely the correct partition is cheap: 100 keyed stitches built in under 100 ms with zero +timers armed, and 100 per-tenant seams cost well under 40 kb each with **zero** connection pools +— a seam owns no transport, so the "4,000 pools" the literature warns about isn't a cost this +construction has. + + + **A per-tenant seam isolates the rate budget and *not* the breaker.** Measured in one run: + the quiet tenant left at t=0 (rate isolated) while 3 of 3 healthy tenants got `503` (breaker + shared). The most isolated-looking construction is half a fix, and the missing half is the + one that causes outages. + + **And `throttle: { pool: 'host' }` silently re-keys the *circuit* onto the host.** A + per-tenant `name` partition evaporates, and an unrelated endpoint for an unrelated tenant + measured `503`. + + + +## What StitchAPI does not solve here + +1. **No per-tenant declaration exists for either resource.** `ThrottleOptions.pool` is + `'stitch' | 'host'`; `CircuitOptions` is `{ failures, cooldown, key }`. `pool: 'principal'`, + `throttle.key`, `throttle.tenancy` and `circuit.tenancy` are all compile errors. You smuggle + tenancy through key strings. +2. **`oauth2` defaults to `tenancy: 'app'`** — three different customers measured **one** token + fetch and one shared `Authorization` header. Nothing at the call site hints at it. +3. **`tenancy` partitions the token _cache_, not the _credential_.** All tenants' tokens are + minted from one `client_id`, because `Secret` is a niladic thunk with no context. Per-customer + credentials need a custom `AuthStrategy.apply(req, ctx)` reading `ctx.principal` — the only + user-reachable hook that sees the bound principal at call time. +4. **Global quota _and_ per-tenant fairness is not expressible in one construction.** A member + throttle stacks tighten-only on the seam bucket, so declaring "1000/m to the vendor" puts the + noisy neighbour straight back (quiet tenant returns to t=2000). +5. **Breaker records have no TTL.** A churned tenant's key was still resident after a virtual + _year_, and nothing sweeps them — while the rate counter beside it does expire. +6. **`seam.stitch()` pins every stitch it creates.** Measured with `WeakRef` after a forced GC: + 200/200 root-created still reachable, versus **0/200** created through `seam.as(p).stitch()`. + The per-request shape is the one that doesn't leak; the only release for the other is + `seam.close()`, which also closes the store. +7. **Seam ids are a creation-order counter**, so per-tenant seams over a shared durable store + collide across worker processes non-deterministically. +8. **`CircuitOpenError` names nothing about who tripped it.** When the breaker is shared, you + cannot tell from the error which tenant caused the outage. + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/multipart-upload.mdx b/apps/docs/content/docs/scenarios/multipart-upload.mdx new file mode 100644 index 00000000..b0c24c7d --- /dev/null +++ b/apps/docs/content/docs/scenarios/multipart-upload.mdx @@ -0,0 +1,146 @@ +--- +title: 'The upload you must clean up after' +description: 'Multipart upload is four steps, and the fourth — abort on failure — is the one no HTTP client models. Skip it and the parts bill forever, invisibly.' +prerequisites: ['/docs/reference/surfaces', '/docs/reference/helpers'] +--- + +## The problem + +A 5 GB upload can't go in one request, so you use S3-style multipart: + +1. `POST ?uploads` → an `UploadId` +2. `PUT ?partNumber=N&uploadId=…` × N → each returns an **`ETag` response header** +3. `POST ?uploadId=…` with the **ordered** `{ PartNumber, ETag }` list → the object +4. …and on **any** failure, `DELETE ?uploadId=…` + +**Step 4 is the one nobody models.** Abandon an upload and every part already sent stays in +the bucket and bills as storage indefinitely — while being invisible to `aws s3 ls` and to the +console's objects tab. AWS's own FinOps guidance puts incomplete multipart uploads at [up to +20% of an S3 bill](https://aws.amazon.com/blogs/aws-cloud-financial-management/discovering-and-deleting-incomplete-multipart-uploads-to-lower-amazon-s3-costs/), +and there is a [lifecycle rule](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html) +that exists purely to clean up after clients that didn't. + +That's a **compensating action**: a failure in step 2 or 3 obliges you to make a _different_ +API call. No HTTP client models it. The rest is ordinary orchestration made fiddly — the part +result is a header, the list must be in part order not completion order, concurrency needs a +bound, and progress needs XHR because `fetch` cannot report bytes sent. + +## The common solutions + +| Approach | Where it breaks | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| Single `PUT` of the whole file | One blink and 5 GB is gone; above 5 GB S3 refuses outright. | +| Vendor SDK (`@aws-sdk/lib-storage`) | Correct, and right if you're on AWS. Large dependency, and the shape recurs on every non-AWS API. | +| Hand-rolled loop + `try/finally` | What most teams write. The abort is one early `return` from being skipped, and nothing tells you when it was. | +| tus / resumable protocol | Better where you control the server. Not an option against S3's own API. | +| Lifecycle rule as the safety net | Necessary, not a fix — you still pay for N days of orphans on every failed upload. | + +## What StitchAPI does + +Three things become configuration, and they're worth having: + +```ts +const putPart = stitch({ + method: 'PUT', + url: 'https://s3.example.com/{key}?partNumber={part}&uploadId={uploadId}', + // S3's own transient error is 500 InternalError — NOT in the default set. + retry: { attempts: 3, on: [429, 500, 502, 503, 504] }, + // One stitch, called N times. `pool: 'host'` because the default is per-stitch. + throttle: { concurrency: 4, pool: 'host' }, + adapter: xhrAdapter(), // fetch cannot report bytes sent + kind: { + id: 'http', + // The part's result is a HEADER. Without a surface it is unrecoverable — + // `.inspect()` carries no headers at all. + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, + }, +}); +``` + +**Measured:** per-part retry re-sent only the failing part (arrival order `[1,2,3,4,3]`, one +initiate, zero orphans). `throttle: { concurrency: 3, pool: 'host' }` held peak in-flight at +**3** across 8 parts. `xhrAdapter` reported 4 upload ticks per part _before the response +existed_; the same call through `fetchAdapter` reported **zero**. + +## What it does not do — and this is the point of the page + +**There is no compensation seam.** `Hooks` is exactly `{ onRequest, onResponse, onError, +onRetry }`, and `onError` is not a failure hook — it's the `catch` around the transport. On an +HTTP 500 the measured hook sequence was `[onRequest, onResponse]` with **zero** `onError` +calls. `HookContext` has no run-scoped slot to hold an `UploadId`, and `linked()` has no +`finally`. + +So the cleanup is a plain `try/finally` in your own orchestration function — and it has to be, +because the `UploadId` only exists there. Measured, with a part failing and no user cleanup: + +| | orphaned parts | DELETEs | +| --------------------------- | ----------------------------------- | ------- | +| no cleanup | **3 (15 MiB), 1 dangling UploadId** | 0 | +| cancelled via `AbortSignal` | **2** | 0 | +| `timeout.total` expiry | **3** | 0 | +| user-written `try/finally` | 0 | 1 | + +Cancellation is not cleanup: the engine cancels in-flight work and forgets the work that +landed. + +## StitchAPI vs the common solution + +**141 lines against 163** hand-rolled — 22 shorter, and the difference attributes exactly to +the retry loop with backoff, the concurrency pool, the retryable-status set and URL assembly, +all of which became config. + +What did _not_ shrink is the half the scenario exists for: the `try/finally`, the +loud-cleanup rule, the per-part high-water progress map and the input-order assembly are +byte-for-byte identical on both sides. The library is a bystander for the compensation. + +## What StitchAPI does not solve here + +1. **The compensating call.** No hook, no config key, no surface position runs on failure. +2. **`hooks.onError` is not a failure hook** — zero calls on an HTTP 500. +3. **Cancellation ≠ cleanup.** `AbortSignal` and `timeout` both leave orphans. +4. **`all()` bounds nothing** — measured peak **8** over 8 members — and it hands every member + the _same_ input, so one stitch × 8 members produced 8 PUTs all carrying `partNumber=1`. +5. **`throttle.concurrency` defaults to `pool: 'stitch'`.** Eight stitches at `concurrency: 3` + each measured a peak of **8**. Use `pool: 'host'` or a seam bucket. +6. **`all()` discards partial results on fail-fast** — 2 parts stored, **0 nameable** for the + abort. There is no `allSettled`; only an `onResponse` side channel recovers them. +7. **The default `retry.on` excludes 500**, which is S3's own transient error. Measured: 4 PUTs, + 1 failed part, 3 orphans with the default set. +8. **Whole-upload retry is neither flagged nor prevented** — `retry` on the outer orchestration + measured **3 initiates, 9 orphaned parts, 45 MiB**. +9. **A progress tick has no identity** — `{ direction, loaded, total }` only, so a shared + `onProgress` across a fan is unattributable. And ticks are cumulative _within_ a part, so + summing them overshoots: naive `Σ loaded` measured **400** against a real 160. +10. **Response headers are absent from `.inspect()`**, so without a surface the `ETag` is + unrecoverable on the awaited path. + + + **Two ways cleanup lies to you, both measured.** + + **`.safe()` on the abort cannot throw.** Inside a correct-looking `try/finally`, pointed at + a wrong `UploadId`: **0 accepted DELETEs, 3 orphaned parts, and nothing thrown anywhere**. + The `finally` ran. That is a permanent invisible bill with a clean code review — make the + cleanup failure loud. + + **Cleanup inside `Surface.execute` runs after the caller returns.** Measured at the instant + the caller's promise settled: **3 orphans, 0 DELETEs**; the DELETE landed several turns + later. In a lambda, or any process that exits on the error, the later half never happens — + and the code reads as though the engine owns the lifecycle. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/n-plus-one-fanout.mdx b/apps/docs/content/docs/scenarios/n-plus-one-fanout.mdx new file mode 100644 index 00000000..082eaa9f --- /dev/null +++ b/apps/docs/content/docs/scenarios/n-plus-one-fanout.mdx @@ -0,0 +1,135 @@ +--- +title: 'One list, a hundred follow-up calls' +description: 'cache.coalesce collapses in-flight duplicates — 100 concurrent calls over 30 ids made 30 requests. A coalesced failure is not shared, and that is where it loses.' +prerequisites: + ['/docs/guides/resilience/throttle', '/docs/guides/data/pagination'] +--- + +## The problem + +`GET /orders` returns 100. Each carries a `customerId`, so you make 100 more calls. It is the +most common composition shape in API integration, and every part of it is a decision. + +**N is unknown until runtime**, and every call needs a _different_ input. Then: + +- **Concurrency** — all 100 at once trips a rate limit; one at a time wastes the afternoon. And + vendor governance is often concurrency-based, so a requests-per-second cap doesn't protect you. +- **Partial failure** — `Promise.all` rejects on the first error **and discards the successes**. +- **The thundering herd** — 100 calls that 429 together and back off by the same amount retry + together. Deterministic backoff _re-clusters_ the burst. +- **Duplicates** — 100 orders commonly reference far fewer customers. + +## The common solutions + +| Approach | Where it breaks | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `Promise.all(ids.map(fetch))` | Unbounded concurrency, and one failure discards every success. | +| `Promise.allSettled` + `p-limit` | Correct — two dependencies and a hand-rolled join. | +| Sequential loop | Safe, N× the latency. | +| A batch endpoint | Best where offered, and then you inherit [partial-failure semantics](/docs/scenarios/batch-partial-failure). | +| Cache / dedupe by id | Free quota — only if **in-flight** duplicates collapse too. | +| `?expand=` on the list | The real fix. Rarely offered. | + +## What StitchAPI does + +**`cache.coalesce` collapses in-flight duplicates, and that is the headline.** 100 concurrent +calls over 30 distinct ids made **30 requests** — one per id, exactly the floor — with every +call in flight and **not one response landed**. Seventy callers were served without a request of +their own, from a single `cache: { ttl }` block and no user code. The same cache with +`coalesce: false` made **100**. + +Three more things are configuration: + +```ts +const customer = stitch({ + url: 'https://api.vendor.com/customers/{id}', + throttle: { concurrency: 8 }, // measured: peak 8 exactly, against 100 unbounded + cache: { ttl: '60s' }, // the dedupe + retry: { attempts: 3 }, // jittered by default +}); +``` + +**Measured:** bounded concurrency held **peak 8** exactly. And the retry default genuinely +de-clusters a herd — 100 calls 429'd in the same instant retried across **~98 distinct +milliseconds** under `expo-jitter`, where `'fixed'` **and** `'expo'` both put all 100 into **one +millisecond** (doubling a constant is still a constant on attempt 2). + +`linked` gives the fan-out **one trace** — 1 traceId, 1 root, 101 spans covering the list and +every lookup, with per-call inputs. Assembled: **45 executable lines against 87** hand-rolled, +the difference being the FIFO pool, the retry loop, the retryable-status set and URL assembly, +all of which became config. + +## Where it loses: a coalesced failure is not shared + +The leader's _failure_ releases its joiners to re-run. Measured: 100 concurrent calls for one +id that 404s made **100 requests in two waves** — 1 leader, then 99 followers each re-running +the whole chain. + +Every joiner gets its own honest `HTTP 404`, never a leader artefact — the right _error_ at the +wrong _price_. End to end this is the one place the hand-rolled version wins: **44 customer +requests against 32**, and a deleted customer cost **4 requests against 1**. A `Map` +shares the rejection; the library spends a wasted request per duplicate reference to a broken +id — which is exactly the shape a dead foreign key takes. + +## What StitchAPI does not solve here + +1. **The combinators cannot express this** — and not for the reason you'd guess. The runtime + length is fine (`all(ids.map(…))` compiles). **The input is the wall**: `all()` spreads one + `StitchInput` into every member, so 100 members made **100 requests for one distinct id**. + It also bounded nothing (peak 100) and fail-fast **discarded 99 successful fetches** while + still paying for them — the auto-cancel prevented **zero** requests. +2. **`allSettled` semantics are absent**, and the documented workaround — compose `.safe()` + members by hand — does not typecheck, because `Member` is brand-gated. +3. **In-flight dedupe of failures**, above. +4. **A fan-shaped trace over per-call inputs.** You get a fan (`all()`, wrong inputs) or per-call + inputs (`linked`, which chains each call under the previous — measured **depth 101, fan-out + 1** for calls that ran at peak 100 concurrently). Not both. +5. **Deep-copying joined results** — see the aliasing callout. +6. **A batch endpoint.** Nothing can invent one. + + + **A declared concurrency budget silently multiplies across stitch objects.** One stitch called + 100 times at `concurrency: 8` → peak **8**. One hundred *separate* stitches each declaring 8 → + peak **100**, because the limiter is per stitch. `pool: 'host'` repairs it — **and adding a + `store` silently breaks the repair again** (peak 100), because the store-backed throttle never + reads `pool`. The store is exactly what you add to make the *rate* budget cross-process, and it + un-pools the *concurrency* on the way past with the config unchanged. A seam-level + `concurrency` survives both. + + Related: **a backing-off call holds its slot.** With a bound of 4 and a 1 s backoff, ~95% of + the budget sat idle on sleeping calls, and the retry re-queued at the **back** of the FIFO. + + + + + **`cache: { ttl: 0 }` caches forever.** It is the obvious spelling for "dedupe but don't + cache", and `expires === 0` reads as live — a later fan-out added **0 requests**. There is no + coalesce-only spelling. + + **`Retry-After` defeats the jitter by default.** `retry.respect` is on, so a 429 carrying + `Retry-After: 2` put all 100 retries back into **one millisecond** with `expo-jitter` still + declared. `respect: false` restores the spread and is all-or-nothing. + + **Coalesced and cached callers share one object by reference.** 20 rows over 5 customers gave + **5 distinct objects** — mutating row 0 changed row 5. The aliasing arrives with the + optimisation. + + Two smaller ones: **`sensitive: true` silently disables coalescing**, and coalescing is + GET/HEAD only until you name `methods`. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/oauth2-refresh-token-rotation.mdx b/apps/docs/content/docs/scenarios/oauth2-refresh-token-rotation.mdx new file mode 100644 index 00000000..573a3766 --- /dev/null +++ b/apps/docs/content/docs/scenarios/oauth2-refresh-token-rotation.mdx @@ -0,0 +1,196 @@ +--- +title: 'OAuth2 refresh tokens that rotate' +description: 'A single-use refresh token plus two concurrent workers revokes the whole account. What the usual fixes cost, and what a custom auth strategy buys you.' +prerequisites: + ['/docs/guides/auth/oauth2', '/docs/guides/state/pluggable-store'] +--- + +## The problem + +Your backend holds a per-account **refresh token** for a third-party API — +Atlassian, Asana, Xero, Slack — and exchanges it for short-lived access tokens. +The provider implements +[RFC 6819 §5.2.2.3](https://datatracker.ietf.org/doc/html/rfc6819#section-5.2.2.3) +replay detection: redeeming a refresh token returns **a new one** and invalidates +the old, and presenting an already-redeemed token is read as evidence of theft. + +So the provider does not reject one call — it **revokes the entire token family**. +The user is silently disconnected and has to re-authorize in a browser. + +Now add ordinary concurrency. A sync job, a webhook handler, and a user request +all hit `401` at the same instant. Each independently decides to refresh. The +first redemption rotates the token; every other one presents a consumed token and +trips replay detection. + +The failure mode is not "a request failed." It is "the integration lost the +account," and it only appears under load. + + + This is not a beginner's mistake. It has been filed against [OpenAI + Codex](https://github.com/openai/codex/issues/10332), the [MCP TypeScript + SDK](https://github.com/modelcontextprotocol/typescript-sdk/issues/1760), + and [oauth2-proxy](https://github.com/oauth2-proxy/oauth2-proxy/issues/1992) + — and it was a CVE in an auth library whose whole job is this + ([GHSA-392p-2q2v-4372](https://github.com/better-auth/better-auth/security/advisories/GHSA-392p-2q2v-4372)). + + +## The common solutions + +| Approach | Where it breaks | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Refresh-on-401 interceptor | The baseline bug. N concurrent 401s ⇒ N redemptions ⇒ family revoked. | +| In-process promise memo | Correct for **one** process. Silently insufficient at two workers — and it looks fixed in dev, where there is one. | +| Distributed lock (Redis `SETNX`) | Works, and makes refresh a distributed-systems problem: lock TTL vs latency, crash-while-holding, fencing. | +| Proactive refresh before expiry | Shrinks the window; does not close it. The skew boundary is a moment every worker crosses together. | +| Provider grace period | Not yours to choose. Auth0 and Okta offer one; Atlassian and Asana do not. | +| Dedicated refresh worker | Genuinely correct. Costs a deployable, and a cold access token now waits on a queue round-trip. | + +There is no one-liner. The honest minimum is **coordination scoped to the account +and spanning processes**, plus **durable persistence of the rotated token before +the old one is treated as spent**. + +## What StitchAPI does + +Not `oauth2()`. That strategy implements the `client_credentials` grant, which has +no refresh token by design — it reads only `access_token` and `expires_in` from +the token response, so a rotated `refresh_token` is discarded. + + + **`params` will let you look like you configured rotation.** It merges into the + token-request body and can override `grant_type`, so + `params: { grant_type: 'refresh_token', refresh_token: rt }` compiles, and + the **first redemption returns 200**. The second sends the same consumed token + and the provider revokes the family. There is no error and no type friction at + the moment you write it. Don't. + + +What StitchAPI gives you is the **seam**: `AuthStrategy` is a public exported type, +and a strategy receives an `AuthContext` with an async `vault` it owns, plus a +`shouldRefresh`/`refresh` pair wired into the call. You write the rotation policy; +the runtime owns where it runs. + +```ts +import type { AuthContext, AuthStrategy } from 'stitchapi'; + +export function rotatingRefresh(opts: RotatingRefreshOptions): AuthStrategy { + const accessKey = `rr:${opts.key}:access`; + const refreshKey = `rr:${opts.key}:refresh`; + let inFlight: Promise | undefined; + + const redeem = async (ctx: AuthContext): Promise => { + const stored = (await ctx.vault.get(refreshKey)) as string | undefined; + const res = await opts.adapter({ + url: opts.tokenUrl, + method: 'POST', + body: { + grant_type: 'refresh_token', + refresh_token: stored ?? opts.seedRefreshToken, + client_id: opts.clientId, + client_secret: opts.clientSecret, + }, + bodyType: 'form', + }); + const body = (res.body ?? {}) as TokenResponse; + if (res.status >= 400 || !body.access_token) + throw new Error(`refresh_token grant failed: HTTP ${res.status}`); + + // Persist the ROTATED token first — the old one is spent server-side the + // moment the provider answered, so this write must land before use. + if (body.refresh_token) + await ctx.vault.set(refreshKey, body.refresh_token); + const ttl = body.expires_in ? body.expires_in * 1000 : undefined; + await ctx.vault.set(accessKey, { token: body.access_token }, ttl); + return body.access_token; + }; + + // Coalesce concurrent redemptions; clear on settle so a failure never sticks. + const redeemOnce = (ctx: AuthContext): Promise => + (inFlight ??= redeem(ctx).finally(() => { + inFlight = undefined; + })); + + return { + name: 'rotatingRefresh', + async apply(req, ctx) { + const cached = await ctx.vault.get(accessKey); + req.headers['authorization'] = + `Bearer ${cached?.token ?? (await redeemOnce(ctx))}`; + }, + shouldRefresh: (res) => res.status === 401, + async refresh(ctx) { + await redeemOnce(ctx); + }, + }; +} +``` + +Attach it like any other strategy, to any stitch: + +```ts +const issues = stitch({ + baseUrl: 'https://api.atlassian.com', + path: '/ex/jira/:cloudId/rest/api/3/search', + auth: rotatingRefresh({ key: `jira:${accountId}` /* … */ }), + store: redisStore, +}); +``` + +**Measured:** 20 concurrent cold callers produce **1** redemption and **0** +replays. Cross-worker exclusion — built on `vault.increment(key, ttl) === 1` to +acquire and `vault.set(key, undefined)` to release — collapses **3 workers × 10 +callers to 1** redemption, and costs **42 more lines**. + +## StitchAPI vs the common solution + +The line count is not the win. A correct axios interceptor with a Redis lock is +about as long. Three things change: + +- **The policy is one object, not a call-site convention.** Every stitch that + names this strategy gets it — the CLI, an HTTP route, and an agent tool + included. Nothing can call the API and miss the interceptor. +- **The token never reaches the caller.** It lives in the vault and the outgoing + header. A caller, an agent included, receives data — never the credential. See + [Capability, not credential](/docs/concepts/capability-not-credential). +- **The store is already there.** `vault` is scoped, async, and shared by `key` + + `store`, so "where does the rotated token live" is answered before you start. + +## What StitchAPI does not solve here + +Blunt, because getting this wrong costs an account: + +1. **`oauth2()` does not implement the refresh-token grant.** Rotation is entirely + yours to write. +2. **`params` is a footgun for this.** It works once, then revokes the family. +3. **A shared `store` is a cache, not a lock.** Two cold workers fire one token + request _each_ — measured: 2 workers ⇒ 2 requests, 2 workers × 10 callers ⇒ + still 2. The store only helps once a write has landed. +4. **The built-in single-flight is per strategy instance, in-process, and + time-windowed.** 20 _simultaneous_ 401s coalesce to 1 refresh; 20 staggered + 8 ms apart produced **10**. Coalescing is not identity-scoped. +5. **There is no distributed-lock primitive.** `StitchStore` is + `get`/`set`/`increment`/`close` — no compare-and-set, no fencing token, no + blocking wait. A lock built on `increment` leans on cross-process atomicity + that the store contract only guarantees _within_ a process; confirm your + backend before relying on it. +6. **Nothing enforces write-before-use ordering.** The vault is a plain KV. The + "persist before the old token is spent" discipline is yours. +7. **`cookieSession` is not an alternative path.** Its refresh hook receives + `{ ok, status }` — the login response body never reaches you, so it cannot + carry a rotated token. +8. **One refresh per call.** If the retry after a refresh also 401s, the call + fails. A lock timeout surfaces to the caller as a 401. + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/pii-in-the-logs.mdx b/apps/docs/content/docs/scenarios/pii-in-the-logs.mdx new file mode 100644 index 00000000..4fad2ce4 --- /dev/null +++ b/apps/docs/content/docs/scenarios/pii-in-the-logs.mdx @@ -0,0 +1,158 @@ +--- +title: "The customer data you didn't mean to log" +description: 'Response bodies reach 13 destinations and metadata reaches 11 — with nothing in between. An output allowlist takes it to zero; sensitive: true does not, and only gates the cache.' +prerequisites: + ['/docs/guides/observability/trace-sinks', '/docs/guides/validation/drift'] +--- + +## The problem + +You integrate a vendor API that returns customer records — names, emails, addresses, health or +legal detail. You add tracing, because you are responsible about observability. Six months later +someone greps the log aggregator and finds all of it in plain text. + +Nobody logs PII on purpose. **Middleware does** — the two most-cited causes in the field are a +debug endpoint returning full user objects and +[a logging middleware that captures raw request bodies](https://hoop.dev/blog/why-pii-leakage-happens-in-apis/), +and the second is exactly what a well-instrumented client library is. + +Three things make it structural rather than careless. **You cannot enumerate what is sensitive in +advance** — a denylist of field names misses `primaryContactMail`, `profile.contact.mail`, and +an address inside a free-text note. **The vendor adds a field and it starts flowing**, which makes +this a drift problem, not a configuration one. And **the regulation is about defaults**: GDPR +Article 25 mandates protection by design _and by default_, so "we can turn redaction on" is not +the same thing. + +## The common solutions + +| Approach | What it is | Where it breaks | +| ----------------------------- | ----------------------------- | --------------------------------------------------------------------------------------------- | +| **Denylist of field names** | Scrub `email`, `ssn`, `card`. | Misses renamed, nested and free-text fields. Silently incomplete the day the vendor adds one. | +| **Allowlist of safe fields** | Log only what you named. | Correct by construction; needs the whole response shape — the thing that drifts. | +| **Don't log bodies at all** | Metadata only. | Safe and often unusable: the body is what you need when debugging an integration. | +| **Scrub at the aggregator** | Filter on ingest. | The data already left your process and crossed a network. | +| **Gateway/sidecar redaction** | Strip in a proxy. | Another hop, and it cannot know which field is sensitive in _your_ domain. | + +## What StitchAPI does + +### The exposure is binary, which makes it auditable + +Measured with seven distinct sentinels — a name, an email, an SSN, a nested +`profile.contact.mail`, an email inside a prose `note`, one in `contacts[1].email`, and a renamed +`primaryContactMail`. **Thirteen destinations carry all seven. Eleven carry zero. Nothing is +partially redacted.** + +| carries the whole body | carries none of it | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| the `result` event · `fileSink` at any non-zero cap · `.inspect().raw` · `.inspect().data` · `JSON.stringify(inspect())` · `.report()` · `StitchError.body` · `JSON.stringify(err)` · the cache entry | `start` / `progress` / `done` / `error` events · `consoleSink` · `loggerSink` · `otlpSink` · `StitchError.message` · `String(err)` · `err.stack` | + +Exactly **one event** carries the response body — `result`, on `data`. "The event spine leaks" is +really "one event leaks", which is a much smaller thing to reason about. + +### An `output` allowlist genuinely filters — 9 lines + +This is the answer, and it works because of an asymmetry +[measured in scenario 20](/docs/scenarios/precision-loss): the engine serves the **validated** +value, so `output` filters where `input` merely checks. + +```ts +const SAFE = z.object({ + id: z.string(), + status: z.string(), + total: z.number(), +}); + +const getCustomer = stitch({ url: '…', output: drift(SAFE) }); +``` + +A four-field schema took the JSONL sink, the `result` event, the console line, the cache entry +and the whole `.inspect()` wrapper from **7 sentinels to 0** — at depth (`profile.contact` gone) +and inside array elements (`contacts[].email` gone), **without naming a single PII field +anywhere**. That is the allowlist property: it survives the vendor adding a field. + +### `drift()` gives you the inventory without the data + +Wrapped in `drift()`, the same allowlist emits a value-free record of everything it stripped — +seven `undeclared` findings whose paths name the fields and whose details are **kinds only** +(`undeclared field (string)`), with zero sentinels across every finding, every drift event and +every sink. That is exactly the _log the detection, not the data_ shape the guidance recommends. + +When the vendor adds `taxId` at three levels, you get exactly three new `undeclared` findings. + +### Declarative credentials never enter the spine + +`auth.apply` runs on a request clone inside the attempt loop, while the `start` event was built +from the pre-auth request — so a `bearer` token or `apiKey` in query or cookie is **0 of 3** even +for a naive custom sink. Hand-rolled request credentials are scrubbed by every built-in sink: +`"authorization":"[REDACTED]"`, a scrubbed `url`, and no headers at all on console, logger or +OTLP. + +## What StitchAPI does not solve + +1. **`sensitive: true` does not mean "do not log".** It changed the leak at **1 of 11** + destinations — the cache. Across all of `packages/core/src` there is exactly **one** read of + the value, `engine.ts:1022`, inside `ensureCache`. No sink, no trace module and no event + builder mentions it. It means _do not persist this to the cache_, and while it is set the + JSONL sink still writes the full body to disk. It also survives onto the public `__config`, so + `.report()` prints `"sensitive":true` beside the record it did not protect. +2. **`.inspect({ redact: true })` removes 0 of 7.** The shared denylist is the _credential_ list + (`token`, `secret`, `password`, `apikey`, `signature`, …) reused — and no PII field name is on + it. `redact: ['mail', 'email']` does work, at depth and across array elements, but only for + names you enumerate; a renamed key and an address in free text are unreachable by + construction. There is no stitch-level or process-level default — ADR 0018's `defaultInspect` + was never implemented. +3. **`JSON.stringify(err)` leaks the whole record; `console.error(err)` does not.** + `StitchError.body` is an own enumerable property while `message` is not. So `err.stack` and + `String(err)` are clean, and **`logger.error({ err })` is the leak.** The same shape repeats on + success: ADR 0016's non-enumerability protects `.inspect().raw` and nothing else, because + `.inspect().data` holds the same record enumerably. +4. **A validation error can carry the value into a log.** `validationErrors` + (`drift.ts:50-56`) copies the validator's own message into `detail`, and Zod's enum message + quotes back the value it received — so an SSN reaching an enum field is written out verbatim. + Measured reaching the JSONL file, `consoleSink` **and** `loggerSink` — the two + sinks that are otherwise 0 of 7. Only OTLP stays clean, because it drops `detail`. +5. **The `fileSink` body cap is a size control, not a privacy control.** On a 2.9 KB body all + seven sentinels still persisted into the `preview`; only an eighth planted past character 2048 + was absent. It keeps a _prefix_, so reordering the vendor's JSON changes which fields leak. +6. **Only `hooks.onResponse` covers the failure path.** On a 200, `interpret`, `transform` and the + hook are equivalent. On a 500 the engine throws carrying the untouched response, so + `StitchError.body` stays at 7/7 under `transform` and under a stripping `interpret` — the hook + holds at 0/7 only because it _mutated_ `ctx.res.body` in place. Its type is `(ctx) => void`, + and mutating the body there is nowhere described as a privacy mechanism. +7. **The boundary and the drift signal are mutually exclusive.** Stripping in the hook takes + `drift()` to **0 findings**, because drift diffs exactly the bytes the boundary removed. + Having both means re-implementing `drift.ts`'s walker — 23 lines, and not exported. +8. **Nothing here is upstream of the `Adapter`**, which read the bytes first. A spy inside the + transport still sees all seven. + + + **A credential that rides the payload is treated exactly like PII.** An + `access_token` in a *response* body, or a `client_secret` in a *request* + body, is written to the JSONL log in full, because that sink's redactor is a + five-name **header** denylist rather than the deep secret-key scrubber. The + same file already ships that scrubber and applies it to a request body for + the `serve` SSE transport — the disk sink simply never calls it. + `redactHeaders` is the one config-reachable way to point the redactor at a + body key, and its type and JSDoc both say "header names", so nothing tells + you that works. + + +## The assembled setup + +**42 executable lines across 2 seams** — `hooks.onResponse` plus `output: drift(SAFE)` — reaches +**0 of 9 destinations on both the success and the failure path**, and still recovers a names-only +inventory of every undeclared field. Of those 42 lines, 23 re-implement the walker `drift.ts` +already contains. + +The cheaper variants are honest about their edges: the allowlist alone is 9 lines and 1 of 10 on +a 200 (only `.inspect().raw`, by design) but leaves the 500 path untouched; the boundary alone is +7 lines and 0 of 10 on both paths, at the cost of the entire drift signal. + +## See also + +- [Trace sinks](/docs/guides/observability/trace-sinks) — the sinks measured above +- [Drift](/docs/guides/validation/drift) — the inventory channel +- [Scenario: the ID that changed on the way in](/docs/scenarios/precision-loss) — where the + `output`-uses-its-parsed-value asymmetry was measured +- [Scenario: the agent picks the arguments](/docs/scenarios/agent-holds-the-tool) — the other + scenario about data reaching somewhere it should not diff --git a/apps/docs/content/docs/scenarios/precision-loss.mdx b/apps/docs/content/docs/scenarios/precision-loss.mdx new file mode 100644 index 00000000..a9538ae0 --- /dev/null +++ b/apps/docs/content/docs/scenarios/precision-loss.mdx @@ -0,0 +1,158 @@ +--- +title: 'The ID that changed on the way in' +description: 'JSON.parse turns a 64-bit snowflake into a different number, silently. wire.response text plus transform recovers the exact digits in 16 lines — and a bigint in params vanishes.' +prerequisites: ['/docs/reference/config-types', '/docs/guides/validation/drift'] +--- + +## The problem + +You integrate an API whose IDs are 64-bit integers — Discord and Twitter/X snowflakes, database +primary keys. The vendor sends them as JSON numbers. Measured through StitchAPI's real default +transport: + +| sent | received | | +| --------------------- | --------------------- | --------------- | +| `1234567890123456789` | `1234567890123456768` | off by 21 | +| `9007199254740993` | `9007199254740992` | 2⁵³+1 | +| `9223372036854775807` | `9223372036854775808` | int64 max | +| `9007199254740991` | `9007199254740991` | control, intact | + +`JSON.parse` produces IEEE 754 doubles, so integers above 2⁵³ are not all representable. No +throw, no warning. + +Three things make it hard rather than merely annoying. **The damage happens before your code +runs** — by the time any hook or schema sees the value it is a `Number` and the digits are gone. +**It is data-dependent, so it arrives on a date**: IDs below 2⁵³ round-trip perfectly, and +snowflakes are time-ordered, so the failure lands fleet-wide at once. And **the fix is not +local** — not using `JSON.parse` means values become `BigInt` or `string`, which breaks +`JSON.stringify`, cache serialisers, arithmetic and every validator expecting `number`. + +It is also a cross-language interop bug specifically: Python and Java parse the same payload +exactly, so the vendor's tests pass and it is entirely on your side of the wire. + + + **Money is a different problem wearing the same hat.** `19.99` round-trips + through the wire fine, because the nearest double's shortest form _is_ + `"19.99"`. The value is still inexact (`19.98999999999999843681`), so + decimals fail in **arithmetic**, not in **transport**. Only integers above + 2⁵³ are a wire-fidelity bug. + + +## The common solutions + +| Approach | What it is | Where it breaks | +| --------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------- | +| **Use the vendor's string field** | Read `id_str` instead of `id`. | Correct and free — when the vendor provides one. Most do not. | +| **`json-bigint` / custom parser** | Replace `JSON.parse` wholesale. | Correct at the boundary; now every consumer must handle `BigInt`. | +| **Reviver on `JSON.parse`** | `JSON.parse(text, reviver)`. | **Does not work** — the reviver receives the already-parsed `Number`. | +| **Regex the raw text** | Quote big integers before parsing. | Works, and is a JSON parser written in regex. Breaks on numbers inside strings. | +| **Keep everything as strings** | Treat IDs as opaque text. | The most robust answer, enforced by convention across every layer. | + +## What StitchAPI does + +### The repair is config, not a custom adapter — 16 lines + +`wire: { response: 'text' }` is an `else if` branch at `http-adapter.ts:123` that returns +**before** the JSON branch at `:135`. So the stock `fetchAdapter` hands you the verbatim bytes +and `transform` runs pre-parse: + +```ts +const getThing = stitch({ + url: 'https://api.vendor.test/v1/things/{id}', + wire: { response: 'text' }, // stop the transport parsing + transform: parseBigIntsAsStrings, // parse it yourself + output: z.object({ id: z.coerce.string() }), +}); +``` + +Measured end to end, this delivered the **exact sent digits** `1234567890123456789`, with zero +findings and a result that `JSON.stringify`s with no replacer. The `transform` is a +single-pass scanner — correct where a regex is not, leaving +`"order 1234567890123456789 shipped"` and escaped-quote strings untouched. + +### A detector that stays loud without failing the call — 15 lines + +```ts +output: drift( + z.object({ + id: z + .number() + .transform((n) => (Number.isSafeInteger(n) ? n : String(n))), + }), +); +``` + +produces `warn|coerced|id|number -> string`, inserts a `drift` event into the spine, and the call +still resolves `ok: true`. So it reaches `.inspect().findings`, `.report()`, any `TraceSink`, and +`loggerSink` at warn level — because a finding's level _is_ its log level. + +Over 20,000 random Discord-range snowflakes: **zero false negatives**, 0.535% false positives +(roughly the spacing of representable doubles at 1.1×10¹⁸). Walking the boundary shows +`lossless=false` never co-occurs with `flagged=false`, so **false negatives are impossible**, not +merely unobserved. + +### Three lossless paths, and two that are not + +Splitting by **decoder**, not by surface. `stream` with the default `decode: 'bytes'`, +`decode: 'lines'`, and `download` all bypass the JSON branch entirely and deliver verbatim bytes. +`decode: 'ndjson'`, `decode: 'json'` and `sse` all corrupt, because each calls `JSON.parse` in +its own file. Note a bare 19-digit SSE payload _is_ valid JSON, so even an unstructured `data:` +line corrupts. + +### Two places the library already got this right + +`cache.ts:42` tags bigint cache keys deliberately, so `42n` cannot collide with the string +`"42n"` — two identical bigint queries coalesced to one call. And `trace.ts` ships a `bigintSafe` +replacer so tracing cannot break the call it observes, which makes a `fileSink` the one +diagnostic surface in this whole scenario that ends up holding the vendor's real digits. + +## What StitchAPI does not solve + +1. **The default path corrupts, and says nothing.** The whole spine is `start`, `progress`, + `result`, `done` — **four events, zero drift, zero error, zero info** — and the sent digits + appear nowhere in it. `.report()` adds nine keys and no findings. Nothing downstream is + withholding a warning it could have given: `hooks.onResponse`, `Surface.interpret`, + `.inspect().raw`, `.report()` and a `TraceSink` all hold the already-parsed number. `raw` + means pre-**validation**, not pre-parse. +2. **A `bigint` in `params` silently vanishes.** `expandTemplateVar` (`util.ts:392`) branches on + `string | number | boolean`, so a bigint produces `…/v1/things/` — no error, no event. The + sibling `query` slot handles bigint **exactly**, and a `form` body does too. Of four outbound + positions bigint works in two, throws in one (JSON body — loud, and correct), and silently + vanishes in one. **The two URL positions disagree with each other.** +3. **Reading an id and handing it straight back is eight lines and wrong.** A number in `params` + goes out as `1234567890123456800` — a **third** distinct digit string, and not the one a + debugger shows you. That is the `Unknown Channel` shape, with nothing reported. +4. **`wire.response` and `transform` are independent keys.** Set the first and forget the second + and the call silently returns a **string** instead of an object. They are independent at the + type level too: `transform` is `(body: unknown) => unknown`, so a parser written + `(text: string)` does not typecheck in the slot even though `wire.response: 'text'` guarantees + a string at runtime. +5. **`transform` is redacted from `__config`**, so "is this stitch repaired?" is only half + auditable — `wire` shows, `transform` does not. +6. **`wire.response` is an HTTP key** and does nothing for `sse` or `stream({ decode: 'ndjson' })`, + which parse in their own files. +7. **The seam that solves this has no guide page.** `wire.response` appears in the docs only in + passing, in the GraphQL guide's list of what `wire.body` does _not_ do. The one config key + that recovers a corrupted ID is effectively undiscoverable. +8. **The detector buys visibility, not correctness.** The value it reports, + `1234567890123456800`, is still not the value the vendor sent. + + + Under a BigInt repair, `JSON.stringify(report)` throws `Do not know how to + serialize a BigInt` — and `.report()` is documented as safe to log. A + JSON-backed `store` throws on the write and the throw is **fatal** (`ok: + false`), not a silent cache miss. `memoryStore` survives, because it holds + values by reference and never encodes. Adding a bigint replacer to a JSON + store fixes the throw and introduces a subtler bug: `typeof data.id` is + `bigint` on a cache miss and `string` on a hit — a type that depends on + cache state, which a cold-cache test suite never sees. + + +## See also + +- [Config types](/docs/reference/config-types) — where `wire.response` lives +- [Drift](/docs/guides/validation/drift) — the channel the detector reports through +- [Scenario: the export that eats the heap](/docs/scenarios/large-response-memory) — the other + scenario decided by which decoder runs +- [Scenario: a canary rollout of a response-shape change](/docs/scenarios/intermittent-drift) — + `drift()` used for its intended purpose diff --git a/apps/docs/content/docs/scenarios/provider-failover.mdx b/apps/docs/content/docs/scenarios/provider-failover.mdx new file mode 100644 index 00000000..912ce4a1 --- /dev/null +++ b/apps/docs/content/docs/scenarios/provider-failover.mdx @@ -0,0 +1,159 @@ +--- +title: 'Failing over to the backup provider' +description: 'Everything per-provider is free and declarative. The routing between them is entirely yours — and the combinator named for this job bills you twice on every successful call.' +prerequisites: + ['/docs/reference/helpers', '/docs/guides/resilience/circuit-breaker'] +--- + +## The problem + +You depend on a provider that will eventually be down — an LLM API, an SMS gateway, a payment +processor — so you line up a second one of the same shape. + +Two techniques wear similar clothes: + +- **Failover** — try the primary; on failure, try the backup. **One** call on the happy path. +- **Hedging** — fire both immediately, take the first. **Two** calls, always, for better tail + latency. + +Picking the wrong one is a bill, not a bug report. + +And the trigger has to **classify**, not just notice. The industry consensus is consistent: +`404`, `429` and `5xx` are availability errors — try the next provider. A `400` is a bad +request — **stop**, because your payload is malformed and the backup will reject it identically. + + + **`any()` is named for failover and priced as a hedge.** Its docstring says *"failover across + interchangeable sources — a primary and a mirror, two regions, two providers"*, over an + implementation that starts **every** member eagerly. + + Measured: ten calls where the primary succeeded every time cost **10 primary + 10 backup + requests — 20 requests for 10 answers**, 100% amplification against a provider that never + failed. The same providers under `try`/`catch`: `[10, 0]`. + + **"The losers are auto-cancelled" is not "the losers are free."** The abort runs in a + `finally` *after* the winner settles, so the loser's request always arrives: 10/10 backup + requests completed, **zero** aborted. And `any` has **no preferred member** — a healthy + primary that was merely 10 ms slower *lost*, and was aborted mid-flight. + + + +## The common solutions + +| Approach | Where it breaks | +| -------------------------- | ---------------------------------------------------------------------------------------------------- | +| Sequential fallback | The correct default — one call in the happy path. Adds the primary's timeout to the failure path. | +| Concurrent "first success" | Best latency, **double spend on every call** — including the 99% that didn't need it. | +| Hedge after a delay | The nuanced answer. Needs a threshold, and amplifies an outage exactly when you can least afford it. | +| Gateway / router | Complete, and a third party in the path plus a bill. | +| Retry, not failover | Right for a `429`; useless when the provider is genuinely down. | +| Classify then route | What everyone converges on, and what hand-rolled failover usually skips. | + +## What StitchAPI does + +**Everything per-provider is free and declarative — and that's most of the work.** Two +providers with different origins, paths, auth strategies, retry policies, breakers, timeouts +and response shapes compose with zero glue: + +```ts +const primary = stitch({ + url: 'https://primary.example.com/v1/complete', + auth: bearer(env('PRIMARY_KEY')), + retry: { attempts: 2, on: [429, 503] }, + circuit: { failures: 3, cooldown: '30s', key: 'llm:primary' }, + pick: 'choices.0.text', + transform: (v) => ({ provider: 'primary', value: v }), // attribution +}); +// …and a `backup` with a different path, `x-api-key` auth, and `pick: 'output'`. +``` + +Measured: neither credential appeared on the other provider, and per-stitch `pick` normalised +two different response envelopes into one string. + +**The routing between them is yours** — about 30 lines, on `linked`: + +```ts +const FAILOVER_ON = new Set([404, 408, 429, 500, 502, 503]); + +const answer = await linked(async (run) => { + try { + return await run(primary, input); + } catch (err) { + // A 400 means the REQUEST is broken — the backup will reject it too. + if (!isStitchError(err) || !FAILOVER_ON.has(err.status ?? 0)) throw err; + return run(backup, input); + } +}); +``` + +**Measured:** 10 successful calls sent `[10, 0]`, every one credited to the primary. A `400` +stopped the chain at `[1, 0]` with a real `StitchError` 400 and the provider's `invalid_request` +body intact. A `503` retried the primary twice, then failed over — `[2, 1]` — in **one trace +tree** with the spine `primary ← root, backup ← primary`. With both providers down the caller +got the **last real error**, not an aggregate. + +**30 lines against 104** hand-rolled for the same feature set: the library carries ~74%, all of +it per-member, and **zero** of the routing. + + + Use `linked` rather than a bare `try`/`catch`. Both produce the same request + counts, but `linked` measured **one** traceId with a `primary → backup` + spine, while the bare version produced **two unrelated root traces** — the + failover becomes invisible to an on-call engineer at exactly the moment it + matters. + + +## What StitchAPI does not solve here + +1. **No sequential-fallback combinator.** `all`, `any` and `race` are one eager implementation + with three different joins — each measured `[1, 1]` on a single call. `linked` is sequential + but returns a Promise, not a node, so the flow can't be nested, handed to a seam, or + introspected. +2. **No failure classification for routing.** `retry.on` is exactly the right vocabulary, scoped + to the wrong target — it re-hits the _same_ endpoint. +3. **`AggregateError` drops everything you'd route on.** `status` and `body` are both + `undefined`; the actionable `400` survives only inside `.errors[0]`, which no `StitchError` + API points at. +4. **No winner identity.** The result is the winner's raw body, and the `pick` that normalises + two envelopes destroys the only attribution. The group emits **zero** events of its own — + a composition is not a span — so the trace can't break the tie either. Fix it with + `transform`, one line per member. +5. **A cancelled member emits nothing terminal** — `start`, `progress`, then silence. Zero + `error`, zero `done`. A span-based backend reads that as a leak. +6. **No hedge delay or threshold anywhere.** `race` measured **2.00×** amplification healthy + _and_ degraded — the doubling is the steady state, not an outage behaviour. The delayed + hedge everyone actually recommends took ~11 lines of raw `AbortController` and got the right + profile: `[10, 0]` healthy, `[10, 10]` degraded. +7. **A breaker is a health gate, not a budget gate** — 10 healthy calls with `circuit` on both + members still measured `[10, 10]`. +8. **`Composable` is not user-authorable.** `makeComposable` is unexported and the member gate + checks only the brand, so a hand-branded node **compiles** and then throws `TypeError`. + + + **A per-call header reaches every provider.** Config headers merge *under* input headers, and + each auth strategy only overwrites its own header name — so a per-call + `headers: { authorization: 'Bearer …' }` intended for the primary was measured **arriving at + the backup verbatim**. One vendor is handed another vendor's credential, silently, with no + type error. Keep per-provider credentials in each stitch's `auth`, never in the call input. + + **And two `url`-only stitches share one breaker.** With neither `name` nor `path` they both + key on the literal string `'stitch'` — measured, the primary's outage opened the *backup's* + breaker and fast-failed 3 of 5 healthy calls. Setting `name` fixes it completely, which + makes a diagnostic label load-bearing. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/stale-fixture.mdx b/apps/docs/content/docs/scenarios/stale-fixture.mdx new file mode 100644 index 00000000..1a85cbff --- /dev/null +++ b/apps/docs/content/docs/scenarios/stale-fixture.mdx @@ -0,0 +1,159 @@ +--- +title: 'The mock that passed for six months' +description: 'Your fake goes stale and the suite keeps saying green. Resilience and streams test perfectly offline — here is the definitive table of which time-driven features manualClock actually drives.' +prerequisites: ['/docs/guides/testing/mocking', '/docs/guides/validation/drift'] +--- + +## The problem + +You integrate a vendor API. You write tests. You cannot call the real API on every CI run — it +is slow, rate-limited, costs money and mutates state — so you test against something fake. + +Then the vendor changes the API, and **your tests keep passing.** + +This is the one scenario in this pass where the failure mode is the test suite actively lying to +you. Every approach trades one wrongness for another: a recording goes stale silently (the +classic shape is a cassette holding a session token that later expires, so the failure surfaces +somewhere unrelated); a hand-written mock encodes your misunderstanding faithfully, because the +same person wrote the mock and the code from the same reading of the docs; a vendor sandbox +exists for initial development rather than regression and lags production. The consensus is that +no single approach suffices — you need virtualisation for volume _and_ live verification for +accuracy. + +Which narrows the question for a client library to: **can it tell you your fake has drifted from +the real thing?** + +## The common solutions + +| Approach | What it is | Where it breaks | +| ---------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------- | +| **Record/replay cassettes** | Record real traffic once, replay forever. | Goes stale silently. Re-recording is a process, not a check. | +| **Hand-written mocks** | Fixtures you write from the docs. | Encode your misunderstanding faithfully. Cannot catch what you got wrong. | +| **Vendor sandbox** | The vendor's test environment. | Lags production, misses edge cases, slow, rate-limited. Not built for regression runs. | +| **Contract testing (Pact)** | Both sides verify a shared contract. | Needs vendor participation. Not available for a third party. | +| **Hit production in CI** | The only truly accurate option. | Slow, costly, mutating, and flaky for reasons unrelated to your code. | +| **Schema/contract snapshot** | Validate responses against a schema. | The honest middle ground — but only if the _same_ schema guards prod and the fixtures. | + +## What StitchAPI does + +### Resilience tests need no vendor and no waiting + +This is the strongest result. With `mockAdapter` + `manualClock` + `collectStitchEvents`, retry, +throttle and circuit behaviour is assertable to the millisecond with zero real time elapsed: + +- **Attempt counts**, three independent ways: `mockAdapter.callCount()` → `3`, + `result.attempts` → `3`, and five `progress` events phased + `request, retry, request, retry, request`. +- **Circuit transitions** — the whole closed → open → half-open → closed trace reads off + `callCount()` as `1, 2, 2 (blocked), 3, 4`, because an open circuit doesn't move the transport + count. +- **Throttle spacing**, exact and self-reporting: requests at virtual `0 / 500 / 1000` for + `'2/s'`, with `progress{phase:"throttled"}.waited` reading `500` then `1000`. +- **Backoff curves** are exact under the clock — requests at virtual `0 / 1000 / 3000` for an + `expo` base-1000 curve. + +### Streams are fully deterministic + +`streamThenError` over an `sse` stitch produced `start, progress, delta, delta, delta, error, +done` with all three deltas intact, `error.message: "socket reset"`, `done.ok: false` — and +**five repeat runs produced one distinct outcome, byte-identical.** `gatedStream` holds a +connection open on a promise you resolve; `sseStream` writes well-formed frames. + +### A shared schema catches a fixture that drifts from the contract + +Point the same `output` schema at production and at the fixture, and all four mutations fail the +call — a field removed, renamed, retyped, and nulled — with `drift()` naming each: +`error|invalid|paid|Expected boolean, received string`. + +### Targeting three environments is one config slot + +`extends: { baseUrl, adapter }` aims one endpoint object at fixture, sandbox and production, and +the difference is visible through the same schema: `ok`, `ok`, `contract violation (drift)`. A +`baseUrl` thunk (`string | (() => string)`) retargets between calls without rebuilding. + +## What StitchAPI does not solve + +1. **The scenario's actual direction is invisible offline.** Everything above catches the + _fixture_ drifting from the schema. The scenario is the _vendor_ drifting while the fixture + sits still — measured, that run is **test `ok: true` with zero findings, production + `ok: false`, five keys different**. Nothing offline closes it, because offline the only bytes + are the fixture's. The comparison that catches it is 8 lines and every seam it needs already + exists; what is missing is a place to put a call you are only allowed to make sometimes. +2. **`manualClock` drives six time-driven features and not the other six.** The table below is + the one to keep. A test written against a wall-clock row **passes without asserting + anything**. +3. **`mockAdapter` validates almost nothing.** It defaults `status` to 200 and lowercases header + names; beyond that it served statuses `999, -1, 0, 1.5, 200.7` and bodies of type `Date`, + `Map`, class instance, `undefined`, `function`, `bigint` and Symbol-keyed — all verbatim, + through the full engine. The consequence is a green test for code that cannot work: a fixture + built from `new Invoice(...)` gives the caller `data.total === 42` from a **prototype + getter**, where the same object over a JSON wire is `{"id":"inv_1"}` and `data.total` is + `undefined`. +4. **`stubStitch` runs none of the `input` schemas.** One line of calling code passing `42` where + the schema says `z.string()`: the real stitch errors and no request leaves the process; the + stub **resolves** and records `{"params":{"id":42}}`. There is no slot to hand it the + contract — `StubStitchOptions` is `{name,status,config,events}`, and `config` is the redacted + read-out shape, which carries no schema. +5. **A fixture cannot say when it was recorded.** Of all exported names across the main entry and + `stitchapi/testing`, exactly one matches `/fixture|cassette|record|snapshot|stale|fresh|expire/` + — `adapterContractFixture`, the transport echo contract for plugin authors. `__config` has no + metadata slot, so "recorded on 2026-02-04" is not expressible. +6. **Retry backoff delays are absent from the event stream.** `progress{phase:"retry"}` carries + `detail: "status 503"` and `waited: undefined`, where the throttle and reconnect paths both + set `waited`. You can assert the backoff, but only by reading `clock.now()` yourself. + +### Which features `manualClock` actually drives + +| Feature | Driven by | Measured | +| ------------------------------- | ----------------- | -------------------------------------------------------------------------- | +| `retry` backoff | **`manualClock`** | `advance(5000)` → 3 calls; `advance(0)` → 1 call, pending | +| `throttle` rate | **`manualClock`** | `advance(3000)` → 3 calls | +| `throttle` concurrency | **`manualClock`** | holder releases on virtual time → queued callers proceed | +| `circuit.cooldown` | **`manualClock`** | `advance(60_000)` past a 30s cooldown → half-open probe reaches the vendor | +| `timeout` (per-attempt) | **`manualClock`** | `advance(2000)` past a 1s timeout → error | +| `Retry-After` | **`manualClock`** | reads the injected clock — **and that is the trap**, see below | +| **`timeout.total`** | wall clock | a 1000ms budget survived **2700 virtual ms** across 3 attempts, `ok: true` | +| **`cache.ttl`** | wall clock | `advance(600_000)` past a 60s TTL still served the cached entry | +| **`memoryStore` TTL** | wall clock | a 1s entry survived 60,000 virtual ms | +| **event `at` / `done.elapsed`** | wall clock | `at = 1785941469178` while `clock.now() = 0`; `done.elapsed` reads `0` | +| **OAuth2 token expiry** | wall clock | 600,000 virtual ms past a 60s `expires_in` refetched nothing | +| **AWS SigV4 signing date** | wall clock | `new Date()`, and no `clock` option exists to pass | +| `paginate` | no time | no inter-page delay knob; 3 pages fetched at `clock.now() = 0` | + + + **`Retry-After` is a trap precisely because it honours the clock.** + `parseRetryAfter` computes `httpDateEpoch - clock.now()`, and + `manualClock()` starts at `0` — so an HTTP-date header meaning "5 seconds" + becomes a wait of roughly **20,000 days**. Start the clock at a realistic + epoch if a fixture carries a dated `Retry-After`. + + + + **`timeout.total` does not "ignore" the clock — it resets.** The per-attempt + clamp _does_ fire on virtual time; the wall-anchored part is the deadline + (`wallT0 + total`). Virtual sleeps never drain it, so each attempt gets the + full budget back. On a real clock it behaves correctly. This is specifically + a testing unsoundness, and it was the pass's own prediction that got the + mechanism wrong. + + +## The best available setup + +**98 executable lines across 5 seams**, closing four of the five gaps: a wire-shape guard +(rejects the class-instance fixture with `fixture is not a wire shape ($: LiveInvoice)`), a +`contractStub` that runs the input schema, a config lint that refuses to pair a `manualClock` +with `cache` or `timeout.total`, and a fixture datestamp +(`getInvoice recorded 2026-02-04 (182d old)`). + +The fifth does not close offline. Dating a fixture fails your suite on the **calendar**, not on +the drift. The only thing that actually detects the drift is a live comparison — measured +returning `DISAGREE live=contract violation (drift) fake=ok` — and it needs a real call. + +## See also + +- [Mocking guide](/docs/guides/testing/mocking) — `mockAdapter`, `stubStitch`, `manualClock` +- [Drift](/docs/guides/validation/drift) — what `output` findings mean +- [Scenario: a canary rollout of a response-shape change](/docs/scenarios/intermittent-drift) — + detecting vendor drift in production, which is the half this scenario cannot do offline +- [Scenario: a stream that fails after 800 tokens](/docs/scenarios/mid-stream-failure) — the + behaviour `streamThenError` reproduces deterministically diff --git a/apps/docs/content/docs/scenarios/unconfirmed-write.mdx b/apps/docs/content/docs/scenarios/unconfirmed-write.mdx new file mode 100644 index 00000000..d4515208 --- /dev/null +++ b/apps/docs/content/docs/scenarios/unconfirmed-write.mdx @@ -0,0 +1,128 @@ +--- +title: "The charge you can't confirm" +description: 'A timeout tells you nothing about the server. idempotency.keyOf fixes the restart and the race in configuration alone — the default key does not, and it double-charged a re-driven job.' +prerequisites: + ['/docs/guides/resilience/idempotency', '/docs/recipes/idempotent-writes'] +--- + +## The problem + +You POST a charge. The connection times out. **You have no idea whether the money moved.** + +A timeout does not distinguish "the request never arrived" from "it was processed and the +response was lost". No client-side care removes that — you can only make the _retry_ safe. And +idempotency keys, which do, bring their own edges: the replay returns the **original** outcome +including a failure, the key has a **TTL shorter than your job queue**, and the same key with a +different body is an **error**. + +## The common solutions + +| Approach | Where it breaks | +| ---------------------------------- | ------------------------------------------------------------------------------------- | +| Retry blindly | Double-charges. The failure the whole scenario exists to prevent. | +| Never retry a write | Safe and expensive — every blip becomes a support ticket. | +| Key minted per call | Correct for retries _inside_ the call; useless across a restart. | +| Key derived from the business fact | The right answer — stable across processes, restarts and queues. | +| Query-then-decide | The standard recovery, and it must run **first** — after the write it can't un-write. | +| Persist intent first | Durable, and now you own a two-phase workflow. | + +## What StitchAPI does + +**`idempotency.keyOf` fixes the restart _and_ the concurrency race in configuration alone.** + +```ts +const charge = stitch({ + method: 'POST', + url: 'https://api.vendor.com/charges', + // Derived from the business fact — stable across processes, restarts and queues. + idempotency: { + keyOf: (input) => `charge:${(input.body as Charge).invoiceRef}`, + }, + retry: { attempts: 3 }, +}); +``` + +**Measured**, across six workloads including a crash-and-re-drive, a lost response, a TTL +expiry and two concurrent runs: + +| | charges created | intended | +| ---------------------------------------------- | -------------------------- | -------- | +| `idempotency: true` (the default), no recovery | **8** — two duplicates | 6 | +| derived `keyOf` + a query-first recovery | **5** (the sixth declined) | 6 | + +Two things work well and are worth naming. The **same key is carried on every attempt** of a +call — 3 attempts, 1 key, 1 charge — so a response lost _after_ the charge was processed is +**recovered** by the retry replaying the stored `200`. And a **cached failure is not retried by +default**: a stored `500` produced exactly 1 request under `attempts: 4`, because 500 isn't in +the default `retry.on`. + + + **`idempotency: true` is per call, not per intent.** The default key is a `randomUUID()` + minted when the request is built, so a queue re-driving a job after a crash mints a *new* + key. Measured: **2 distinct keys, 2 charges, for 1 intended payment.** + + And the library warns about exactly this — unless you follow the advice. The nudge fires only + when there is a random key *and* no `retry`, on the reasoning that a retry is the thing a + random key does protect. True, and narrower than most readers will assume: **adding `retry` + silences the warning while leaving the restart case wide open.** + + + +## The recovery has to run first + +A query-then-decide recovery only works **before** the write. Running it after a failure still +double-charged on the TTL workload — because a recovery that runs after the write cannot +un-write it. That is 19 of the 43 lines, and no hook can do it: a hook cannot change the +outcome. + +## What StitchAPI does not solve here + +1. **Restart safety by default** — see the callout. `keyOf` is the fix and it is opt-in. +2. **TTL expiry.** A prune turns a correct, stable key into a second charge — measured **2 + charges at a 25 h delay against a 24 h TTL**, 1 charge at 23 h. The duplicate arrives as a + clean `200` with **no replay marker**, so nothing client-side can notice. `timeout.total` is + per call and cannot bound the gap. +3. **Timeout disambiguation.** A dropped request (**0 charges**) and a lost response (**1 + charge**) produced **field-for-field identical** errors: `StitchError`, `status: undefined`, + `attempts: 1`, `timed out after 5000ms`, `body: undefined`. The `TimeoutError` class is + flattened away and unexported, surviving only in `hooks.onError`. +4. **"Don't retry into the unknown."** `retry.on` gates statuses only — a transport failure is + retried even with `on: []`. The only lever is `attempts: 1`. +5. **"Don't retry a replayed failure."** Inexpressible declaratively: `interpret` runs _after_ + the retry check, and `retry.on`'s predicate sees only the status. The vendor's replay marker + is visible to `hooks.onResponse` and absent from `StitchError`, which carries no headers. +6. **No event carries the idempotency key** — so with the random default, a query-by-key + recovery is impossible: you cannot ask about a key you never learned. +7. **Nothing expresses "sticky for a decline, fresh for a blip."** A stable key makes a recorded + failure sticky for the whole TTL — measured, a declined card stayed declined — while the + random key never reaches the record and simply charges again on the second run. Both + behaviours are defensible; neither is selectable. + + + **`keyOf: (i) => JSON.stringify(i.body)` fails by charging twice, not by erroring.** Key + order alone moved the hash — measured **2 keys, 2 charges, statuses `[200, 200, 200]`**, no + `409` anywhere, because the vendor never saw the same key twice. Derive from a business + reference, or canonicalise before hashing. An input **schema does not canonicalise** what + `keyOf` sees. + + Two smaller ones: **pagination mints a key per page** (3 pages, 3 keys), and + **`verdict: { accept: [409], flag: 'ok' }` swallows an idempotency conflict**, returning the + error payload as data — the same trap as + [the signature scenario](/docs/scenarios/expiring-signatures). + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/unstable-pagination.mdx b/apps/docs/content/docs/scenarios/unstable-pagination.mdx new file mode 100644 index 00000000..53c3bc46 --- /dev/null +++ b/apps/docs/content/docs/scenarios/unstable-pagination.mdx @@ -0,0 +1,148 @@ +--- +title: 'The page that moved while you were reading it' +description: 'Offset pagination over a live collection silently returns wrong lists. A client cannot fix that — but it should not report a clean run over data it lost.' +prerequisites: + ['/docs/guides/data/pagination', '/docs/guides/validation/validation'] +--- + +## The problem + +You page through a live collection — `?offset=0&limit=100`, then `100`, then `200` — while +other people are inserting and deleting rows. You end up with a list, and it is quietly wrong. + +**Offset is a position in a result set, not a position in the data.** The two cases go in +opposite directions, and it's worth getting them the right way round: + +- **An insert behind your cursor** pushes rows to _higher_ indices, so your next fixed offset + lands on a row you already read. Measured: `skipped []`, **`duplicated ["r04"]`** — 11 items + for 10 distinct rows. An offset insert can never cause a skip. +- **A delete behind your cursor** pulls rows to _lower_ indices, so your next offset jumps past + one. Measured: **`skipped ["r05"]`**, `duplicated []`. + +And the case people don't believe until they see it: **a non-unique sort key breaks it with no +writes at all.** Ten rows in, ten rows out, nothing created or destroyed — measured +`skipped ["r05"]` _and_ `duplicated ["r03"]`, because the server returned tied `created_at` +values in a different order on each query. + +The fix is **keyset (seek) pagination** on a composite `(sort, id)` cursor. That is a _server_ +capability — no client can make an offset API consistent. What a client can do is **notice**. + + + **Every damaged run above reported success.** `ok: true`, `error: null`, + `findings: []`, `status: 200`, `attempts: 1` — byte-identical to a clean + run. Measured on the insert, the delete, the tie case, and two more below. + + +## The common solutions + +| Approach | Where it breaks | +| ------------------------- | ----------------------------------------------------------------------------- | +| Offset/limit, as offered | Duplicates on insert, skips on delete, both under ties with no writes at all. | +| Keyset / seek pagination | The correct fix — if the vendor implemented it. | +| Snapshot / point-in-time | Ideal where offered. Rare in REST. | +| Sort by an immutable key | Removes the mutable-sort case, not the insert/delete cases. | +| Client-side dedupe by id | Fixes duplicates, leaves skips invisible — **and see the callout below**. | +| Reconcile against `total` | The obvious check, and it misses the case that matters. | + +## What StitchAPI does + +**Keyset is four lines, and it works.** `next` receives the previous page's raw body, so a +composite cursor is straightforward: + +```ts +paginate: { + next: (body) => { + const rows = (body as Page).rows; + const last = rows.at(-1); + return last + ? { query: { after_ts: last.created_at, after_id: last.id } } + : undefined; + }, + items: (v) => (v as Page).rows, +} +``` + +**Measured:** against a real seek endpoint, every workload that broke offset came back complete +and clean — the insert (which had cost a duplicate), the delete (a skip), and the ties (both) — +`skipped []`, `duplicated []`, 10 rows in cursor order. + + + **Sending a composite cursor does not make an endpoint a seek endpoint.** + The same four lines against a vendor that *accepts* `(after_ts, after_id)` + but orders by `created_at` alone lost `["r03"]` on one collection and + duplicated `["r13"]` on another — with zero writes. The client half of + keyset is four lines and it is not the half that decides. + + +**Detection, when only offset is on offer, is yours to write.** The library ships no comparison +of any page to any other. The seam that works is `output` — a validator over the _aggregated_ +array, which runs after the loop: + +```ts +output: reconcile, // dedupes, or fails the run with the ids named +``` + +Measured over 8 workloads: **zero false negatives** — every run that lost rows was flagged — +and 3 false alarms on undamaged runs, which is the right trade for a sync job. + +**The signal that carries the delete case is not the one anyone writes.** It is that the +**declared `total` moved** (10 → 9). A plain length-vs-total check fired **0 of 4** times; the +deduped variant missed the delete entirely — because the delete removed a row from `total` at +the same instant it removed one from your result, so the arithmetic balances perfectly while +`r05` is gone. + +## StitchAPI vs the common solution + +**84 lines against 74 hand-rolled — the library version is longer.** The detection is identical +in both, and a raw paging loop is cheaper to write than the declarative equivalent. The two +agreed on rows and verdict across all 8 workloads. + +What the 74 lines don't have is the resilience stack: one `retry` line recovered a page that +answered `500` mid-run — 4 wire requests, 10 rows, nothing skipped. Every page runs inside its +own attempt loop, so retry, auth, throttle and the circuit breaker apply per page for free. + +## What StitchAPI does not solve here + +1. **No dedupe, no reconciliation, no page-to-page comparison.** Nothing in `paginate`'s three + fields (`next`, `items`, `pages`) looks at what the last page contained. +2. **Four different endings share one `break` and one successful result** — the collection + ended, a page came back empty, the page cap was hit, or drift emptied a page mid-run. Measured + `pages: 50` silently returning **200 of 220 rows**, `ok`, with no event distinguishing cap + from end. +3. **`total` is not surfaced anywhere structured** — it's a field in a body you have to catch + yourself, in `transform` or `hooks.onResponse` (`next` never sees the terminal page's). +4. **No per-page state.** Every cross-page fact is a closure you own — with the reuse hazard + below. +5. **You cannot both succeed and warn.** `output` either replaces the value or fails the run. +6. **`drift()` cannot express this.** Deduping an array re-indexes it, so the findings come back + as `coerced`/`undeclared` on element paths — none of them says "duplicate". +7. **`.report()` is a fresh probe.** It re-paginated the collection and did not reproduce the + duplicate at all — it describes a run it just made, never the run you made. + + + **Deduping inside the loop can cause the data loss it was meant to prevent.** `items` and + `transform` run *per page*, above the break. On a workload where page 2 repeated page 1 + verbatim, the deduper emptied that page, the loop treated zero items as the end, and the run + finished `ok` having **skipped `["r05"…"r10"]` — 6 rows lost by the fix**, against a declared + total of 14. Dedupe in `output`, after the loop, not in `items`. + + **And a deduping `items` on a reused stitch returns `[]` — successfully — on every call after + the first**, because the `seen` set outlives the call. The natural way to write it is defined + once and called many times. + + + +## See also + + + + + + + diff --git a/apps/docs/content/docs/scenarios/webhook-receipt.mdx b/apps/docs/content/docs/scenarios/webhook-receipt.mdx new file mode 100644 index 00000000..3a7b53f3 --- /dev/null +++ b/apps/docs/content/docs/scenarios/webhook-receipt.mdx @@ -0,0 +1,156 @@ +--- +title: 'Receiving a signed webhook' +description: 'StitchAPI does not receive webhooks — that is your server. Here is exactly where the line falls, measured, and what the library does own on the far side of it.' +prerequisites: + ['/docs/concepts/the-stitch', '/docs/guides/state/pluggable-store'] +--- + +## The problem + +Stripe, GitHub, Slack push events at an endpoint you host. You verify the signature, decide +it's genuine, and act. This is the only scenario in this section where **someone is calling +you** — and the [stitch is a per-call primitive, not a +server](/docs/concepts/the-stitch): inbound webhooks are your application's job. + +So this page is about **where the line falls**, and what's on each side of it. + +What makes the receipt half hard: + +- **Signature verification needs the exact bytes.** The provider signs the raw payload; any + JSON round-trip yields logically identical bytes that don't match. This is the famous + `express.raw()`-before-`express.json()` rule, and the most-reported webhook bug there is. +- **At-least-once means duplicates are normal.** Dedup on the event id, with a TTL longer than + the provider's retry window (Stripe: 3 days). +- **Order cannot be trusted.** `subscription.updated` can arrive before `.created`. +- **Ack fast.** Providers time out in seconds and retry on slowness. + +## The common solutions + +| Approach | Where it breaks | +| ---------------------------------------- | ----------------------------------------------------------------------------------- | +| Provider SDK verifier (`constructEvent`) | Correct, and the right answer per vendor. One per provider. | +| Framework raw-body middleware | The standard fix, and entirely about ordering — get it wrong and it fails silently. | +| Hand-rolled HMAC | Fine, and easy to get subtly wrong: constant-time compare, timestamp tolerance. | +| Webhook gateway (Svix, Hookdeck) | Most complete at scale. A third party in the path, and a bill. | +| Dedup on the event id | Necessary. The TTL is the trap — it must outlive the retry window. | +| Fetch-on-receipt | Makes payload order irrelevant. Costs an API call per event. | + +## Where the line falls + +**`serve` is not a webhook endpoint**, and the measurements are unambiguous. A real `serve()` +process answered **404** to a correctly signed Stripe POST at `/webhooks/stripe`, `/webhook`, +`/`, `/stitch` and `/hooks/v1/billing` — the route table is exactly `GET /` and +`POST /stitch/:name`. On the one route that reaches user code: + +- the body has already been through `JSON.parse`, and the deepest user-reachable seam got a + parsed object with **no raw string anywhere**; +- **the inbound headers are dropped entirely** — `stripe-signature` reaches nothing, so there + is no signature to verify even if you had the bytes; +- a form-encoded provider (Slack) is rejected `400 invalid JSON body` before any of it. + +The byte gap is real and measured: the provider signed **162 bytes**; a parse→stringify +round-trip produces **153** logically identical bytes, and verification returns +`bad-signature`. + +There is also **no inbound-signature primitive anywhere** in the packages. Enumerating all 72 +runtime exports, the four that match `/verif|hmac|…/` are BYO-plugin conformance suites. The +one HMAC in the repo is `@stitchapi/aws-sigv4`, whose key is imported with +`usages: ['sign']` — it structurally cannot verify. + +## What StitchAPI does own + +Everything after the ack, and it is worth having: + +```ts +const billing = seam({ + baseUrl: 'https://api.example.com', + auth: bearer(env('API_KEY')), + throttle: { rate: '10/s' }, +}); + +// Fetch-on-receipt: the payload is a hint; the API is the truth. +const getSubscription = billing.stitch({ + path: '/subscriptions/{id}', + retry: { attempts: 3 }, + timeout: { total: '10s' }, +}); +``` + +**Measured:** acting on payload order with a reversed pair left the app believing +`trialing/free` while the server said `active/pro` — a silent, self-inflicted downgrade. +Fetch-on-receipt converged both events on `active/pro` at a cost of 2 calls for 2 events, and +the call survived two 503s in 3 attempts with the retry, backoff, auth and deadline all as +config rather than handler code. + +The **dedup ledger** is also first-class, and the store is yours to supply — `redisStore`, +`cloudflareKvStore` and `denoKvStore` all implement the same interface, so durability is a +one-line change and `verifyStoreContract` proves a BYO one conforms (measured: all 11 rules, 0 +violations). + + + **Use `increment`, not `get`-then-`set`, to claim an event id.** Measured + with 3 concurrent deliveries of one id: `get`+`set` returned `[true, true, + true]` — three charges — while `increment(key, ttl)` returned `[true, false, + false]`, exactly one. The default `memoryStore` is also not durable: + `close()` clears it, so a deploy inside the retry window re-processes + everything still in flight. + + +## The boundary, as a number + +A complete, honest implementation — a `node:http` server you own, plus StitchAPI downstream — +rejects forged signatures (`400 bad-signature`), rejects a genuine MAC on a 10-minute-old +timestamp (`400 stale`, zero side effects), converges a reversed pair, and acks a duplicate at +**zero API calls**. + +| half | lines | what it is | +| ------------ | -------------------------------------- | ----------------------------------------------------------------- | +| **Receipt** | **154** (96 server + 58 `node:crypto`) | imports _nothing_ from `stitchapi` at runtime — one `import type` | +| **Reaction** | **63** | almost all config: one seam, two stitches, a 4-line version guard | + +**71% of the code by line, 100% by concern, is the half StitchAPI does not participate in.** +That is the honest shape of this scenario, and it is by design. + +## What StitchAPI does not solve here + +1. **Route ownership, raw bytes, and the inbound headers.** All three are absent from `serve`. +2. **HMAC verification and constant-time comparison.** No primitive exists. `xxh128` is + **unkeyed** and non-cryptographic — it will produce a plausible digest that authenticates + nobody. +3. **Replay/timestamp tolerance.** Yours. +4. **Non-JSON bodies.** `serve` rejects them before any user code. +5. **Any queue, outbox, dead-letter or backpressure.** `pipelineStages` on a maximally + configured stitch shows no stage matching queue/detach/defer — `result` is last, and the + pipeline never releases a caller early. +6. **The ack/work split.** Worse: `serve` consumes the run to completion before writing a + byte, so with `retry: { attempts: 3 }` and a 5 s backoff the ack went out only at attempt 3, + after **exactly 10 virtual seconds** — Stripe's timeout to the second. The retry policy + manufactures the duplicate it was meant to survive. +7. **Write-order conflict.** Fetch-on-receipt fixes _payload_ order, not _write_ order: two + concurrent handlers with snapshots v2 and v3 landed on v2 under last-write-wins. A version + guard recovered it — four lines the library does not express. +8. **`.deleted` events.** The fetch returns 404, indistinguishable from "never existed", so the + event type stays load-bearing. + + + **`void call(input)` is a no-op, not fire-and-forget.** It is the spelling anyone reaches + for to ack-then-continue, and it made **0 HTTP calls and raised 0 errors** — a stitch call + is a lazy thenable that starts on `.then`, so the work is silently dropped *after* the + provider was told 200. `void call(input).then(…)` does run it, and is then unsupervised: + unhandled it surfaced as an `unhandledRejection`, and `.safe()` reported the failure + nowhere. + + **And `serve` is unauthenticated by design.** An unsigned, forged body under the size cap + ran the stitch and returned 200. Anyone who can reach the port can run your stitches — it + is a local front door, not an internet-facing endpoint. + + + +## See also + + + + + + + diff --git a/docs/scenarios/LEDGER.md b/docs/scenarios/LEDGER.md new file mode 100644 index 00000000..fc88c5ed --- /dev/null +++ b/docs/scenarios/LEDGER.md @@ -0,0 +1,168 @@ +# Scenario ledger + +Real-world API integration scenarios researched by the `/loop` scenario pass. One row +per scenario, so later iterations don't re-cover ground. + +**Flow per scenario:** web research → capture in `docs/scenarios/.md` → a subagent +proves (or fails to prove) it with **runnable offline code** under +`docs/scenarios/proofs//` → then either a published page at +`apps/docs/content/docs/scenarios/.mdx` **or** an issue draft in +`docs/scenarios/issue-drafts/.md`. + +Issue drafts are **not filed** — they accumulate here for review when the loop stops. + +| # | Scenario | Slug | Verdict | Outcome | +| --- | ----------------------------------------------------- | ------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | OAuth2 rotating refresh tokens under concurrent calls | `oauth2-refresh-token-rotation` | achievable with user code | [page shipped](../../apps/docs/content/docs/scenarios/oauth2-refresh-token-rotation.mdx) + 1 issue draft (`params` footgun) | +| 2 | Cost-based rate limits reported in the response body | `cost-based-rate-limits` | achievable with user code | [page shipped](../../apps/docs/content/docs/scenarios/cost-based-rate-limits.mdx) + 1 issue draft (3 body-verdict footguns) | +| 3 | Batch writes with per-item partial failure | `batch-partial-failure` | achievable with user code | [page shipped](../../apps/docs/content/docs/scenarios/batch-partial-failure.mdx) + 1 issue draft (`paginate` silent data loss) | +| 4 | Async job triangle — submit, poll, download | `async-job-polling` | achievable with user code | [page shipped](../../apps/docs/content/docs/scenarios/async-job-polling.mdx) + 1 issue draft (clock + diagnostic side effects) | +| 5 | A stream that fails after 800 tokens | `mid-stream-failure` | achievable with user code (resumable feeds: **achievable outright**) | [page shipped](../../apps/docs/content/docs/scenarios/mid-stream-failure.mdx) + 1 issue draft (**SSE reconnect replays completed streams — a bug in #622**) | +| 6 | ETag revalidation and the bodyless 304 | `conditional-requests-304` | achievable with user code | [page shipped](../../apps/docs/content/docs/scenarios/conditional-requests-304.mdx) + 1 issue draft (`cache` cannot revalidate; surfaces can't see the principal) | +| 7 | Multipart upload and the mandatory abort | `multipart-upload` | achievable — but the library is a **bystander for the cleanup** | [page shipped](../../apps/docs/content/docs/scenarios/multipart-upload.mdx) + 1 issue draft (no compensation seam) | +| 8 | Receiving a signed webhook | `webhook-receipt` | **split — receipt OUT OF SCOPE by design, reaction in scope** | [page shipped](../../apps/docs/content/docs/scenarios/webhook-receipt.mdx) + 1 issue draft (`void call()` drops work) | +| 9 | One tenant's revoked token, everyone's outage | `multi-tenant-blast-radius` | achievable with user code (~3 strings per tenant) | [page shipped](../../apps/docs/content/docs/scenarios/multi-tenant-blast-radius.mdx) + 1 issue draft (**resilience has no tenancy axis — 9/9 blast radius**) | +| 10 | Failing over to the backup provider | `provider-failover` | achievable with user code (~30 lines of routing) | [page shipped](../../apps/docs/content/docs/scenarios/provider-failover.mdx) + 1 issue draft (`any()` priced as a hedge; per-call header broadcast) | +| 11 | Pagination over a live collection | `unstable-pagination` | achievable with user code — keyset in 4 lines; detection is yours | [page shipped](../../apps/docs/content/docs/scenarios/unstable-pagination.mdx) + 1 issue draft (dedupe in `items` **causes** data loss; 4 endings share one break) | +| 12 | A canary rollout of a response-shape change | `intermittent-drift` | achievable with user code — 9 declarative lines + ~84 for the rate | [page shipped](../../apps/docs/content/docs/scenarios/intermittent-drift.mdx) + 1 issue draft (a `coerced` finding can't grade the coercion; nullable has no level) | +| 13 | The export that eats the heap | `large-response-memory` | achievable with user code — one seam, ~75 lines, **NDJSON only** | [page shipped](../../apps/docs/content/docs/scenarios/large-response-memory.mdx) + 1 issue draft (**`.stream()` is not memory-bounded; `decode: 'json'` buffers**) | +| 14 | The signature that expired in your own queue | `expiring-signatures` | **ACHIEVABLE** — the queue/retry halves need no user code at all | [page shipped](../../apps/docs/content/docs/scenarios/expiring-signatures.mdx) + 1 issue draft (SigV4 ignores the injected clock; a skew 403 opens the breaker) | +| 15 | The charge you can't confirm | `unconfirmed-write` | achievable with user code — 43 lines, 19 of them the recovery | [page shipped](../../apps/docs/content/docs/scenarios/unconfirmed-write.mdx) + 1 issue draft (**`idempotency: true` double-charged a re-driven job; `retry` silences the warning**) | +| 16 | One list, a hundred follow-up calls | `n-plus-one-fanout` | achievable with user code — ~8 lines, the partial-failure branch | [page shipped](../../apps/docs/content/docs/scenarios/n-plus-one-fanout.mdx) + 1 issue draft (a coalesced failure is not shared; a `store` un-pools `pool: 'host'`) | +| 17 | The deprecation you never saw | `deprecation-headers` | achievable with user code — 3 seams, ~112 lines, **0 config keys** | [page shipped](../../apps/docs/content/docs/scenarios/deprecation-headers.mdx) + 1 issue draft (hooks can rewrite the call; seam-level `kind` is a compile error that works) | +| 18 | The agent picks the arguments | `agent-holds-the-tool` | **credential boundary held** (30 scans, 0 hits); argument boundary is the user's — safe exposure = 3 seams, 47 lines | [page shipped](../../apps/docs/content/docs/scenarios/agent-holds-the-tool.mdx) + 2 issue drafts (unfiltered MCP error channel leaks a query credential; input schemas check but never filter) | +| 19 | The mock that passed for six months | `stale-fixture` | **split** — resilience/streams test perfectly offline; the scenario's own direction is **invisible** offline | [page shipped](../../apps/docs/content/docs/scenarios/stale-fixture.mdx) + 1 issue draft (**`manualClock` covers 6 of 12 time-driven features; 2 bugs in the testing kit**) | +| 20 | The ID that changed on the way in | `precision-loss` | **achievable in config** — `wire.response` + `transform` = 16 lines, exact digits; the default corrupts silently | [page shipped](../../apps/docs/content/docs/scenarios/precision-loss.mdx) + 1 issue draft (**a `bigint` in `params` silently vanishes while `query` handles it**) | +| 21 | The customer data you didn't mean to log | `pii-in-the-logs` | **achievable** — an `output` allowlist filters 7 → 0 without naming a PII field; assembled = 2 seams, 42 lines | [page shipped](../../apps/docs/content/docs/scenarios/pii-in-the-logs.mdx) + 1 issue draft **HELD (security)** | +| 22 | The migration you have to run twice | `dual-run-migration` | achievable with user code — 69 lines, 5 seams; **1 of 5 isolation channels is safe by default** | [page shipped](../../apps/docs/content/docs/scenarios/dual-run-migration.mdx) + 1 issue draft (`diff`/`classifyDiff` unreachable; `pool: 'host'` re-keys the breaker) | + +## Open issue drafts + +Eighteen are filed (#640–#645, #648, #650, #651–#660). **Two are deliberately held back** as +security-sensitive — see the note below. + +| Draft | Severity | Ask | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`oauth2-params-rotation-footgun`](issue-drafts/oauth2-params-rotation-footgun.md) **→ [#657](https://github.com/rejifald/StitchAPI/issues/657)** | high | `params` can express a rotating grant that succeeds once, then revokes the account | +| [`body-verdict-footguns`](issue-drafts/body-verdict-footguns.md) **→ [#651](https://github.com/rejifald/StitchAPI/issues/651)** | high | `verdict.flag` returns `ok: true` on an error envelope; ~~`.safe()` drops `RateLimitError.body`~~ (fixed by #662); `backoff` fn silently vanishes | +| [`paginate-silent-data-loss`](issue-drafts/paginate-silent-data-loss.md) **→ [#644](https://github.com/rejifald/StitchAPI/issues/644)** | **high — a bug, not a footgun** | a zero-item page ends `paginate` with `ok: true` and the remainder unfetched; "finished" and "gave up" are the same value | +| [`clock-and-diagnostic-side-effects`](issue-drafts/clock-and-diagnostic-side-effects.md) **→ [#652](https://github.com/rejifald/StitchAPI/issues/652)** | **high ×2** | `timeout.total` is wall-clock while its sleeps use the injected clock, so a `manualClock` test of it passes vacuously; `.inspect()`/`.report()` re-issue the request and duplicated a job submit | +| [`sse-reconnect-replays-completed-streams`](issue-drafts/sse-reconnect-replays-completed-streams.md) **→ [#640](https://github.com/rejifald/StitchAPI/issues/640)** | **highest — a bug in freshly shipped #622** | `sse: { reconnect: true }` reopens a **completed** id-less stream 4× and delivers `ABCDEABCDEABCDEABCDE` to the consumer, ending `ok: true` | +| [`cache-cannot-revalidate`](issue-drafts/cache-cannot-revalidate.md) | medium (capability gap) | `cache` is a value store so an ETag can never reach it; a surface can't see the bound principal, which is what makes a hand-written ETag store leak across credentials | +| [`no-compensation-seam`](issue-drafts/no-compensation-seam.md) **→ [#656](https://github.com/rejifald/StitchAPI/issues/656)** | medium (capability gap, sharp edges) | nothing runs on failure, so a mandatory cleanup call can't be expressed — and the two natural ways to hand-write it (`.safe()` on the abort; cleanup inside `Surface.execute`) are silently wrong | +| [`void-call-drops-work`](issue-drafts/void-call-drops-work.md) **→ [#660](https://github.com/rejifald/StitchAPI/issues/660)** | **high** | `void call(input)` makes **0 HTTP calls and 0 errors** — the idiomatic fire-and-forget spelling silently drops the work; plus `backoff.base` clamped by `max` without warning | +| [`resilience-has-no-tenancy`](issue-drafts/resilience-has-no-tenancy.md) **→ [#641](https://github.com/rejifald/StitchAPI/issues/641)** | **highest production impact** | `throttle`/`circuit` have no `tenancy`, so one customer's revoked token failed **9 of 9** healthy customers and never self-healed. Fix is one option name on two interfaces, on an existing axis | +| [`any-is-priced-as-a-hedge`](issue-drafts/any-is-priced-as-a-hedge.md) **→ [#643](https://github.com/rejifald/StitchAPI/issues/643)** | **high** (a credential leak + silent spend) | a per-call `authorization` for the primary **arrived at the backup verbatim**; and `any()` is documented as failover while calling every member on every call — 20 requests for 10 answers | +| [`paginate-cannot-report-a-partial-run`](issue-drafts/paginate-cannot-report-a-partial-run.md) **→ [#645](https://github.com/rejifald/StitchAPI/issues/645)** | **high** (companion to the draft above) | deduping in `items` — the standard mitigation — emptied a page and **lost 6 rows**; four different endings share one `break` and one successful result | +| [`drift-cannot-grade-a-coercion`](issue-drafts/drift-cannot-grade-a-coercion.md) **→ [#654](https://github.com/rejifald/StitchAPI/issues/654)** | medium-high (flagship, mostly working) | a `coerced` finding is byte-identical for `"12345"→12345` and `"abc"→0`; and "nullable = warn, value intact" has no spelling — `.nullable()` makes a rollout invisible | +| [`streaming-is-not-memory-bounded`](issue-drafts/streaming-is-not-memory-bounded.md) **→ [#659](https://github.com/rejifald/StitchAPI/issues/659)** | **high — two located bugs** | `engine.ts:1443` retains every chunk so `.stream()` measures the same as `await`; and `decode: 'json'` buffers the array it streams, tripping its own cap at 37k rows | +| [`sigv4-ignores-the-injected-clock`](issue-drafts/sigv4-ignores-the-injected-clock.md) **→ [#658](https://github.com/rejifald/StitchAPI/issues/658)** | medium (+ a third clock instance) | SigV4 signs with `new Date()` so skew is untestable on a virtual clock; a skew 403 opens the dependency's breaker; `onRequest` runs after signing | +| [`idempotency-default-is-not-restart-safe`](issue-drafts/idempotency-default-is-not-restart-safe.md) **→ [#642](https://github.com/rejifald/StitchAPI/issues/642)** | **highest stakes — measured in charges** | `idempotency: true` minted a new key on a re-driven job → **8 charges for 6 intended payments**; and adding `retry`, which the warning itself advises, silences the warning | +| [`coalescing-does-not-share-failures`](issue-drafts/coalescing-does-not-share-failures.md) **→ [#653](https://github.com/rejifald/StitchAPI/issues/653)** | medium (+ a strong positive) | in-flight coalescing genuinely works (100 calls / 30 ids → 30 requests), but a coalesced FAILURE releases every joiner — 100 requests for one 404ing id; and a `store` silently un-pools `pool: 'host'` concurrency | +| [`hooks-can-rewrite-the-call`](issue-drafts/hooks-can-rewrite-the-call.md) **→ [#655](https://github.com/rejifald/StitchAPI/issues/655)** | medium-high (a docs/behaviour mismatch) | the hooks guide says hooks never change what a stitch returns; mutating `ctx.res.status` turned a vendor 200 into a thrown 503. Plus the definitive accessor→headers table | +| [`mcp-error-channel-leaks-a-query-credential`](issue-drafts/mcp-error-channel-leaks-a-query-credential.md) | medium-high (**scoped** leak + a real positive) | the MCP error channel renders `Error.message` unfiltered, so a DNS failure under `apiKey({ in: 'query' })` put the key in the model's context. **The credential boundary itself held: 30 scans, 0 hits.** Plus a rename bypasses the registry allow-list | +| [`input-schemas-check-but-never-filter`](issue-drafts/input-schemas-check-but-never-filter.md) **→ [#648](https://github.com/rejifald/StitchAPI/issues/648)** | **high** (not MCP-specific) | `validateInput` discards the parsed value while `validateOutput` returns it, so a stripping schema — the Zod/Valibot/ArkType default — does not strip. Declaring a strict schema to constrain an untrusted caller silently does nothing | +| [`testing-kit-clock-gaps-and-two-bugs`](issue-drafts/testing-kit-clock-gaps-and-two-bugs.md) **→ [#650](https://github.com/rejifald/StitchAPI/issues/650)** | **high — 2 bugs + a soundness table** | `manualClock` drives 6 of 12 time-driven features (OAuth2 expiry is NEW and undocumented); `stubStitch().safe()` throws on a sync throw; `mockAdapter` violates the library's own `abort` rule | +| [`bigint-in-params-vanishes`](issue-drafts/bigint-in-params-vanishes.md) | medium-high (silent data loss) | `expandTemplateVar` (`util.ts:392`) omits `bigint` while `stringifyLeaf` includes it, so the same id is exact in `query` and gone in `params`; plus `.report()` throws under a bigint body | +| [`diff-primitives-are-unreachable`](issue-drafts/diff-primitives-are-unreachable.md) | medium (surface gap + a coupling trap) | `diff` and `classifyDiff` both ship and neither is exported from any of the 17 subpaths; `DriftOptions.ignore` suppresses by path so it hides real regressions with benign ones; `pool: 'host'` silently re-keys the circuit onto the host | + +> **Held back from filing — security-sensitive, review before disclosing.** +> +> 0. [`adr-0018-findings-can-leak-a-value`](issue-drafts/adr-0018-findings-can-leak-a-value.md) — +> ADR 0018 §4's "findings never leak a secret" is false for hard validation, and the disk sink +> skips the deep scrubber that sits in its own file. Section 3 is ordinary and splittable. +> +> 1. [`mcp-error-channel-leaks-a-query-credential`](issue-drafts/mcp-error-channel-leaks-a-query-credential.md) — +> an unfiltered MCP error channel puts an `apiKey({ in: 'query' })` credential into a +> model's context from zero lines of user code. +> 2. [`cache-cannot-revalidate`](issue-drafts/cache-cannot-revalidate.md) §2 — a measured +> cross-principal cache leak (`bob` received `viewer: tok-alice`). §1 is a plain capability +> gap and could be filed on its own. +> +> **Triage note — two, in this order.** +> +> 1. [`resilience-has-no-tenancy`](issue-drafts/resilience-has-no-tenancy.md) — highest +> production impact. One customer's revoked token failed **9 of 9** healthy customers, and the +> outage does not self-heal. Not a bug (everything behaves as documented) but the composition +> has a 100% blast radius, and the fix is one option name on two interfaces, on an axis +> `CacheOptions`/`OAuth2Options` already carry. +> 2. [`sse-reconnect-replays-completed-streams`](issue-drafts/sse-reconnect-replays-completed-streams.md) — +> a genuine bug in code that shipped in **#622**. It delivers duplicated content to end users +> on a stream that never failed, and the run ends `ok: true`. + +### Patterns across the pass + +**0. One stale docs reference, verified and left unfixed.** +`apps/docs/content/docs/concepts/run-identity.mdx:26` and `:33` describe "each step of a +`pipe()`" and label a diagram `a pipe(): step 1`. `stitchapi/pipe` exports exactly +`all, any, linked, race` — the construct described is `linked()`. A two-line docs edit, left +out of the scenario commits to keep them scoped. + +**1. Achievable, but only off the documented path — 7 for 7** (scenario 8 is the exception that +proves the rule: it is out of scope by design, and the docs already say so). Every in-scope +scenario was solvable, +and in none of them did the built-in the docs point at carry it. `throttle` sends you to +`delegate` (status-keyed, wrong); `paginate` looks like the loop and is a trap twice over; +`retry.respect` is inert on the body path; `cache` cannot revalidate. **A custom `Surface` has +now been part of the answer in six of seven** — `interpret` in 2, 4 and 7, `execute` in 5 and +6, both in 3. Scenario 7 is the exception that sharpens the point: a surface lifted the ETag, +but nothing in the library touched the requirement the scenario existed for. +Worth deciding: is this signposting — an "if the signal is in the body, write a surface" +pointer from each guide — or are the built-ins scoped one notch too narrow? + +**2. `verdictOf` is mandatory by convention, not by construction.** Every surface written in +this pass had to remember to compose it first, and the one proof that omitted it returned a +404 as `ok: true`. A correctness requirement currently enforced by documentation. + +**2a. The buffered and streaming paths disagree about `interpret`, undocumented.** It runs for +every response including non-2xx on a buffered stitch (measured on `[200, 304, 404]`), and +**zero times** on a streaming one. Anyone reasoning from one path to the other will be wrong. + +**2b. SIX time-driven features ignore the injected clock — settled deliberately in scenario 19.** +`manualClock` drives `retry` backoff, `throttle` (rate and concurrency), `circuit.cooldown`, the +per-attempt `timeout` and `Retry-After`. It does **not** drive `timeout.total`, `cache.ttl`, the +`memoryStore` TTL beneath it, event `at`/`done.elapsed`, **OAuth2 token expiry** or SigV4. ADR +0010 §4 documents **four** as deliberate; **SigV4 and OAuth2 expiry are documented nowhere**, and +`auth.ts` has no clock plumbing at all. Measured cost: a `timeout: { total: 1000 }` call survived +**2700 virtual ms** and returned `ok: true`. Superseded note below — + +**2b (superseded). THREE time-driven features ignore the injected clock** — `timeout.total` (4), `cache.ttl` +(6) and **SigV4 signing** (14) all read wall-clock while their neighbours use `clock`. Three +point fixes are worth less than one audit plus a line in the testing guide. + +**2f. `verdict.flag`'s absent-path rule has produced a silent success FOUR times** — a THROTTLED +envelope returned as data (2), `flag: 'UnprocessedItems'` inert because arrays are truthy (11), +a `RequestTimeTooSkewed` 403 swallowed (14), and an `idempotency_key_in_use` 409 swallowed +(15). Each time the fix was ~6 lines of +`Surface.interpret`. An absent flag meaning "no signal" is defensible; it being the _quiet_ +answer four times running is the pattern. **This is now the single most-repeated finding of +the pass.** + +**2b-bis. The buffered and streaming paths disagree about six things, all silently.** Across +scenarios 5 and 13: `retry` inert, `interpret` never called, `pick` never called, `transform` +never called, `output` validates but doesn't transform, and `stream({ kind })` drops the +surface. The engine already warns about one ignored slot (an undrawable upload-progress bar +emits an `info` event) — these six get nothing. A single "this config slot does nothing on a +stream" diagnostic would cover the class. + +**2c. `paginate`'s `items.length === 0` break has now cost data in three separate scenarios** +(3, 4 and 11) — a zero-progress batch round, a drifted page mid-collection, and a deduper doing +its job. It is one line (`engine.ts:984`) and it is the single most expensive default found in +this pass. + +**2d. `.report()` / `.inspect()` are fresh probes — four sightings now** (4, 7, 11, 12). In +scenario 12 `.report()` called immediately after a drifting call reported **zero** findings, +because the probe hit a clean response, _and_ it added a tick to the rate's denominator. The +name says "tell me about that run"; the behaviour is "make another one". + +**2e. `trace` + `ctx.spanId` is the one place cross-call state is the design, not a leak.** +Scenarios 7, 9 and 11 all wanted per-call or cross-call state and found closures that leak or +no slot at all. Scenario 12 found the answer: a `TraceSink` sees every event of every call and +`spanId` collapses them per call. Worth pointing at from the guides that need it. + +**3. My pre-verification hypotheses were wrong in every single scenario.** Usually about which +primitive would carry the solution — and twice (9, 10) wrong _optimistically_. Scenario 11 was +the first wrong about a matter of **fact**: it had offset drift's causation backwards (insert +→ skip, delete → duplicate; it is the reverse), and would have shipped a page teaching it wrong. +That is the strongest argument for the executable-proof bar: +a docs-and-source audit would have shipped four wrong pages. diff --git a/docs/scenarios/agent-holds-the-tool.md b/docs/scenarios/agent-holds-the-tool.md new file mode 100644 index 00000000..69696a98 --- /dev/null +++ b/docs/scenarios/agent-holds-the-tool.md @@ -0,0 +1,156 @@ +# Scenario: the agent chooses the arguments + +**Researched:** 2026-08-05 · **Status:** ✅ verified (8 claims, 181 checks, offline) · page shipped +**Slug:** `agent-holds-the-tool` + +--- + +## The use case + +You already call a vendor API from your code. Now an LLM agent needs to call it too — over MCP, +as a tool. The model picks _which_ call and _what arguments_, from a prompt that may contain +text you did not write. + +## Why it is not straightforward + +**The model is now part of the request path**, and it is the least trustworthy part. Three +hazards, all documented in the wild: + +- **The credential must never reach it.** A survey of over **10,000 real MCP servers** found + credentials, API keys and PII leaking at rates **exceeding 10%**. The standing advice is + blunt: _"never pass a client token through to upstream APIs."_ A token in a tool schema, an + argument, a result, or an error message is a token in the model's context — and therefore in + its output, its logs, and any downstream tool it calls. +- **The arguments are attacker-influenced.** A scan of popular MCP servers found **43% with + command-injection flaws, 22% allowing path traversal, and 30% exploitable via SSRF**. The + input schema stops being ergonomics and becomes the security boundary: if the model can name a + URL, a path, or a header, so can a prompt injection. +- **The loop is the cost.** _"The most common production incident is not a model giving the + wrong answer; it is an agent that decides to retry, and retry, and retry."_ One agent scanning + a network reached a **$6,531** bill in days with no hard limits; and a single runaway agent + exhausts a shared pool, 429-ing every other agent in the fleet. + +And the framing problem underneath: an agent tool is an API you are exposing to an untrusted +caller, but it is usually written as if it were an internal function. + +## Evidence this bites real projects + +- **Credential leak rate** — [Checkmarx, MCP security risks and real incidents](https://checkmarx.com/learn/mcp-security-risks-real-world-incidents-and-security-controls/): + over 10,000 servers analysed, >10% leaking credentials/keys/PII. +- **The Equixly scan** — 43% command injection, 22% path traversal, 30% SSRF across popular MCP + servers. +- **Tool poisoning** — a WhatsApp MCP server whose _tool description_ instructed the model to + exfiltrate message history through a benign-looking call + ([Unit 42 on MCP attack vectors](https://unit42.paloaltonetworks.com/model-context-protocol-attack-vectors/)). +- **The control list** is consistent across the guidance: allow-list and validate every tool + input, never forward a client token upstream, block SSRF egress, and require human + confirmation for anything irreversible + ([CSA agentic MCP best practices](https://labs.cloudsecurityalliance.org/agentic/agentic-mcp-security-best-practices-v1/)). +- **Runaway cost** — [OpenLegion on agent rate limiting](https://www.openlegion.ai/en/learn/ai-agent-rate-limiting) + and [the $6,531 case](https://www.nexgismo.com/blog/ai-agent-budget-guards-stop-runaway-api-costs). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| **One MCP server per integration** | Hand-write a server wrapping the vendor. | Full control, and you write and secure the auth, validation and limits yourself — which is what the >10% leak rate is measuring. | +| **One tool per endpoint** | Narrow, typed tools the model picks between. | The safest shape: the schema _is_ the allow-list. Costs a tool definition per endpoint and a lot of context. | +| **One generic "run the call" tool** | The model names the call and passes arguments. | Compact and far more dangerous: the argument object becomes the attack surface, and it must be constrained by something other than the tool schema. | +| **Gateway in front** | Policy, quotas and egress rules outside the app. | The enterprise answer. Another hop, and it cannot see intent. | +| **Human confirmation on writes** | Ask before anything irreversible. | The one control that survives prompt injection. Needs a place to hook it. | +| **Token/cost budgets** | Cap spend, not just request count. | Catches the runaway loop that a request-per-minute cap never will. | + +**Summary of the state of the art:** never let the credential into the model's context, +constrain the arguments with something the model cannot widen, bound the loop by cost as well as +count, and gate irreversible actions on a human. + +--- + +## What to verify against StitchAPI + +"Agent-native" is a headline claim and the MCP surface is the least-tested thing in this pass. +[Scenario 8](webhook-receipt.md) touched `serve` and found it unauthenticated by design; nothing +has yet tested MCP. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **The surface exposes three generic tools**, not one per endpoint: `run_stitch`, + `list_stitches`, `describe_stitch` (`mcp.ts:46,73,85`). `run_stitch` takes + `{ name, input }`, where `input` is described as _"The stitch input object"_ — a free-form + object. +- **That makes `input` the security boundary**, and [scenario 10](provider-failover.md) measured + something directly relevant: `const headers = { ...(cfg.headers ?? {}), ...(input.headers ?? +{}) }` (`engine.ts:232`) — **input headers merge _over_ config headers**. If the model's + argument object reaches that merge unfiltered, a prompt injection can set headers on a call it + did not author. +- **The capability boundary is the counter-claim.** The docs are emphatic that the caller — _"an + agent included"_ — receives data and never the credential. This scenario is the sharpest test + of that sentence there is. +- Scenario 9 measured breakers and throttles are shared unless keyed by hand, which is the + runaway-containment question in a different suit. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Does the credential reach the model, anywhere? Check the tool + schema, `list_stitches`, `describe_stitch`, a successful result, an error, and the trace. + Include a stitch with `auth: bearer(env(...))` and one with a `Cookie` jar. +2. **C2** — **DECIDING CLAIM.** Can the model's `input` redirect or rewrite the call? Try + `headers`, `query`, `params`, and anything URL-shaped. If a model-supplied `authorization` or + host reaches the wire, that is SSRF/header-injection through the tool boundary. +3. **C3** — is there an **allow-list**? Can the model run _any_ registered stitch, or only ones + opted in? What does `list_stitches` disclose — names only, or config? +4. **C4** — **error rendering**: does a failure leak the internal URL, header names, or the + response body to the model? +5. **C5** — **runaway containment**: do `throttle`/`circuit` apply on the MCP path, and can a + budget be expressed in anything other than request count? +6. **C6** — is there a **confirmation seam** for an irreversible call, or is every registered + stitch equally callable? +7. **C7** — **schema quality**: is `input` typed enough for a model to use correctly, and does a + declared input schema constrain what the model may send? +8. **C8** — assemble the safest available exposure and report the seam and line count. + +C1 and C2 decide this. The capability boundary is the product's central promise, and a generic +`run_stitch` tool is exactly the shape that tests whether the promise holds when the caller is +adversarial. + +--- + +## Verification result + +**All 8 claims verified**, 181 checks across 8 runnable scripts under +`proofs/agent-holds-the-tool/`, re-run by me before writing anything up. + +| Claim | Verdict | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — does the credential reach the model? | **HELD.** 34 exchanges, 30 payload scans, 14,529 bytes, 5 held secrets, **zero hits** — including a vendor 401 with a credential-shaped body, and stdio. Controls confirm the wire carried them. | +| C2 — can `input` rewrite the call? | **Header hypothesis REFUTED**; 5 other levers real | +| C3 — allow-list? | The registry object, and it is usable — but `--module` sweeps everything, and a rename bypasses it | +| C4 — error rendering | StitchAPI's own errors are terse; **the channel is unfiltered** — one real leak | +| C5 — runaway containment | `throttle`/`circuit` apply; 1 tool call ≠ 1 request; no cost budget | +| C6 — confirmation seam | None, either direction. User code can refuse, never ask | +| C7 — schema quality | Validates but **does not filter** | +| C8 — assembled | 47 lines, 3 seams | + +### Hypotheses that were wrong + +**The central one.** I predicted `engine.ts:232`'s input-headers-win merge would let a model +forge `authorization` or `host`. It cannot: `sanitizeAgentInput` (`mcp.ts:125-130`) deletes +`input.headers` before the merge unless the stitch declares an `input.headers` schema. Six +model-supplied headers → zero on the wire. The code has an explicit comment saying this is +deliberate defence-in-depth, and it is correct. + +That is the second time this pass that reading the source produced a confident, wrong prediction +that running code corrected — see [unstable-pagination](unstable-pagination.md), where I had +offset-drift causation backwards. + +**What I under-weighted.** I framed this as a credential question. The credential boundary was +the strongest thing measured; the _argument_ boundary is where everything real lives, and the +sharpest finding — a query parameter pinned in a configured path being merely a default — was +not on my list of claims at all. + +### Outputs + +- Page: [agent-holds-the-tool.mdx](../../apps/docs/content/docs/scenarios/agent-holds-the-tool.mdx) +- Drafts: [mcp-error-channel-leaks-a-query-credential](issue-drafts/mcp-error-channel-leaks-a-query-credential.md), + [input-schemas-check-but-never-filter](issue-drafts/input-schemas-check-but-never-filter.md) diff --git a/docs/scenarios/async-job-polling.md b/docs/scenarios/async-job-polling.md new file mode 100644 index 00000000..ce9ea5f9 --- /dev/null +++ b/docs/scenarios/async-job-polling.md @@ -0,0 +1,159 @@ +# Scenario: submit, poll, download — the async job triangle + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `async-job-polling` + +**Verification:** 9 proof scripts, run offline on an injected clock, in +[`proofs/async-job-polling/`](proofs/async-job-polling/). Published page: +[`scenarios/async-job-polling.mdx`](../../apps/docs/content/docs/scenarios/async-job-polling.mdx). +Escalated to a draft: [`issue-drafts/clock-and-diagnostic-side-effects.md`](issue-drafts/clock-and-diagnostic-side-effects.md). + +| Claim | Verdict | Measured | +| ----------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — `Location` header → next URL | reachable | `interpret`/`onResponse` get the full response; hook rewrite gave `POST /jobs → 3× GET /jobs/job-1`, 1 submit. Nothing built-in follows it | +| C2 — poll loop as a `Surface` | PASS | 5 polls at exactly 30 s virtual spacing; `Failed` stopped on the first terminal body, 17/20 attempts unspent | +| C3 — `Retry-After` on the body path | **nothing honors it** | server asked 30 s, measured gaps **7 ms**; surface-read works; capped expo fallback 1000/2000/4000/5000/5000 | +| C4 — `paginate` | refuted, worse than predicted | it _does_ loop (default `items` wraps a non-array as one item) — but gaps `0,0,0`, and it cannot fail | +| C5 — one deadline over the triangle | PASS, two routes | one-stitch + `timeout.total` → 253 ms; three stitches + one `AbortSignal` → 6 polls/virtual hour | +| C6 — `linked` trace chain | PASS | 3 starts, **1 traceId**, 3 spans, each parented to the last; without `run`, 3 traceIds and 0 parents | +| C7 — single-use download | default is safe | `retry.on` excludes 404 → `200,404` even at `attempts: 5`; per-stitch split gave 20 poll / 1 download | +| C8 — resumability | entirely user-side | engine writes **0** store keys on submit; hand-rolled resume works, 1 submit total | +| C9 — assembled | PASS | 1 submit → 5 polls at server pacing → 1 download; **110 lines vs 49** hand-rolled, byte-identical wire behavior | + +**Wrong hypotheses, fourth time running.** The capture predicted `paginate` would break +immediately at `items.length === 0` because a job-status body has no items array. False — the +default `items` wraps a non-array body as `[value]`, so it loops cleanly. Its real +disqualification is that it cannot **wait**, and that a paginated poll cannot fail (`Failed` +is aggregated as just another value). + +**The finding worth carrying forward** is a genuine design tension, not a gap: **you can have +one deadline over the triangle, or per-hop retry policies, but not both.** Collapsing the +triangle into one stitch buys `timeout.total` and loses the per-hop retry split (the poll's +patience becomes the single-use download's — measured: 8 shared attempts burned on a dead +link) plus concurrency safety. Three stitches under `linked` keep both and replace the +config-level deadline with a caller-owned `AbortSignal`. This is the first scenario in the +pass where the honest answer is "pick which guarantee you want". + +--- + +## The use case + +You ask an API to do something slow: a Salesforce Bulk API 2.0 export, a Shopify bulk +operation, a report render, a video transcode, a data extract. The API cannot answer in one +request, so it answers **`202 Accepted`** with a job id and a `Location`, and you come back +later. + +This is the [asynchronous request–reply pattern](https://learn.microsoft.com/en-us/azure/architecture/patterns/asynchronous-request-reply), +and it is one of the three or four shapes every integration eventually meets. + +## Why it is not straightforward + +It is not one call. It is **three different endpoints and a loop between them**: + +1. `POST /jobs` → `202` + `Location: /jobs/{id}` + often `Retry-After` +2. `GET /jobs/{id}` → repeatedly, until `state` reaches a terminal value +3. `GET ` → the payload, frequently a pre-signed URL on a different host + +Each step contributes its own difficulty: + +- **The next URL comes from the previous response's _header_.** `Location` is not in the + body. A client that only threads body fields through cannot express step 1 → step 2. +- **Terminal state is in-band, and so is failure.** Salesforce transitions `Open → +UploadComplete → InProgress → JobComplete | Failed | Aborted`, all at HTTP 200. `Failed` + is a successful HTTP response carrying bad news — the same shape that defeats status-code + logic in every other scenario in this section. +- **The wait is long and the pacing is the server's.** Jobs run for minutes to hours. Each + Salesforce batch can take up to 10 minutes. Correct clients poll on `Retry-After` when the + server sends one, and back off exponentially when it doesn't. +- **The total budget spans the whole triangle, not one call.** "Give up after an hour" is a + deadline over submit + N polls + download. A per-call timeout cannot express it, and a + per-attempt one certainly cannot. +- **The result URL expires, and sometimes is single-use.** Pre-signed links have a TTL, and + in some systems [expire on first successful fetch, 404-ing afterwards](https://community.developers.refinitiv.com/discussion/comment/16277). + A retry of the _download_ can therefore fail permanently in a way that looks transient. +- **Restart loses the job.** If your process dies mid-poll, the job is still running server-side. + Re-submitting duplicates hours of work; the correct move is to reattach to the stored id. + Best-practice writeups say to persist the id — almost no client helps you do it. + +## Evidence this bites real projects + +- **jsforce** — [`#298`](https://github.com/jsforce/jsforce/issues/298): "How to set + pollTimeout for Bulk job using alternative api method?" The polling timeout for + `waitForResults=true` is **hardcoded**; the documented workaround is to set + `waitForResults=false` and write your own polling loop. +- **go-salesforce** — [`#139`](https://github.com/k-capehart/go-salesforce/issues/139): + "Feature Request: Make bulk job polling timeout configurable" — larger datasets fail with + `context deadline exceeded`. +- **salesforcer** — [`#13`](https://github.com/StevenMMortimer/salesforcer/issues/13): + "Error with Bulk Query After Timeout". +- **Salesforce** — [How bulk queries are processed](https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/asynch_api_bulk_query_processing.htm) + and [troubleshooting query timeouts](https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/bulk_api_2_0_troubleshoot_query_timeouts.htm). +- **Azure Architecture Center** — [Asynchronous Request-Reply](https://learn.microsoft.com/en-us/azure/architecture/patterns/asynchronous-request-reply), + the canonical description of the pattern. + +Note what the three library issues have in common: the poll loop exists inside the SDK, its +timeout is not configurable, and the escape hatch is _"turn the helper off and write the loop +yourself."_ That is this scenario's signature — the helper is either too rigid or absent. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **SDK's built-in waiter** (`waitForResults=true`) | The vendor polls for you. | Hardcoded timeout (jsforce #298, go-salesforce #139). Fine until your job is big, then unfixable without abandoning the helper. | +| **Hand-rolled `while` + `sleep`** | Poll, check state, sleep, repeat. | Correct and universal. Outside the HTTP client, so timeout, circuit breaking, and tracing see three unrelated calls rather than one operation. | +| **Fixed-interval polling** | `setInterval` every 5 s. | Hammers the API for hour-long jobs and ignores `Retry-After`. The bill and the rate limit both notice. | +| **Exponential backoff polling** | Double the gap up to a cap. | The right default when there's no `Retry-After` — but it must still be capped, or an hour-long job's last gap overshoots the finish by minutes. | +| **Webhook instead of polling** | Ask the API to call you back. | Strictly better where offered. Needs a public endpoint, a receiver, and a fallback poll anyway for missed deliveries — so it is _additional_ work, not a replacement. | +| **Queue + separate worker** | Submit, persist the id, poll from a scheduled worker. | The production answer for hour-long jobs, and the only one that survives a restart. Costs infrastructure. | + +**Summary of the state of the art:** persist the job id, poll on the server's own pacing with +a capped backoff, treat in-band `Failed` as a failure, bound the whole triangle with one +deadline, and don't retry a single-use download. Every one of those is a line of code; the +difficulty is that no client models the _operation_, only the three calls. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **There is no poll/until primitive.** Nothing in the config vocabulary waits for a state. +- **The likely seam is scenario 2's:** a custom `Surface` whose `interpret` reads `state` and + returns `{ ok: false, retry: true, after }` while the job is running, with `retry.attempts` + as the poll bound. If so, polling is "retry, but the failure is 'not done yet'". Whether + `after` can come from a `Retry-After` **header** (not just a computed number) is open. +- **`paginate` should fail immediately here**, and for a new reason: scenario 3 measured the + loop breaking at `items.length === 0` (`engine.ts:984`), and a job-status response has no + items array at all. Worth confirming — it is the second scenario where `paginate` looks + loop-shaped and isn't. +- **`linked` (`pipe.ts:357`) chains runs into one trace** — "a sequence of awaits draws one + trace chain", with ancestors as plain typed variables. That covers _tracing_ the triangle. + It says nothing about a shared deadline, so the "give up after an hour" budget is probably + unexpressible. +- **`Location` → next request URL** is the untested mechanic. Can a response header become the + next call's path/baseUrl through any documented seam? + +**Claims to test with runnable offline code:** + +1. **C1** — can a `Location` header from the `202` become the next call's URL, without the user + parsing it by hand outside the library? +2. **C2** — can a custom `Surface` express the poll loop (in-band `InProgress` → wait → poll + again; `JobComplete` → done; `Failed` → a real failure)? Measure poll count and gaps. +3. **C3** — can the poll wait come from the response's **`Retry-After` header**, and fall back + to a capped exponential when absent? (`retry.respect` honors it for status-driven retries — + does anything honor it for a body-driven one?) +4. **C4** — confirm `paginate` cannot express this, and say exactly how it fails. +5. **C5** — is there ONE deadline over submit + polls + download? Try `timeout.total`, + `linked`, a seam. Measure what each actually bounds. +6. **C6** — does `linked` really produce one trace chain across the three endpoints? Measure + the events, and whether a failure in the middle is attributable to the operation. +7. **C7** — the download step: can a **single-use / expiring** result URL be fetched without + `retry` turning a permanent 404 into three? +8. **C8** — resumability: can a stitch reattach to a stored job id and resume polling without + re-submitting? Or is that entirely user-side? +9. **C9** — assemble the best answer available, run it, report the seam and line count, and + compare honestly against the hand-rolled `while` loop. + +C5 and C8 are the ones most likely to come out negative. A poll loop that cannot express +"give up after an hour" is not a solution to an hour-long job. diff --git a/docs/scenarios/batch-partial-failure.md b/docs/scenarios/batch-partial-failure.md new file mode 100644 index 00000000..be8c74b5 --- /dev/null +++ b/docs/scenarios/batch-partial-failure.md @@ -0,0 +1,142 @@ +# Scenario: batch endpoints that fail one item at a time + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `batch-partial-failure` + +**Verification:** 7 proof scripts, run offline (183 checks), in +[`proofs/batch-partial-failure/`](proofs/batch-partial-failure/). Published page: +[`scenarios/batch-partial-failure.mdx`](../../apps/docs/content/docs/scenarios/batch-partial-failure.mdx). +Escalated to a draft: [`issue-drafts/paginate-silent-data-loss.md`](issue-drafts/paginate-silent-data-loss.md). + +| Claim | Verdict | Measured | +| ------------------------------------------ | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| C1 — built-in `retry` on a partial failure | cannot | 1 request, resolves `ok: true` with rows gone; forced with `on: 200`, **10 duplicate writes on a healthy batch** | +| C2 — `paginate.next` as the residue loop | works, then loses data | 3 requests / 6 items / **0 duplicates** — but a zero-progress round ended the run with **4 of 6 rows never written, `ok: true`** | +| C3 — backoff between rounds | absent | six rounds at t=0; `throttle` gives one fixed ratio; every growing curve was user code the engine never reported | +| C4 — can a `Surface` rewrite the request | no — but a hook can | `SurfaceOutcome.retry` resent the identical body (4 duplicate writes); assigning `ctx.req.body` in `onRequest` gave 0 | +| C5 — retryable vs terminal per-item | expressible | 400 doc sent **once** (vs 5 naively), never written; but reaches the caller only via a closure | +| C6 — is the residue reachable | **no** | absent from result, error, `.inspect()`, `.report()`, events, trace; the `next` ledger reported `cdef` for a true residue of `def` | +| C7 — assembled from the public API | works | 4 rounds at t=0/1000/3000/7000, 0 duplicates, residue returned as data — **50 lines vs 28 hand-rolled** | + +**The framing below was wrong in both directions.** It nominated `paginate` as the promising +candidate and concluded no seam could rewrite a request between attempts. Both are false: + +- `paginate` expresses the loop and then **fails both deciding claims** (C3, C6) and silently + drops data on a zero-progress round. As this scenario's answer it is a trap, not a solution — + which is why it produced the strongest issue draft of the pass so far. +- **`hooks.onRequest` can rewrite the request**, stays inside the resilience chain, and is what + makes the scenario achievable. Paired with `SurfaceOutcome.after` the backoff is engine-owned, + budget-aware and observable. + +Also worth recording: the assembled answer is **larger** than the hand-rolled `while` loop it +replaces. The honest pitch is not brevity — it is that `timeout.total`, the circuit breaker, +`attempts`, and retry events keep working, all measured. + +--- + +## The use case + +You write records in bulk — DynamoDB `BatchWriteItem`, Elasticsearch `_bulk`, SQS +`SendMessageBatch`, Salesforce sObject Collections, Google Sheets `batchUpdate`. One HTTP +request carries 25, 100, or 1,000 items. + +The API answers **HTTP 200**, and inside the body reports that _some_ of them didn't land. + +## Why it is not straightforward + +**The retry unit is smaller than the request.** Every HTTP client retries by replaying the +identical request. Here that is actively wrong: re-sending all 100 items to fix the 7 that +failed re-applies 93 writes that already succeeded, and on a non-idempotent endpoint that is +duplicate data, not just waste. The correct behaviour is to **rewrite the request body to the +failed subset and send that**, repeatedly, until the subset is empty. + +No mainstream HTTP client models a retry that changes the request. + +Four supporting difficulties: + +- **The failure is invisible to status-code logic.** DynamoDB returns 200 with + `UnprocessedItems`; Elasticsearch returns 200 with `errors: true` and a per-item `status`. + A client checking `res.ok` sees success. +- **Backoff is mandatory, not optional.** AWS is explicit: retrying unprocessed items + immediately will simply throttle again, because the cause is capacity. The retry loop + _must_ wait, and wait longer each round. +- **Failures are not uniform.** In one Elasticsearch response, item 3 may be a `429` + (retry it) and item 7 a `400` mapping error (retrying it forever is a hang). A correct loop + partitions per-item failures into retryable and terminal. +- **Termination needs a real bound.** If the subset never empties, the loop must stop and + surface _which_ items never landed — the caller needs the residue, not just an error. + +## Evidence this bites real projects + +- **Logstash** — [`elastic/logstash#1631`](https://github.com/elastic/logstash/issues/1631): + "rejected docs in bulk indexing partial failure are **silently lost**". The strongest + statement of the failure mode: a 200 with per-item rejections, and the data is simply gone. +- **elasticsearch-py** — [`#1004`](https://github.com/elastic/elasticsearch-py/issues/1004): + `streaming_bulk` retries only `429`, and with `raise_on_error=False` errors are aggregated + **without their data**, so you cannot tell which items to resend. +- **GitLab** — [`gitlab#12372`](https://gitlab.com/gitlab-org/gitlab/-/issues/12372), + "intelligently retry bulk-insert failures when indexing", and + [`gitlab#351600`](https://gitlab.com/gitlab-org/gitlab/-/issues/351600). +- **AWS** — [Error handling with DynamoDB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html) + and [BatchWriteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html): + retry `UnprocessedItems`, and _"if you retry the batch operation immediately, the underlying + write requests can still fail due to throttling"_. Helpfully, `UnprocessedItems` is shaped + exactly like `RequestItems`, so the resend needs no transformation. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| -------------------------------------------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **Client's built-in retry** | `retry: { attempts: 3 }` on the batch call. | Replays **all** items. Fixes the 7 by re-writing the 93. Wrong unit, and duplicate side effects. | +| **Hand-rolled `while` loop** | Read the residue, rebuild the request, sleep, repeat. | Correct, and what most teams end up with. Lives outside the HTTP client, so timeout, circuit breaking, and tracing no longer see the real call. | +| **SDK helper** (`streaming_bulk`, AWS SDK batch writers) | The vendor does it for you. | Only where a vendor SDK exists, and the policy is theirs: elasticsearch-py retries `429` only, and drops the failed items' data. | +| **Ignore partial failure** | Check the HTTP status, move on. | The Logstash bug. Silent data loss, discovered later by absence. | +| **Split into single-item calls** | One request per record. | Correct and trivially retryable — at 100× the requests, which is what the batch endpoint existed to avoid. | + +**Summary of the state of the art:** a loop that (1) reads the failed subset from a 200 body, +(2) rebuilds the request from it, (3) waits with growing backoff, (4) separates retryable from +terminal failures, and (5) reports the residue. Every team writes this by hand, and most get +(4) or (5) wrong. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- The interesting candidate is **`paginate`**, not `retry`. `PaginateOptions.next(prevBody, +pagesFetched)` returns _"the input (merged over the original) for the next page, or + `undefined` to stop"_ (`types.ts:1413-1417`). That is structurally exactly the required loop: + read the residue off the previous body, return `{ body: { RequestItems: unprocessed } }`, + return `undefined` when empty. `items` (`:1419`) aggregates the successes; `pages` + (`:1421`, default 50) is the termination bound. +- **The suspected miss: there is no delay.** `PaginateOptions` is three fields — `next`, + `items`, `pages`. No per-iteration wait, no backoff curve. AWS says backoff is mandatory + here, so a loop that fires the next attempt immediately is the documented way to fail. + Whether `throttle` can stand in (fixed spacing, and it paces unrelated traffic too) is the + question that decides this scenario. +- Scenario 2's answer does **not** transfer: `SurfaceOutcome.retry` re-sends the _same_ + request. Confirm whether a surface can rewrite the outgoing body between attempts at all. +- Naming: nothing about "pagination" suggests "retry the failed subset of a write". Even if + it works, this is a signposting gap of the same kind found in scenario 2. + +**Claims to test with runnable offline code:** + +1. **C1** — does built-in `retry` resend all items? Measure the duplicate writes it causes. +2. **C2** — can `paginate.next` express "resend only the residue" for a DynamoDB-shaped + `UnprocessedItems` response, terminating when empty, aggregating successes via `items`? +3. **C3** — can any **backoff** be introduced between those iterations? Try `throttle`, + and anything else in the working tree. Measure the actual gaps. If the only answer is a + fixed spacing, say what that costs versus exponential. +4. **C4** — can a **surface** rewrite the request body between attempts (the scenario-2 seam)? +5. **C5** — can retryable (`429`) and terminal (`400`) per-item failures be separated, with + the terminal ones surfaced to the caller rather than retried forever? +6. **C6** — when the loop gives up, can the caller get **the residue** — the items that never + landed — or only an error? +7. **C7** — the assembled best answer from the public API: write it, run it, report the seam + and the line count, and say plainly whether it is better or worse than the hand-rolled + `while` loop it replaces. + +C3 and C6 are the ones that decide page vs. issue. A loop that cannot back off is not a +solution to this scenario, however elegant the rest is. diff --git a/docs/scenarios/conditional-requests-304.md b/docs/scenarios/conditional-requests-304.md new file mode 100644 index 00000000..76602674 --- /dev/null +++ b/docs/scenarios/conditional-requests-304.md @@ -0,0 +1,155 @@ +# Scenario: the free poll — ETag revalidation and the bodyless 304 + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `conditional-requests-304` + +**Verification:** 9 proof scripts, run offline (170 checks), in +[`proofs/conditional-requests-304/`](proofs/conditional-requests-304/). Published page: +[`scenarios/conditional-requests-304.mdx`](../../apps/docs/content/docs/scenarios/conditional-requests-304.mdx). +Escalated: [`issue-drafts/cache-cannot-revalidate.md`](issue-drafts/cache-cannot-revalidate.md). + +| Claim | Verdict | Measured | +| ------------------------- | --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| C1 — bare stitch on a 304 | silent success carrying nothing | `ok: true`, `data: undefined`, `error: null`; `verdict.accept: [304]` is a no-op | +| C2 — replay the validator | works, seams differ | wire `[(none), "v1.t1", "v1.t1"]` → `[200,304,304]`, 1 billed of 3 | +| C3 — 304 → cached body | **PASS, and `interpret` DOES run on non-2xx** | counter recorded `interpret` on `[200, 304, 404]`; three seams give `[1,1,2,2]` at 2 billed of 4 | +| C4 — `output` schema | breaks a bare poll; correct with substitution | bare 304 → `ok: false`, `contract violation (drift)` | +| C5 — built-in `cache` | **cannot revalidate** | value store, not response store; hit spine has no `request` phase; a stored `undefined` is a permanent miss | +| C6 — per-credential | split | `tenancy: 'principal'` protects the built-in cache; a user store keyed `METHOD URL` leaked **bob ← alice's data** | +| C7 — weak validators | byte-exact | `W/"v1.t1"` survives both directions; strong-for-weak → 304, the server's comparison to make | +| C8 — the payoff | 8/10 polls free, zero staleness | TTL bills 1/10 but **never saw the change** (5 of 10 polls stale) | +| C9 — assembled | PASS | **87 lines, ONE seam** (`Surface.execute`) vs **79** feature-matched hand-rolled | + +**The capture's predicted NOT ACHIEVABLE did not materialise, and the reason matters.** It +worried that a 304 might be rejected before `interpret` runs, citing scenario 5's "`interpret` +is dead code". Measured with a counter: `interpret` runs for every response including non-2xx — +that finding is specific to `runStreaming`. Worth carrying: the two paths differ, and the +asymmetry is undocumented. + +**Two capture corrections.** `CacheOptions` has a tenth field the list omitted (`keyOf`, +`types.ts:1202`), which cannot defeat tenancy — `deriveCacheKey` still folds the principal in. +And `cache.ttl` does **not** honour an injected `clock`; every proof needing an expiring cache +had to inject a clock-backed store. + +**First scenario where the honest answer is "this is a wash on size."** 87 lines against 79, +identical behaviour on all four shapes. The 8-line difference attributes exactly — two helpers +that exist only because the engine hands a surface a shared, never-case-folded header record. + +--- + +## The use case + +You poll an API for changes — a GitHub repo's issues, a feed, a config document. Most of the +time nothing has changed, and you'd like not to pay for finding that out. + +HTTP has an answer. Keep the `ETag` from the last response, send it back as `If-None-Match`, +and the server replies **`304 Not Modified`** with no body. On GitHub, a 304 **does not count +against your primary rate limit** at all: 600 polls where 90% are unchanged cost 60 requests. + +## Why it is not straightforward + +**A 304 is a status that means "use what you have".** It carries no body, and it is not a 2xx. +Both halves are awkward for a client: + +- Treat it as a failure and every unchanged poll is an error. +- Treat it as a success and the caller receives `undefined` where the resource should be. +- The only correct behaviour is to **substitute the previously cached body** — which means the + cache and the request path have to know about each other. + +Everything else follows from that: + +- **The ETag and the body must be stored together**, and the ETag replayed as a request header + on the next call. A TTL cache alone cannot do this: TTL answers "is my copy young enough", + revalidation answers "is my copy still correct", and only the second one is free. +- **Validation runs on nothing.** A response schema applied to a 304's empty body fails, so any + output contract has to be bypassed or fed the cached value instead. +- **ETags are per-credential.** GitHub caches them per token; rotate the token and every stored + ETag is void. A cache keyed only by URL will replay another principal's validator. +- **ETags are per-page, not per-collection.** A 304 on page 1 of 5 says nothing about pages 2–5. + Callers routinely assume "nothing changed" from the first page's 304. +- **Weak validators compare weakly.** `W/"abc"` and `"abc"` are not interchangeable, and + `If-None-Match` is specified to use weak comparison. +- **Some servers never match.** Apache's default ETag embeds the file inode, so behind a load + balancer two servers produce different ETags for identical content and revalidation _never_ + succeeds — every poll is a full 200 and the feature silently does nothing. +- **GraphQL has no ETags at all.** GitHub's GraphQL API can't do this, so the cheap-polling + strategy is REST-only and you cache by query+variables hash yourself. + +## Evidence this bites real projects + +- **GitHub** — [best practices for the REST API](https://docs.github.com/rest/guides/best-practices-for-using-the-rest-api): + a conditional request returning 304 does not count against the primary rate limit when + correctly authorized. +- **github-buttons** — [`#33`](https://github.com/buttons/github-buttons/issues/33): using + `If-None-Match` specifically to stop exceeding the rate limit. +- **GitHub community** — [discussion #156480](https://github.com/orgs/community/discussions/156480) + on frequent polling, and [#189255](https://github.com/orgs/community/discussions/189255). +- **Practitioner guidance** is consistent on the two traps: a 304 has **no body** and treating + it as an ordinary success is a common cache bug; and inode-derived ETags behind a load + balancer make revalidation fail invisibly. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **TTL cache only** | Cache for N seconds, re-fetch after. | Never revalidates, so every refresh is a full request that counts. Fresh data is also up to N seconds stale — you pay full price _and_ get staleness. | +| **Hand-rolled ETag store** | Keep `{etag, body}`, set `If-None-Match`, swap the body in on 304. | Correct, and what most teams write. Easy to get the per-token keying wrong, and the substitution has to happen below whatever parses/validates the response. | +| **An HTTP caching proxy** | Let a proxy or `http-cache-semantics` handle it. | The most standards-correct answer. Adds a dependency or a hop, and in-process clients often can't use one. | +| **Treat 304 as an error and retry** | It's not 2xx, so it's a failure. | Actively wrong: turns the _success_ case into an error, and a retry re-sends the same validator for the same 304. | +| **Ignore conditional requests** | Just poll. | What most integrations do. On GitHub it costs 10× the rate-limit budget for identical data. | +| **Poll a cheap sentinel** | Check `updated_at` on a small endpoint first. | Works where such an endpoint exists; it is a per-API workaround, not a mechanism. | + +**Summary of the state of the art:** store the validator with the body, replay it, and swap +the cached body in on a 304 — keyed by _credential_, not just URL. The pieces are individually +trivial; the difficulty is that the swap has to happen inside the response path, below parsing +and validation, which is exactly where most client libraries have no seam. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **`cache` is TTL-only.** `CacheOptions` is `{ ttl, tenancy, vary, methods, entries, coalesce, +version, transformVersion, trustTransform }` (`types.ts:1136-1191`) — no `etag`, no + `revalidate`, no `staleWhileRevalidate`. And `If-None-Match` / `304` appear **nowhere** in + `packages/core/src`; the only `ETag` mention is a comment in `fingerprint.ts:53` about + strong/weak _schema_ fingerprints, which is unrelated. So conditional requests look entirely + absent as a feature. +- **`cache.tenancy: 'principal'`** (`types.ts:1147`) is a real match for the per-token ETag + constraint, if revalidation existed to use it. +- **The likely seam is the one that carried scenarios 2 and 4:** a custom `Surface.interpret` + returning `{ ok: true, data: cachedBody }` on a 304, plus `hooks.onRequest` setting + `If-None-Match` (scenario 3 proved a hook can mutate the outgoing request). The open question + is **ordering**: is a 304 rejected by `classifyStatus`/`verdict` _before_ `interpret` runs? + Scenario 5 found `interpret` is dead code on streaming surfaces, so "does this hook actually + run here" is a question worth asking directly rather than assuming. +- **Does a cache hit even reach the network path?** Scenario 4 measured that a cache hit never + fires `onResponse`. If a hit short-circuits, then "revalidate instead of serving stale" may + not be expressible through `cache` at all, and the ETag store has to live outside it. + +**Claims to test with runnable offline code:** + +1. **C1** — what does a bare stitch do with a `304`? Success with an empty body, or a failure? +2. **C2** — can `hooks.onRequest` set `If-None-Match` from a stored ETag, and is the ETag + readable off the previous response? Measure the header actually sent. +3. **C3** — **DECIDING CLAIM.** Can a 304 be turned into "return the cached body" — so the + caller receives the resource, not `undefined`? Try `Surface.interpret`, `verdict.accept`, + `transform`, hooks. Establish whether `interpret` runs at all for a non-2xx status. +4. **C4** — does an `output` schema reject the 304 (empty body), and can the substituted body + be validated instead? +5. **C5** — can the built-in `cache` participate — storing the ETag alongside the body — or must + the ETag store be separate? Does a cache hit short-circuit before any revalidation could run? +6. **C6** — per-credential keying: does `cache.tenancy: 'principal'` (or `vary`) keep one + principal's ETag from being replayed for another? Measure a cross-principal case. +7. **C7** — weak validators: is `W/"abc"` preserved byte-exact on the way out? (A client that + normalizes or strips `W/` breaks revalidation against a compliant server.) +8. **C8** — the rate-limit payoff: measure requests-that-count over a 10-poll run with 9 + unchanged, versus the same run with plain TTL caching. +9. **C9** — assemble the best available answer, run it, report seam and line count, and compare + honestly with the hand-rolled version. + +C3 decides this one. If a 304 cannot be turned back into the cached body inside the response +path, then the feature is not merely absent — it is unreachable, and this becomes the pass's +first genuine NOT ACHIEVABLE. diff --git a/docs/scenarios/cost-based-rate-limits.md b/docs/scenarios/cost-based-rate-limits.md new file mode 100644 index 00000000..40d0ecd0 --- /dev/null +++ b/docs/scenarios/cost-based-rate-limits.md @@ -0,0 +1,138 @@ +# Scenario: cost-based rate limits reported in the response body + +**Researched:** 2026-08-04 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `cost-based-rate-limits` + +**Verification:** 6 proof scripts, run offline (125 checks), in +[`proofs/cost-based-rate-limits/`](proofs/cost-based-rate-limits/). Published page: +[`scenarios/cost-based-rate-limits.mdx`](../../apps/docs/content/docs/scenarios/cost-based-rate-limits.mdx). +Footguns escalated to a draft: +[`issue-drafts/body-verdict-footguns.md`](issue-drafts/body-verdict-footguns.md). + +| Claim | Verdict | Measured | +| ------------------------------------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | +| C1 — `retry` on a 200-with-THROTTLED | FAIL | predicate receives 1 arg (`200`); `on: 200` retried successes — 3 requests, 900 points for one call | +| C2 — wait computed from the body | FAIL for `backoff`, PASS via surface | `backoff` fn is a type error, invoked **0** times if cast past; `SurfaceOutcome.after` honored at exactly **6000 ms** | +| C3 — `extensions.cost` reachable | PASS via one seam | `hooks.onResponse` only; `StitchError.body` undefined, `.inspect().raw` null on throttle | +| C4 — `throttle.delegate` on a 200 | FAIL | status-keyed (default `[429]`); `on: 200` fires on successes too | +| C5 — `throttle.rate` as a cost budget | FAIL | points token throws at construction; 10 absorbable calls took **18,000 ms** | +| C6 — assembled from the public API | PASS | custom `Surface`, **73 lines**, 8/8 succeeded against a neighbour draining the bucket | + +**The framing below was wrong.** The pre-verification hypotheses correctly predicted C1–C5, +then concluded the built-ins failing might total to _not achievable_. They missed the seam that +actually solves it: a custom **`Surface`** whose `interpret` sees every body and whose +`SurfaceOutcome.after` carries a computed wait into the engine's own retry loop +(`surface.ts:34-37`, honored at `engine.ts:785-805`; ADR 0022 Decision 5). It is public and +exported. The real gap is not capability — it is that **nothing points there**: `throttle`'s +own doc comment sends readers to `delegate`, which is status-keyed and cannot reach a +body-reported quota. + +Also refuted: `.inspect().raw` is not a route to the payload — post-`pick` on success, and +`null` on the throttled response despite `source: 'live'`. + +--- + +## The use case + +An app talks to the **Shopify GraphQL Admin API** — the canonical cost-based limiter, and +the one most teams meet first. The same shape appears in GitHub's GraphQL API (point cost), +Atlassian (cost budgets), and Salesforce (per-query governor limits). + +The work is ordinary: sync products, backfill orders, respond to a webhook. What is not +ordinary is how the quota is accounted for. + +## Why it is not straightforward + +Shopify's limiter is a **leaky bucket denominated in query cost**, not requests: a 1,000-point +bucket refilling at 50 points/second. Four properties each break a different standard tool. + +**1. The price is per-query and variable.** One request might cost 11 points, another 900. +"Requests per second" is not a meaningful unit here, so a fixed-rate limiter is either +wasteful or wrong — there is no single spacing that is correct for both queries. + +**2. Over-spending answers `200 OK`.** Shopify returns HTTP **200** with a `THROTTLED` entry +in the GraphQL `errors[]` array — _not_ a `429`. Every retry policy keyed on status codes +sees success and returns the error to the caller. This is the single most-reported trap in +the scenario. + +**3. The wait is arithmetic the server hands you, not a guess.** Each response carries +`extensions.cost` with `requestedQueryCost`, `actualQueryCost`, and a `throttleStatus` of +`{ maximumAvailable, currentlyAvailable, restoreRate }`. The correct wait is +`(requestedQueryCost − currentlyAvailable) / restoreRate` seconds. Exponential backoff with +jitter is strictly worse than the number already in the payload: it over-waits when the +bucket is nearly full and under-waits when it is empty. + +**4. The bucket is the _store's_, not yours.** `currentlyAvailable` reflects every app +touching that shop. A third-party inventory app draining points makes your headroom drop +between two of your own requests +([Shopify/shopify-api-js#602](https://github.com/Shopify/shopify-api-js/issues/602)). No +amount of client-side bookkeeping can predict it — the budget must be re-read from every +response, which makes purely _proactive_ pacing insufficient on its own. + +Put together: the signal is in the **body**, the unit is **cost**, the wait is **computed**, +and the budget is **shared**. A status-code-triggered, curve-based retry is the wrong shape +on all four axes. + +## Evidence this bites real projects + +- **Shopify SDK** — [`Shopify/shopify-api-js#602`](https://github.com/Shopify/shopify-api-js/issues/602): + `currentlyAvailable` drops unexpectedly, because it is the shop's global bucket. +- **Shopify community** — ["limits per query is 1000 but I have 10000 cost available"](https://community.shopify.com/t/graphql-admin-api-rate-limits-limits-per-query-is-1000-but-i-have-10000-cost-available/192109): + the per-query ceiling and the bucket size are different limits, routinely conflated. +- **Practitioner writeups** — [How Shopify's GraphQL rate limits actually work](https://dev.to/masadashraf/how-shopifys-graphql-rate-limits-actually-work-and-how-to-stop-getting-429d-3bnb) + and [a production throttling strategy](https://no7software.co.uk/blog/shopify-graphql-query-cost-production-throttling) + both lead with the same warning: it is a 200, not a 429. +- **Vendor docs** — [Shopify API limits](https://shopify.dev/docs/api/usage/limits). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------ | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| **Retry on 429 + exponential backoff** | The default in every HTTP client. | Never fires — the response is a `200`. Silently returns a THROTTLED error as data. | +| **Body-sniffing retry** | Inspect `errors[].extensions.code === 'THROTTLED'`, then back off exponentially. | Fires correctly, but ignores the arithmetic the server supplied — over- and under-waits by turns. | +| **Compute the wait from `throttleStatus`** | `(requested − available) / restoreRate`, then retry. | Correct, and the state of the art. Requires reading `extensions` — which most GraphQL clients discard when they unwrap `data`. | +| **Client-side cost ledger** | Track spend locally, pre-emptively pause below a threshold (~200 points). | Good for pacing your own traffic; cannot see other apps draining the shop's shared bucket, so it must still reconcile against every response. | +| **Fixed-rate limiter (`N/sec`)** | Pace requests to a safe constant. | Wrong unit. Sized for the worst-case query it wastes most of the quota; sized for the average it throttles on any expensive one. | +| **Global queue with a single worker** | Serialize all calls, pause the queue on deficit. | Genuinely correct and common in production. Costs concurrency and a piece of infrastructure. | + +**Summary of the state of the art:** read `extensions.cost.throttleStatus` off **every** +response (not just failures), retry on a _body_ condition, and wait the _computed_ deficit. +The pieces are simple individually; no mainstream HTTP client wires them together, because +each one needs the body to reach a place where retry decisions are made. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- `StatusMatch = number | number[] | ((status: number) => boolean)` (`types.ts:957`). The + predicate receives the **status only** — never the body. So `retry.on` looks unable to fire + on a 200-with-THROTTLED. Suspected structural gap #1. +- `backoff` is `BackoffCurve | BackoffOptions` — `'expo' | 'expo-jitter' | 'fixed'` plus + `base`/`max` (`types.ts:958-963`). No custom delay function, so a wait computed from the + body has no obvious way in. Suspected structural gap #2. +- `retry.respect` honors a `Retry-After` **header** (`types.ts:993-1002`). Shopify puts the + number in the body instead. +- `throttle.rate` is a `"count/interval"` string and is explicitly _"a minimum spacing between + successive calls … not a token bucket"_ (`types.ts:1006-1019`). Its own doc comment names + this exact situation and points elsewhere: _"Where a real quota needs spending the way the + vendor accounts for it, hand the backoff to an outer gate with `delegate`."_ +- `throttle.delegate` (`types.ts:1046-1048`) is therefore the intended escape hatch — but its + `on` is **also** a `StatusMatch`, defaulting to `[429]`. If it is status-keyed too, a 200 + THROTTLED will not trip it either, and the documented escape hatch does not reach this case. +- `graphql()` fixes `unwrap: 'data'` and treats `errors[]` as a failure (STITCH_GRAPHQL). Does + `extensions` survive anywhere reachable — a hook, the error object, the event stream? + +**Claims to test with runnable offline code:** + +1. **C1** — can `retry` be made to fire on HTTP 200 carrying `errors[].extensions.code === 'THROTTLED'`? +2. **C2** — can the retry wait be **computed from the response body** (`(requested − available) / restoreRate`) rather than from a curve? +3. **C3** — is `extensions.cost.throttleStatus` reachable at all on a `graphql()` stitch — on success (`unwrap: 'data'`), and on the THROTTLED failure? +4. **C4** — does `throttle: { delegate: true }` trip on a 200-with-THROTTLED, or only on statuses? +5. **C5** — can `throttle.rate` express a _cost_ budget (1000 points, refill 50/s) at all? +6. **C6** — if the built-ins fall short: can a user assemble the correct behavior from the public surface (hooks, a custom adapter, `pipe`, delegate + an outer gate)? Write it, run it, and report how much code and which seam carried it. + +The interesting outcome is **C6**. C1–C5 look like "no" from the types; whether that totals +to _not achievable_ or merely _achievable the hard way_ is what decides page vs. issue. diff --git a/docs/scenarios/deprecation-headers.md b/docs/scenarios/deprecation-headers.md new file mode 100644 index 00000000..8387f356 --- /dev/null +++ b/docs/scenarios/deprecation-headers.md @@ -0,0 +1,151 @@ +# Scenario: the vendor told you for six months, in a header + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `deprecation-headers` + +**Verification:** 8 proof scripts (178 checks), run offline, in +[`proofs/deprecation-headers/`](proofs/deprecation-headers/). Published page: +[`scenarios/deprecation-headers.mdx`](../../apps/docs/content/docs/scenarios/deprecation-headers.mdx). +Escalated: [`issue-drafts/hooks-can-rewrite-the-call.md`](issue-drafts/hooks-can-rewrite-the-call.md). + +| Claim | Verdict | Measured | +| -------------------------------- | --------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — where are headers reachable | **definitive: exactly 3 places** | `adapter`, `hooks.onResponse`, `Surface.interpret`. 11 accessors carry none — including the **whole event spine** (4 events, 15 keys, zero headers) | +| C2 — `hooks.onResponse` | sees them, **and can rewrite the call** | return value ignored, but `ctx.res` is live: mutating `status` turned a 200 into a thrown 503. Fires 3× for 1 retried call | +| C3 — a levelled finding | one narrow door | fold into the value → `info \| undeclared`, non-fatal, re-levellable — but carries neither the value nor the endpoint | +| C4 — fleet aggregation | works; the scenario-12 shape transfers | 500 calls / 5 endpoints → _"3 endpoints deprecated … earliest sunset in 12 days: users"_, unchanged at 200:1 skew. But no event carries a header, so the sink saw `null` until a surface folded it | +| C5 — the tripwire | PASS, elegant | `[sunset−1ms, sunset, +1ms]` → `["ok","FAILED","FAILED"]`; burns **no** retry attempts and does **not** open the breaker | +| C6 — both formats | no help reachable | `parseRetryAfter` is _identical_ to the requirement and unexported; `Date.parse("@1735689600")` is **NaN** | +| C7 — noise | one line per call | 600 calls → **360 lines / 3 facts**; `loggerSink` 2400. `levelOf` reaches 600, cannot reach 3 | +| C8 — assembled | PASS | **132 lines vs 81** — the first scenario where the line count goes against the library | + +**A claimed correction that does not survive checking.** The verification reported that +scenarios 6, 7 and 15 "generalised one step too far" about headers. Checked against what +actually shipped, they did not: the multipart page says headers are absent from `.inspect()` +"**so without a surface** the ETag is unrecoverable", the unconfirmed-write page scopes it to +`StitchError`, and the 304 page's own solution _uses_ `Surface.execute`. The issue drafts are +scoped the same way. Nothing was corrected; the accessor table is a **consolidation**, not a +retraction — the first place all eight rows are stated together. + +**The genuinely new thing** is the last row: **no event carries a header**, which is why a +`TraceSink` can only aggregate what a `Surface` folded into the value. That explains the shape +of the answer here and is worth stating next to the trace docs. + +--- + +## The use case + +A vendor is retiring the endpoint you depend on. They announced it in a blog post, sent one +email, and — if they follow the standards — they have been telling you **on every single +response** for six months, in a `Deprecation` and `Sunset` header. + +Then the endpoint goes away and your integration breaks on a Tuesday. + +## Why it is not straightforward + +**The signal arrives on responses that succeeded.** Not on an error, not on a 4xx — on the +`200`s you have been happily consuming all along. So every mechanism a client has for noticing +trouble is pointed the wrong way: nothing failed, nothing retried, no status changed. + +As one write-up puts it: _"The clients that broke never read the blog post — but their code +reads your HTTP responses on every single request."_ + +Then the specifics: + +- **Two headers, two formats.** `Deprecation` (RFC 9745) is a structured-field date — + `@1735689600`. `Sunset` (RFC 8594) is an HTTP-date — `Wed, 01 Jan 2026 00:00:00 GMT`. A client + that parses one and not the other gets half the picture. +- **It is a hint, not a guarantee.** RFC 9745 is explicit: the resource _indicates_, without + guaranteeing, that it will be deprecated. So failing the call is wrong; ignoring it is also + wrong. +- **The useful unit is the fleet, not the call.** "This endpoint is deprecated" is not + interesting once — it is interesting as _which of my forty stitches are deprecated, and which + sunset first_. That needs aggregation across calls, not a per-call log line. +- **The window closes silently.** Between the announcement and the sunset, everything works. The + only thing that changes is the date getting closer, which no runtime notices. +- **Adoption is circular.** The standards are new and under-used _"partly because the value of + the headers is not visible until well-instrumented client libraries notice them."_ A client + that surfaces them is the thing that makes vendors bother to send them. + +## Evidence this bites real projects + +- **RFC 9745** (`Deprecation`) and **RFC 8594** (`Sunset`) define the mechanism, and RFC 9745 + states plainly that a client not interpreting `Sunset` _"can operate as usual and simply may + experience the resource becoming unavailable without recognizing any notification."_ +- **The adoption problem is documented as circular** — the headers are under-used until clients + read them ([Zuplo](https://zuplo.com/learning-center/http-deprecation-header), + [http.dev on Sunset](https://http.dev/sunset)). +- **The failure shape is a genre** — [most deprecation notices die in a changelog nobody + reads](https://oneuptime.com/blog/post/2026-01-30-api-deprecation-headers/view), and the + worked example is stark: _"Day 1: remove /v1/users. Day 1: 47 partner integrations break."_ +- **Zalando's API guidelines** mandate the headers precisely because announcements alone don't + land ([restful-api-guidelines, deprecation](https://github.com/zalando/restful-api-guidelines/blob/main/chapters/deprecation.adoc)). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| -------------------------------------------- | ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| **Read the changelog** | A human subscribes to the vendor's blog. | The documented failure mode. Nobody reads it, and nobody re-reads it six months later. | +| **Log the header** | Print a warning when `Deprecation` appears. | One line per call in a log nobody greps — noise at request volume, and no sense of _which_ endpoints or _when_. | +| **Fail the call after the sunset date** | Turn the hint into a hard stop. | Wrong for a hint, right for a deadline you chose. Useful as a deliberate tripwire, dangerous as a default. | +| **Aggregate to a dashboard** | Count deprecated endpoints and earliest sunset. | The genuinely useful shape, and it needs somewhere to aggregate — a per-call hook has no memory. | +| **Contract tests against the vendor's spec** | Notice at build time. | Catches a _shipped_ change; a sunset announcement is not in the spec, and the change hasn't shipped yet. | +| **Gateway/proxy inspection** | Let infrastructure watch the headers. | Works, and only if you have one in the path. | + +**Summary of the state of the art:** parse both headers, don't fail on a hint, aggregate across +calls so the answer is a _list of endpoints with dates_, and consider a deliberate tripwire when +a sunset you know about arrives. + +--- + +## What to verify against StitchAPI + +This scenario is deliberately built on a thread three earlier ones brushed. [Scenario +6](conditional-requests-304.md) measured `Inspection` carrying no headers, so an `ETag` was +unrecoverable without a surface. [Scenario 7](multipart-upload.md) hit the same wall for a +part's `ETag`. [Scenario 15](unconfirmed-write.md) measured `StitchError` having no `headers`, +so a vendor's replay marker was unreachable. **This is the same question in its purest form: a +signal that lives only in a response header, on a call that succeeded.** + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **Nothing ships for this.** No `Deprecation`/`Sunset` handling is likely to exist anywhere. + That is fine and expected — the question is whether the _seams_ make it a few lines or a + rewrite. +- **`hooks.onResponse` sees the headers** — scenario 11 measured it seeing every page of a + paginated run, read-only. If it can observe but not act, it is the right place for a _report_ + and the wrong place for a _guard_. +- **`TraceSink` is the aggregation seam** — [scenario 12](intermittent-drift.md) measured + `trace` + `ctx.spanId` being the one place cross-call state is the design rather than a leak, + and used it to turn per-call drift findings into a rate. The fleet-level question here has + exactly that shape. +- **Drift has levels and a finding vocabulary.** Whether a header-derived warning can join it — + or whether findings are strictly schema-derived — decides whether this reports through the + same channel as everything else or needs its own. + +**Claims to test with runnable offline code:** + +1. **C1** — is a response header reachable on a **successful awaited** call, at all? Enumerate + every accessor (`await`, `.safe()`, `.inspect()`, `.report()`, the event stream) and say which + carry headers. +2. **C2** — `hooks.onResponse`: does it see `Deprecation`/`Sunset` on a 200, and can it do + anything beyond observe? +3. **C3** — can a header-derived warning become a **finding** in the same channel as drift — + levelled, non-fatal, naming the endpoint? Or does it need a parallel mechanism? +4. **C4** — **DECIDING CLAIM.** Aggregation. Across 40 stitches and many calls, can a + `TraceSink` produce _"these 3 endpoints are deprecated, earliest sunset in 12 days"_? Measure + what identifies the endpoint (`ctx.name`? the URL?) and whether the header reaches the sink. +5. **C5** — the deliberate tripwire: can a call be made to **fail** after a sunset date you + choose, without failing before it? Measure with an injected clock. +6. **C6** — both formats: RFC 9745 structured-field `@1735689600` and RFC 8594 HTTP-date. Is + there any parsing help, or is it two hand-rolled parsers? (Scenario 14 found `parseRetryAfter` + exists but is unexported.) +7. **C7** — noise: at request volume, does the naive approach produce one line per call? Can it + be de-duplicated per endpoint without hand-rolled state? +8. **C8** — assemble the best available answer — parse both, report as findings, aggregate to a + fleet view, optional tripwire — and report the seam and line count. + +C1 and C4 decide this. If response headers are unreachable on the success path and the sink is +the only aggregation point, then this scenario's answer and its ask are the same as three +earlier ones — which would make it worth consolidating rather than filing a fourth time. diff --git a/docs/scenarios/dual-run-migration.md b/docs/scenarios/dual-run-migration.md new file mode 100644 index 00000000..4a69d6e9 --- /dev/null +++ b/docs/scenarios/dual-run-migration.md @@ -0,0 +1,159 @@ +# Scenario: the migration you have to run twice + +**Researched:** 2026-08-05 · **Status:** ✅ verified (8 claims, 142 checks, offline) · page shipped +**Slug:** `dual-run-migration` + +--- + +## The use case + +Your vendor is retiring v1. [Scenario 17](deprecation-headers.md) is how you found out; this is +what you do next. You cannot flip to v2 on faith, so you run both against real traffic, compare +the answers, and cut over when the diff is quiet. + +## Why it is not straightforward + +**Every shadow-traffic guide in the field is written for the service owner.** The standard +architecture is _"Client → API Gateway → Primary Service (v1) with mirrored requests to Shadow +Service (v2)"_, mirroring at the proxy, with a shadow database alongside. You own the gateway, +both services, and both datastores. + +**As the consumer of a third-party API you own none of that.** Which changes every term: + +- **There is no proxy to mirror at.** The duplication has to happen in your own client code, on + the call path, which is exactly where you cannot afford it to go wrong. +- **The shadow spends the vendor's meter, not yours.** Mirroring 100% of traffic doubles your + rate-limit consumption and your bill. The advice — _"shadow a portion of production traffic… + and sample heavily"_ — is a cost control, not a statistical one. +- **You cannot shadow a write.** A mirrored `POST /charges` charges the customer twice. The + entire technique is read-only, and the writes are the calls you are most afraid of migrating. +- **The shadow must not be able to hurt the primary.** A slow or failing v2 must not add latency, + must not fail the user's call, and must not consume the retry/circuit budget the primary + depends on. +- **Telling a real diff from a benign one is the actual work.** The guidance is explicit that a + good implementation needs _"a relevancy model to decide which differences matter and which are + benign noise, like timestamps or reordered fields."_ A v2 that renames `created` to + `created_at`, returns ISO instants instead of epochs, and orders an array differently is + **correct** — and diffs on every single call. + +## Evidence this bites real projects + +- **The canonical architecture, and its assumptions** — + [Safely replacing production services using shadow traffic](https://medium.com/@sonishubham65/safely-replacing-production-services-using-shadow-traffic-with-istio-on-kubernetes-57c0516602e2) + (Istio, mirrored at the mesh) and + [Gloo Edge shadowing](https://docs.solo.io/gloo-edge/latest/guides/traffic_management/request_processing/shadowing/). + Both mirror at infrastructure you must own. +- **The relevancy problem, named** — + [What is shadow testing?](https://www.signadot.com/blog/shadow-testing-superpowers-four-ways-to-bulletproof-apis/): + outputs are diffed to surface regressions, and a good implementation distinguishes real + differences from benign noise like timestamps and reordered fields. +- **Sample heavily, redact** — the same source, on shadowing a portion of traffic into a canary. +- **Dual-write is the write-side analogue** — + [Dark launch patterns](https://oneuptime.com/blog/post/2026-01-30-dark-launch-patterns/view): + dark launches "shine during database migrations where you can write to both and verify + consistency" — which is precisely the thing a third-party consumer cannot do. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------- | -------------------------------------------- | --------------------------------------------------------------------------------------- | +| **Mirror at the gateway** | Proxy duplicates the request. | The standard answer, and unavailable when the endpoint is someone else's. | +| **Dual-call in the client** | Issue both, return v1, log the diff. | Available to you, and now the shadow is inside your latency and failure budget. | +| **Offline replay** | Capture v1 traffic, replay against v2 later. | No user impact, and no live comparison — you find out at replay time, not at call time. | +| **Diff in a batch job** | Log both, compare nightly. | Cheap and slow. A regression lives a day. | +| **Trust the changelog** | Read the migration guide, flip. | Free, and the reason this scenario exists. | +| **Sample a small percentage** | Shadow 1–5% of reads. | The cost control that makes it viable. Needs a spelling. | + +**Summary of the state of the art:** duplicate reads only, sample them, keep the shadow strictly +off the primary's critical path, and put most of your effort into a comparison that ignores +differences you already know about. + +--- + +## What to verify against StitchAPI + +Two findings from earlier iterations make specific, testable predictions here, and both are +**pre-registered** so the proofs can refute them on the record. + +1. **The combinators broadcast one input.** [Scenario 10](provider-failover.md) measured + `runMember` (`pipe.ts:75-86`) building every member's input from the one group input — the + basis of [#643](https://github.com/rejifald/StitchAPI/issues/643). A dual-run is the case that + needs the opposite: v1 and v2 differ in path, parameter names and body shape, so **the + combinator that looks purpose-built for this is the one already measured to be wrong for it.** +2. **Resilience state is shared unless keyed by hand.** [Scenario 9](multi-tenant-blast-radius.md) + measured a 9-of-9 blast radius when one principal's failures opened a shared breaker + ([#641](https://github.com/rejifald/StitchAPI/issues/641)). If a flaky v2 shadow shares a + circuit or throttle with v1, **the experiment can take down the thing it was protecting** — + the worst possible failure for a safety mechanism. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Can the shadow be made unable to hurt the primary? Measure four + channels separately: added **latency**, a **thrown** shadow error reaching the caller, the + shadow consuming the **retry** budget, and the shadow's failures opening a **circuit** the + primary uses. Report which need explicit config and which are safe by default. +2. **C2** — **DECIDING CLAIM.** Can the two calls take **different inputs**? Try `all`/`any`, a + plain `Promise.all`, `linked`, and `.with()`. If the combinators cannot express it, measure + what the working spelling costs. +3. **C3** — is there a **comparison** primitive? `drift()` is schema-anchored (response vs + contract) — can anything compare **response vs response**? What is the minimum hand-written + comparator? +4. **C4** — the relevancy problem: can "ignore `updated_at`, ignore array order, treat + `created` ≡ `created_at`" be expressed declaratively, or is it all user code? Measure the diff + noise on a realistic v1→v2 rename + retype + reorder. +5. **C5** — **writes**: is there any guard that prevents a shadowed non-GET? What is the cheapest + construction that makes shadowing a write impossible rather than merely discouraged? +6. **C6** — **cutover**: can v1→v2 be flipped without a redeploy? (Scenario 19 measured a + `baseUrl` thunk retargets between calls — does that extend to a whole different stitch?) +7. **C7** — **cost**: does the shadow double rate-limit consumption? Is sampling ("shadow 5% of + reads") expressible, and does a shared `throttle` correctly account for both? +8. **C8** — assemble the safest dual-run; report seams, line count, and what it costs. + +C1 and C2 decide this. C1 is the safety property that makes the technique usable at all; C2 is +where the pre-registered suspicion says the obvious tool will fail. + +--- + +## Verification result + +**All 8 claims verified**, 142 checks across 8 scripts, re-run by me before writing up. + +| Claim | Verdict | +| --------------------------------- | ---------------------------------------------------------------------------- | +| C1 — can the shadow be isolated? | **PARTIAL** — 1 of **5** channels safe by default (there are five, not four) | +| C2 — different inputs per member? | **CONFIRMED as predicted** — the shadow got `/v2/customers`, no id | +| C3 — a comparison primitive? | **PARTIAL** — two exist in the tree, neither exported | +| C4 — the relevancy problem | 7 raw diff ops on a _correct_ v2; `ignore` is suppression, not relevancy | +| C5 — write guard | None exists; 8-line Adapter wrapper gives 0 of 3 shadow writes | +| C6 — cutover | Thunk moves the whole path, not just the origin; real cutover is 4 lines | +| C7 — cost | Exactly **2.000×**; sampling is 5 lines of user code | +| C8 — assembled | **69** executable lines, 5 seams; naive 0-of-4 vs safe 4-of-4 | + +### Both pre-registered predictions + +**[#643](https://github.com/rejifald/StitchAPI/issues/643) — CONFIRMED.** `runMember` broadcasts +one input; the shadow's literal URL was `/v2/customers` with no id, and supplying v2's parameter +name made v1 send `/v1/customers/cus_7Q2?customer_id=cus_7Q2`. + +**[#641](https://github.com/rejifald/StitchAPI/issues/641) — CONFIRMED IN 2 OF 5 CONFIGS, AND +REFUTED AS I STATED IT.** Sharing is _not_ the default. It is a **key collision**: identity is +`(store) × ('circuit:' + (circuit.key ?? name ?? path ?? 'stitch'))`. Standalone and +seam-with-distinct-paths both isolate correctly. The two that break — a seam with the **same +path**, and `pool: 'host'` — are both ordinary dual-run shapes, because v1 and v2 usually share a +path and differ by base URL, and are usually on the same host. The sharper statement is better +than my prediction and worth carrying forward over it. + +### Other hypotheses that were wrong + +- **"Four isolation channels."** There are five. The one I missed — `all()` **cancelling the + in-flight primary** when the shadow settles first — is the most dangerous of them. +- **"A thunk moves only the base URL."** `url` is also a thunk and carries the complete path, + with `{param}` interpolation still applying. +- **"There is no comparison primitive."** There are two, `diff` and `classifyDiff`; they are + simply unreachable from all 17 subpaths. +- **"A shared `throttle` won't account for both."** Seams pool correctly by default. + +### Outputs + +- Page: [dual-run-migration.mdx](../../apps/docs/content/docs/scenarios/dual-run-migration.mdx) +- Draft: [diff-primitives-are-unreachable](issue-drafts/diff-primitives-are-unreachable.md) diff --git a/docs/scenarios/expiring-signatures.md b/docs/scenarios/expiring-signatures.md new file mode 100644 index 00000000..8e8388e0 --- /dev/null +++ b/docs/scenarios/expiring-signatures.md @@ -0,0 +1,143 @@ +# Scenario: the signature that expired in your own queue + +**Researched:** 2026-08-05 · **Status:** VERIFIED — **ACHIEVABLE** (the first outright) · page shipped +**Slug:** `expiring-signatures` + +**Verification:** 8 proof scripts (105 checks), run offline, stable across 24 runs, in +[`proofs/expiring-signatures/`](proofs/expiring-signatures/). Published page: +[`scenarios/expiring-signatures.mdx`](../../apps/docs/content/docs/scenarios/expiring-signatures.mdx). +Escalated: [`issue-drafts/sigv4-ignores-the-injected-clock.md`](issue-drafts/sigv4-ignores-the-injected-clock.md). + +| Claim | Verdict | Measured | +| --------------------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — signed per attempt? | **per attempt** | 3 attempts 6 min apart → 3 distinct signatures, ages `[0,0,0]` ms; a 10-min `Retry-After` park still arrived fresh. Signed-once control: 6 min old, **403** | +| C2 — wait before or after signing | **BEFORE — the deciding claim, positive** | 4 calls behind `rate: '1/2m'` → ages `[0,0,0,0]`, all `200`. Pre-signed control: `[0,2,4,6]` min and a **403** | +| C3 — concurrency | same | held 6 min behind `concurrency: 1` → **0 ms** | +| C4 — circuit cooldown | breaker queues nothing | 3 blocked calls → **0 signings**; half-open trial signed fresh | +| C5 — injected clock? | **library loses** | 600 virtual seconds moved the stamp **0 s**; under a default `manualClock()`, **0 of 3** accepted | +| C6 — skew 403 | not retried (good); classification fails | 1 request at `attempts: 4`. But it **counts as a circuit failure**, and `verdict: {accept, flag}` **swallows** it | +| C7 — skew correction | reachable | `shouldRefresh`/`refresh` learned **600,000 ms** from the `Date` header and re-signed the same attempt to `200`, costing no retry budget | +| C8 — assembled | PASS | **4 of 4** through 10-min drift + 6-min queue + breaker, worst age **0 ms**; 26 lines, all for the drift half | + +**The first scenario in the pass to come out ACHIEVABLE outright for its deciding claim**, and +the reason is structural: `acquireWithin` (`engine.ts:629`) sits above `cfg.auth.apply` (`:649`) +inside the attempt loop, and `cloneReq` gives each attempt fresh headers off the _unsigned_ +base. **A StitchAPI throttle cannot expire a signature** — botocore#149 is unreachable here. + +**The capture's hypotheses held**, which is also a first. What it did not anticipate is the +inverse footgun: `hooks.onRequest` runs _after_ signing (`:652`), so a hand-rolled pacing gate +there **re-creates the bug inside a library that doesn't have it** — measured, a 6-minute wait +in `onRequest` aged the signature 6 minutes and got a 403. + +--- + +## The use case + +You call a service that requires **signed requests** — S3 or any AWS API via SigV4, or any +vendor whose auth embeds a timestamp. The signature covers the clock, and the server rejects +anything more than **five minutes** from its own time. That window exists to stop replay +attacks, and it is not negotiable. + +## Why it is not straightforward + +There are three separate ways to fall outside the window, and only one of them is your clock. + +**1. The clock drifts.** Containers inherit the host's time at start and do **not** re-sync +after. A drifting host, a VM resumed from suspend, a laptop out of NTP — and every request +fails with `RequestTimeTooSkewed`. **Retry makes it worse, not better**: the same stale clock +produces the same invalid timestamp on every attempt, so a retry policy burns the budget and +fails identically. AWS's own SDKs have shipped bugs here — +[`aws-sdk-net#3463`](https://github.com/aws/aws-sdk-net/issues/3463), "clock skew correction +causes repeated retries." + +**2. The signature ages in a queue — yours.** This is the sharp one, and it has nothing to do +with your clock being wrong. Sign the request, then hold it: behind a rate limiter, behind a +concurrency cap, behind a retry backoff. The timestamp was minted at _sign_ time and the +request reaches the wire minutes later. As one AWS answer puts it plainly: _"The SDK signs the +request, and then puts the request in a queue. If the queue becomes too large and the request is +pending for more than 5 minutes, then the signature expires."_ The fix filed against botocore +([`#149`](https://github.com/boto/botocore/issues/149)) is exactly this: generate the timestamp +**per signing operation**, not once at construction. + +**3. The retry replays a stale signature.** If signing happens once per _call_ rather than once +per _attempt_, then attempt 2 carries attempt 1's timestamp plus however long the backoff was. +A long backoff — or a circuit-breaker cooldown — guarantees the replay is stale. + +The compounding detail: **a skew failure looks transient.** `RequestTimeTooSkewed` is a 403, +and the natural reading is "auth problem, retry it." So the failure mode most likely to be +retried is the one retry cannot fix. + +## Evidence this bites real projects + +- **`aws-sdk-net#3463`** — [clock skew correction causes repeated retries](https://github.com/aws/aws-sdk-net/issues/3463). +- **`botocore#149`** — the cached-signature-timestamp bug, whose fix is to regenerate the + timestamp per signing operation. +- **Lambda "Signature expired"** — [AWS's own knowledge-centre article](https://repost.aws/knowledge-center/lambda-sdk-signature) + names the sign-then-queue case directly. +- **Containers** — [signature expired when running from Docker](https://www.w3tutorials.net/blog/aws-invalidsignatureexception-signature-expired-when-running-from-docker-container/) + and [RequestTimeTooSkewed: the S3 error that brings teams to a halt](https://www.tech-reader.blog/2025/09/requesttimetooskewed-s3-error-that.html). +- **AWS's clock-skew-correction blog** documents the mitigation SDKs implement: learn the offset + from the server's `Date` header on a skew error, then re-sign with the corrected clock. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------------- | -------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **Fix the clock (NTP/chrony)** | Keep the host in sync. | The correct root fix, and it is infrastructure — not something a client library can do, and not available in every environment. | +| **Sign per attempt** | Re-sign on every retry rather than once per call. | Necessary and cheap. Easy to get wrong by hoisting the signing above the retry loop. | +| **Sign after the wait, not before** | Move signing below the throttle/concurrency queue. | The fix for case 2, and it requires the signing hook to run at the _last_ moment before the wire. | +| **Clock-skew correction** | Read the server's `Date` on a skew error, store the offset, re-sign. | What the AWS SDKs do. Needs somewhere to persist the offset and a way to feed it back into signing. | +| **Don't retry a skew error** | Classify 403-skew as terminal. | Correct, and the opposite of the natural reading — this is the "classify before routing" lesson from [provider failover](provider-failover.md). | +| **Widen the window** | Ask the vendor for longer validity. | Not on offer. Five minutes is a security property. | + +**Summary of the state of the art:** sign as late as possible, sign again on every attempt, +don't retry a skew error blindly, and correct from the server's clock when it tells you. + +--- + +## What to verify against StitchAPI + +This scenario pulls on a thread three earlier ones brushed. Scenario 6 measured +`Surface.buildRequest` running **once per run** while `hooks.onRequest` runs **once per +attempt**, and located `cfg.auth.apply` at `engine.ts:649` — _inside_ the attempt loop. So +per-attempt signing looks likely. The untested part is **where the throttle wait sits relative +to it**. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **`@stitchapi/aws-sigv4` is the one package this pass hasn't touched.** It signs outbound + requests and was measured in [scenario 8](webhook-receipt.md) producing an + `AWS4-HMAC-SHA256` header with a key imported `usages: ['sign']`. +- **The sharp question is ordering.** If the throttle's wait happens _before_ `auth.apply`, the + library is already correct for case 2 and that is worth saying loudly. If it happens _after_, + a stitch with `throttle: { rate: '1/s' }` and a queue signs at t=0 and arrives minutes later. +- **Does signing use the injected `clock`?** Scenarios 4 and 6 found `timeout.total` and + `cache.ttl` reading `Date.now()` while their neighbours use `clock`. If SigV4 does the same, + clock skew is untestable on a virtual clock — and it would be the **third** instance of one + inconsistency. +- **Nothing in the config vocabulary suggests skew correction**, so the "learn the offset from + the server's `Date`" mitigation is presumably user code — the question is whether there is a + seam that can reach the signing input at all. + +**Claims to test with runnable offline code:** + +1. **C1** — is the request signed **per attempt** or once per call? Capture the timestamp on the + wire across a retry with a long backoff. If attempt 2 carries attempt 1's timestamp, that is + the finding. +2. **C2** — **DECIDING CLAIM.** Does the throttle wait happen **before or after** signing? + Configure `throttle: { rate }` so a call queues for a long virtual interval, and measure the + age of the signature when it reaches the adapter. +3. **C3** — same question for `throttle: { concurrency }` — a request held behind a busy pool. +4. **C4** — does the circuit breaker's half-open delay interact the same way? +5. **C5** — does SigV4 signing read the **injected clock** or `Date.now()`? Advance a + `manualClock` and check whether the signature's timestamp moves. +6. **C6** — a `RequestTimeTooSkewed` 403: is it retried by default? Can it be classified as + terminal without swallowing it? (Scenario 9 measured `verdict: { accept, flag }` doing this + for a 401.) +7. **C7** — clock-skew correction: is there any seam that can read the server's `Date` on a + failure and feed a corrected clock back into signing for the next attempt? +8. **C8** — assemble the best available answer, run it, report the seam and line count. + +C2 decides this one. Signing before a wait you control is a bug the client can fix; signing +after it is a property worth advertising. diff --git a/docs/scenarios/intermittent-drift.md b/docs/scenarios/intermittent-drift.md new file mode 100644 index 00000000..95535207 --- /dev/null +++ b/docs/scenarios/intermittent-drift.md @@ -0,0 +1,150 @@ +# Scenario: the vendor changed the shape for 5% of responses + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `intermittent-drift` + +**Verification:** 8 proof scripts, run offline (142 checks), in +[`proofs/intermittent-drift/`](proofs/intermittent-drift/). Published page: +[`scenarios/intermittent-drift.mdx`](../../apps/docs/content/docs/scenarios/intermittent-drift.mdx). +Escalated: [`issue-drafts/drift-cannot-grade-a-coercion.md`](issue-drafts/drift-cannot-grade-a-coercion.md). + +| Claim | Verdict | Measured | +| ---------------------------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — added field | quiet, as it should be | one `info \| undeclared`; 51 added values across 50 array elements collapse to **2** findings; `ignore` silences it. But the value is **stripped** from `data` | +| C2 — removed field | caught, **conditionally** | `error \| invalid` + failed call — _if required_. `.optional()` → zero findings; `.default()` → a fabricated value at `verbose` | +| C3 — the $0-transaction test | **capture half REFUTED, half worse** | default is **safe**: `z.number()` on `"12345"` fails; `z.coerce.number()` on `"abc"` fails. But `z.coerce.number()` on `null` → **`0`** with no `.catch()`, and the finding is byte-identical to the benign one | +| C4 — 5% null | precision excellent, level missing | fired on exactly calls `[20,40,60,80,100]`, naming the field. But `.nullable()` → **nothing**, so the rollout is invisible | +| C5 — declarative severity | partly, keyed on the wrong axis | the 3 soft kinds re-level in one literal; only _addition_ maps 1:1 to a change class; no per-path severity | +| C6 — aggregation | **capture REFUTED — it works** | `5.0% of calls … (5/100, 5 landed 0)`, and a rolling window widening **5.0% → 25.0%** | +| C7 — actionability | depends entirely on the accessor | path always; `await`/`.safe()` carry **nothing** for a soft finding; the trace sink for the same run named the field and both types | +| C8 — assembled | PASS | 6 workloads × 100 calls, **zero $0 charges**; the soft schema produced **ten**. 93 lines vs 92 | + +**Two hypotheses refuted, and both in the library's favour — a first for this pass.** + +- The capture feared StitchAPI would silently coerce a type change into a plausible value. It + does not: the default rejects, and even `z.coerce.number()` refuses `NaN`. The $0 charge is + reachable only through two spellings the _user_ writes. +- The capture predicted aggregation would be the gap, citing every prior scenario's failure to + find cross-call state. Wrong: `TraceSink` + `ctx.spanId` is the one place in the library where + cross-call state is the design rather than a leak. + +**What survives.** A `coerced` finding is `kindOf(old) -> kindOf(new)` with no values, so +`"12345" → 12345` and `"abc" → 0` are indistinguishable; and the fourth industry change class +("nullable is a warning, value intact") has no spelling — the hand-rolled classifier beats +`DriftOptions` on exactly that row. + +--- + +## The use case + +A vendor ships a change to their response shape. Not all at once — **gradually**. A canary at +5% of traffic, then 25, then 50. Or not a rollout at all: the shape simply differs _by data_ — +a geocoder that returns `null` for `formatted_address` only on ambiguous queries. + +Either way your integration sees the new shape on **some** calls and the old shape on the rest. + +## Why it is not straightforward + +**Intermittent breakage is harder than total breakage.** A change that breaks 100% of calls is +found in minutes and rolled back. A change that breaks 5% produces a trickle of odd errors +that looks like flakiness, sits in the backlog for a week, and is fixed only after someone +notices the pattern. + +The change classes are not equal, and treating them alike is the mistake: + +- **A field is added** — non-breaking by every published policy. Should produce no alarm at all, + or you get alarm fatigue on every vendor release. +- **A field is removed** — breaking. Must be loud. +- **A field's type changes** — breaking, and the _dangerous_ one, because a naive cast produces + a **plausible** value. The canonical example: a payment provider changes `transaction_id` + from integer to string, code casts to int, gets `0`, and processes a **$0 transaction**. +- **A field becomes nullable** — warning-level, and the most intermittent of all, because it + only shows up on the data that triggers the null. The geocoding `formatted_address` case is + exactly this: nothing is wrong until a query happens to be ambiguous. + +Then the operational problem, which is the one this scenario is really about: + +- **One finding per call is not a signal.** During a canary you get a drift finding on 5% of + calls. To act you need to know _this is trending_ — 5% yesterday, 25% today — which requires + **counting across calls**. A per-call event carries no memory. +- **And you need it to be actionable at 3am**: which field, what was expected, what arrived. + "Validation failed" is not enough to page someone about. + +## Evidence this bites real projects + +- **The `transaction_id` int→string → `$0` transaction** and the **geocoder returning `null` + for `formatted_address` on ambiguous queries** are both given as canonical schema-drift + failures in [Your API tests are lying to you: the schema drift problem nobody talks + about](https://dev.to/qa-leaders/your-api-tests-are-lying-to-you-the-schema-drift-problem-nobody-talks-about-4h86). +- **The change taxonomy is industry-standard** — [LinkedIn's breaking-change + policy](https://learn.microsoft.com/en-us/linkedin/shared/breaking-change-policy?view=li-lms-2024-06) + and [Xandr's](https://learn.microsoft.com/en-us/xandr/digital-platform-api/breaking-changes) + both classify removal and type change as breaking, and _addition_ as explicitly non-breaking. +- **Nullable-without-notice** is called out repeatedly as its own class: a field becoming + nullable is warning-level, not breaking, and is the one that hides in the data. +- **Canary rollouts are standard practice** ([Google SRE](https://sre.google/workbook/canarying-releases/), + 5–10% → 25 → 50 → 100), and the write-ups note the limitation directly: canarying a + _response-schema_ change doesn't protect the client, it just makes the breakage partial. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | +| **Strict schema validation** | Reject anything that doesn't match. | Catches everything — including the _added field_ that broke nothing. Every vendor release becomes an outage. | +| **Parse loosely, cast defensively** | `Number(x) \|\| 0`, optional chaining everywhere. | Never alarms, and manufactures the `$0` transaction. The failure moves from the boundary into your business logic. | +| **Contract tests in CI** | Assert the shape against a recorded fixture. | Catches it before deploy — and the vendor changed _after_ your deploy. CI can't see a canary in production. | +| **Level the findings** | Additions info, nullability warn, removal/type-change error. | The right model, and it needs a vocabulary most validators don't have. | +| **Log and aggregate** | Emit a finding per call, count them centrally. | The only way to see a 5%→25% trend. Requires the finding to carry the _field_, and somewhere to count. | +| **Pin a vendor API version** | `Accept: application/vnd.x.v3+json`. | The real fix where offered. Doesn't help with data-dependent nulls, and vendors sunset versions. | + +**Summary of the state of the art:** classify by change type, don't fail on additions, be loud +about removals and type changes, treat nullability as a warning, and **aggregate across calls** +— because a 5% signal is only interpretable as a rate. + +--- + +## What to verify against StitchAPI + +Leveled drift is a headline feature and the last one this section hasn't stressed. The +[recipes](../../apps/docs/content/docs/recipes/catch-a-breaking-api-change.mdx) already cover +_catching a breaking change_; this scenario is about the **operational** case — partial, +intermittent, and needing to be interpreted as a rate. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- Scenario 11 measured `drift()` producing findings at `coerced` and `undeclared` levels + (`drift.ts:59-69`), so a **level vocabulary exists**. The question is whether it maps onto the + industry taxonomy: addition → quiet, removal → loud, type change → loud, nullable → warn. +- **The `coerced` level is the one to look at hardest.** If a `transaction_id` of `"12345"` + silently becomes `12345` with an info-level finding, that is correct and useful. If `"abc"` + becomes `NaN` or `0` at the same level, that is the `$0` transaction with a warning nobody + reads. +- Scenario 11 also measured that findings are reachable on `.report().findings` but that + `.safe()` gets a generic message — so **actionability may depend on which accessor you use**. +- **Aggregation is the likely gap.** Every scenario in this pass that needed cross-call state + found none: no per-call slot on `HookContext` (scenario 7), no run-scoped state (scenario 7), + closures that leak across calls (scenario 11). A 5% drift rate needs counting, and counting + needs somewhere to count. + +**Claims to test with runnable offline code:** + +1. **C1** — an **added** field. Does it alarm? At what level, and does it reach the caller? +2. **C2** — a **removed** field. Is it caught, and is it distinguishable in level from C1? +3. **C3** — a **type change** with a plausible coercion: `transaction_id` `12345` → `"12345"`, + and the dangerous variant `"abc"`. What does the caller actually receive — the string, a + number, `NaN`, `0`? At what level? **This is the $0-transaction test.** +4. **C4** — a field becomes **null** on 5% of responses (the geocoder case). Does drift fire + only on those, and does the finding name the field? +5. **C5** — can the four classes be given **different severities** — addition silent, removal + fatal — declaratively? +6. **C6** — **aggregation.** Over 100 calls where 5 drift, can the caller learn "5% of calls + drifted on field X"? Is there any counting, or is each call independent? Where would a + counter live? +7. **C7** — is the finding **actionable**: field path, expected, actual? And which accessors + carry it (`.safe()`, `.report()`, the event stream, a trace sink)? +8. **C8** — assemble the most honest answer for a canary rollout: quiet on additions, loud on + removals/type changes, and a rate you can alert on. Report the seam and line count. + +C3 and C6 decide this one. A drift system that silently coerces a type change is worse than +none, and one that can't be aggregated cannot tell you a rollout is happening. diff --git a/docs/scenarios/issue-drafts/adr-0018-findings-can-leak-a-value.md b/docs/scenarios/issue-drafts/adr-0018-findings-can-leak-a-value.md new file mode 100644 index 00000000..b7004364 --- /dev/null +++ b/docs/scenarios/issue-drafts/adr-0018-findings-can-leak-a-value.md @@ -0,0 +1,97 @@ +# ADR 0018 §4 is false for hard validation, and the disk sink skips its own deep scrubber + +**Status:** drafted, **HELD BACK — security-sensitive.** Review before disclosing. +**Scenario:** [pii-in-the-logs](../pii-in-the-logs.md) +**Proofs:** `docs/scenarios/proofs/pii-in-the-logs/` (8 scripts, 196 checks, offline) + +> Held per the standing instruction not to file drafts that disclose a data-exposure path. Both +> findings below are sensitive-data-in-logs (CWE-532 shape). The third section is ordinary and +> could be split out and filed on its own. + +## 1. ADR 0018 §4's safety claim does not hold for hard validation + +[ADR 0018](../../adr/0018-inspect-raw-redaction.md) line 118 states: + +> This is safe because `detailFor` emits **kinds only, never values** (`string -> number`, +> `undeclared field (string)`) — so `findings` never leak a secret even when `redact` is off. + +The reasoning is sound for `detailFor`, which handles the three **soft** drift kinds. But +`detailFor` is not the only producer of findings. `validationErrors` +(`packages/core/src/drift.ts:50-56`) handles **hard** validation and copies the validator's own +message verbatim: + +```ts +export function validationErrors(issues: Issue[]): DriftFinding[] { + return issues.map((iss) => ({ + level: 'error', + path: renderPath(iss.path), + change: 'invalid', + detail: iss.message, // ← the validator's message, unmodified + })); +``` + +Several validators quote the offending value in that message. Zod's enum message is +`Invalid enum value. Expected 'enterprise' | 'free', received ''`. So a field whose +value is sensitive, validated against an enum, puts that value into `detail`. + +**Measured:** it reaches the JSONL file sink, `consoleSink` **and** `loggerSink` — the two sinks +that carry zero of seven PII sentinels in every other measurement in this scenario. OTLP alone +stays clean, because it exports level/path/change and drops `detail`. + +The ADR's claim is scoped to one of two finding producers, and the scoping is not stated. + +**Ask:** either make `validationErrors` emit a kind rather than the raw message (it already has +`path` and `change`), or narrow the ADR's claim and say plainly that a hard validation `detail` +carries whatever the validator chose to put in it. + +## 2. The disk sink ships a deep scrubber and does not use it + +A credential in a **response body** — `access_token`, `refresh_token`, `session_cookie` — is +written to the JSONL log in full (3 of 3 measured). A `client_secret` in a **request body** is +too. + +The cause is that the file sink's redactor is a five-name **header** denylist rather than +`isSecretKey`. The same file already contains the deep secret-key scrubber and applies it to a +request body for the `serve` SSE transport (`redactEventForTransport`, `trace.ts:85`) — the disk +sink simply never calls it. + +Worth stating alongside: the credential half is otherwise genuinely good. A **declarative** +strategy never enters the event stream at all, because `auth.apply` runs on a request clone +inside the attempt loop while `start` was built from the pre-auth request — 0 of 3 even for a +naive custom sink. Hand-rolled request credentials are scrubbed everywhere. It is specifically +credentials that ride the _payload_ that get PII treatment, which is to say none. + +**Ask:** run the body through `redactSecretsDeep` in the disk sink, as the SSE transport path +already does. + +## 3. Ordinary findings, no disclosure — splittable + +- **`redactHeaders` reaches body keys at any depth**, and its type and JSDoc both say "header + names". It is the one config-reachable way to point the JSONL redactor at a body key, and + nothing says so. A body field named `cookie` becomes `[REDACTED]` while the identical value + under `ssn` does not. +- **Two redaction sentinels in one library** — `REDACTED` (`util.ts`) and `[REDACTED]` + (`trace.ts`), both appearing in the _same_ JSONL record (`url` vs `input.query`). +- **Two path grammars disagree on arrays.** A drift finding prints `contacts[].email`; pasting + that into `.inspect({ redact })` matches nothing — only a bare key or a concrete + `contacts[1].email` works. The two are natural to copy between. +- **`redact` is the second argument**, so `call.inspect({ redact: true })` is a silent no-op (the + object lands in the `input` slot). TypeScript rejects it, so it is only reachable from JS or + through a cast — but the no-op spelling is the shorter and more natural one. +- **ADR 0018 §1's `defaultInspect` was never implemented**, so there is no stitch-level or + process-level way to make every `.inspect()` call redact. +- **`severity: { undeclared: 'error' }` is a compile error and a working runtime kill-switch.** + `DriftSeverity` excludes `error` and the JSDoc says soft drift is always non-fatal, but through + a cast it re-levels the finding and fails the call. There is no runtime guard behind the type. +- **`sensitive: true` survives onto the public `__config`**, so `.report()` prints + `"sensitive":true` beside a record it did not protect — it gates the cache and nothing else + (one read, `engine.ts:1022`). +- **`drift.ts`'s key walker is 23 lines and not exported.** Anyone stripping PII at + `hooks.onResponse` _and_ wanting the drift inventory has to re-implement it, because the + boundary removes exactly the bytes drift diffs. + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +drafting. Runnable proof scripts live under `docs/scenarios/proofs/pii-in-the-logs/` on the +branch `claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/any-is-priced-as-a-hedge.md b/docs/scenarios/issue-drafts/any-is-priced-as-a-hedge.md new file mode 100644 index 00000000..d2dc43f3 --- /dev/null +++ b/docs/scenarios/issue-drafts/any-is-priced-as-a-hedge.md @@ -0,0 +1,121 @@ +# Issue draft — `any()` is named for failover and priced as a hedge, and a per-call header reaches every member + +**Status:** ✅ **FILED** as [#643](https://github.com/rejifald/StitchAPI/issues/643). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`provider-failover`](../provider-failover.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `pipe`, `docs`, `footgun` + +> Two findings. The first is a naming/pricing mismatch with a real bill attached. The second is +> a **cross-vendor credential leak** and is the one I would fix first. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/provider-failover/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. A per-call header is broadcast to every member + +**Severity: high — one vendor receives another vendor's credential.** + +`runMember` (`pipe.ts:75-86`) spreads one `StitchInput` across every member. Everything the +stitch _declares_ stays per-member (verified: `Bearer pk-primary` and `x-api-key: sk-backup` +never crossed). Everything the **caller passes** is broadcast. + +Measured: a per-call `headers: { authorization: 'Bearer per-call-primary-jwt' }`, written for +the primary, **arrived at the backup verbatim**. Config headers merge _under_ input headers +(`engine.ts:232`), and each auth strategy only overwrites its own header name — so nothing +reclaims it. No type error, no warning. + +The same broadcast sends the primary's `model` to the backup, and a member whose template param +wasn't supplied is not an error: `/v1/{deployment}/complete` with no `deployment` expanded to +`/v1//complete` (`util.ts:453-482`, RFC 6570 drops undefined vars), the provider 404'd, and +**the call still succeeded** because the other member answered — a silent misconfiguration that +presents as a permanently-degraded-but-green failover. + +**Ask:** at minimum document that call input is broadcast and per-member credentials must live +in `auth`. Better: a per-member input shaper (`any([{ node, input }])` or a mapping function), +which would also fix the `model`/template-param cases. + +## 2. `any()` calls every member on every call + +**Severity: medium-high — silent double spend, and every test passes.** + +The docstring (`pipe.ts:274-291`) reads as fallback — _"failover across interchangeable +sources… a primary and a mirror, two regions, two providers"_ — while the first line says +CONCURRENTLY and the implementation is `Promise.any` over eagerly-started members +(`pipe.ts:148-152`). + +Measured: **10 calls in which the primary succeeded every time cost 20 provider requests.** The +same providers under `try`/`catch`: `[10, 0]`. Against a metered API that is the backup vendor's +entire bill, on the happy path. + +Three compounding details: + +- **"The losers are auto-cancelled" reads as "the losers are free", and is neither.** The abort + is raised in a `finally` _after_ the winner settles (`pipe.ts:154-156`), so the request always + arrives — 10/10 backup requests measured `completed`, **0 aborted**. When the loser _is_ + cancelled it is still billed for the winner's latency (100 ms loser vs 40 ms winner → 40 ms + billed). Two equally-fast providers → **80 ms of work for one answer, 0 saved**. The better + the backup, the less the cancel saves. +- **`any` has no preferred member.** A healthy primary that was 10 ms slower _lost_ — winner + measured `served_by: 'backup'`, healthy primary aborted mid-flight. Member order carries no + priority, so the construction silently routes away from the provider you chose. +- **Hedging a POST is a correctness bug the types won't catch.** `race` over a POST delivered + the identical `{ charge: { amount: 4200 } }` to **both** providers; nothing in the combinators + inspects `method`. + +**Ask:** the behaviour is defensible — it is a hedge, and hedges are useful. The problem is that +the docstring sells it as the _other_ technique. Either +(a) rewrite the docstring to lead with the cost ("every member is called on every call; this is +hedging, not fallback — for fallback use `linked` + `try`/`catch`"), or +(b) add a genuine sequential combinator and point `any`'s docs at it. A `hedge({ after })` +variant would also close the delayed-hedge gap in §3. + +## 3. Gaps this scenario ran into + +- **No sequential-fallback combinator.** `all`/`any`/`race` are one eager implementation with + three joins (each measured `[1, 1]` on one call). `linked` + `try`/`catch` is the correct + default and works — measured `[10, 0]` healthy, correct failover on 503, **one traceId** with + a `primary ← root, backup ← primary` spine (a bare `try`/`catch` gives two unrelated root + traces). But `linked` returns a **Promise, not a `Composable`** (`pipe.ts:357-369`), so the + flow runs once at the point of definition and cannot be nested in a combinator, handed to a + seam, or introspected. +- **No classification for routing.** `retry.on` is exactly the right vocabulary aimed at the + wrong target (the same endpoint). Of five declarative spellings probed, only `retry.on` and + `verdict.accept` compile. +- **`AggregateError` drops `status` and `body`.** `Promise.any` (`pipe.ts:152`) replaces the + `StitchError` the engine populated (`types.ts:1657-1691`), so every field a catch block routes + on is gone; the actionable 400 survives only in `.errors[0]`, which no `StitchError` API points + at. Interestingly `race` _does_ surface a real `StitchError` 400 with body — and is unusable as + failover, since a 500 primary against a healthy backup also yields 500. +- **No winner identity, and no group span.** `any` resolves to the raw body with no envelope or + index; the per-stitch `pick` that normalises two envelopes destroys the only attribution; and + the group emits **zero** events (`makeComposable`, `pipe.ts:210-220`, is not a span). Note + `all` accepts a **named bag** and returns a keyed object — the one combinator whose semantics + never need member names is the only one that carries them. +- **A cancelled member emits nothing terminal** — `start`, `progress`, then silence; 0 `error`, + 0 `done`, because the cancellation rejects outside the engine (`swallowLateRejections`, + `pipe.ts:90-92`). A span-based backend reads that as a leak or a timeout. +- **No hedge threshold at any level.** `race` measured **2.00×** amplification healthy _and_ + degraded. A breaker can't bound it either — it is a health gate, not a budget gate (10 healthy + calls with `circuit` on both members still measured `[10, 10]`). +- **`Composable` is not user-authorable.** `makeComposable` is unexported and the member gate + (`pipe.ts:188-189`) checks only the brand — a hand-branded node **compiles** and then throws + `TypeError`. So the 30 lines of routing this scenario needed cannot be given back to the + library as a node. + +## 4. A second sighting of a known issue + +Two `url`-only stitches have neither `name` nor `path`, so both key their breaker on the literal +string `'stitch'` (`resilience.ts:353`, `engine.ts:140,265-274,860`). Measured: **one** key +`circuit:stitch` for the pair, and the primary's outage opened the **backup's** breaker — +outcomes `ok, ok, AggregateError, AggregateError, AggregateError`, with the healthy backup +receiving only 2 of 5 requests and the caller's error carrying `status: undefined`, so nothing +even said "circuit open". Setting `name` fixes it (5/5 ok). + +This is the same root cause as [`resilience-has-no-tenancy`](resilience-has-no-tenancy.md), +reached from a different direction — the breaker's partition key is a **diagnostic label** +anyone might "clean up", and there is no warning when two unrelated stitches collide on it. +Worth folding into that issue's fix: a default key that cannot silently collide. diff --git a/docs/scenarios/issue-drafts/bigint-in-params-vanishes.md b/docs/scenarios/issue-drafts/bigint-in-params-vanishes.md new file mode 100644 index 00000000..8392cba7 --- /dev/null +++ b/docs/scenarios/issue-drafts/bigint-in-params-vanishes.md @@ -0,0 +1,81 @@ +# A `bigint` in `params` silently vanishes, while `query` handles it exactly + +**Status:** drafted, not filed +**Scenario:** [precision-loss](../precision-loss.md) +**Proofs:** `docs/scenarios/proofs/precision-loss/` (8 scripts, 191 checks, offline) + +## 1. BUG — the two URL positions disagree, and one of them loses data silently + +Surveyed all four outbound positions with the same `1234567890123456789n`: + +| position | bigint outcome | +| ------------ | -------------------------------------------------------------------------------------- | +| `query` | **exact** — `?since=1234567890123456789` | +| `form` body | **exact** — `id=1234567890123456789` | +| JSON `body` | **throws** `Do not know how to serialize a BigInt`, no request made — loud and correct | +| **`params`** | **vanishes** — URL becomes `https://api.vendor.test/v1/things/`, no error, no event | + +`expandTemplateVar` (`packages/core/src/util.ts:392`) branches on +`string | number | boolean`: + +```ts +if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' +) { +``` + +A `bigint` falls through, and `Object.entries()` is `[]`, so the segment expands to +nothing. The sibling `query` walker (`stringifyLeaf`, `util.ts:332`) **does** list `bigint`. + +The asymmetry is the bug: the same value in the same URL is exact in one slot and gone in the +other, and the failing one fails silently. A request to `/v1/things/` is a request for a +different resource — likely a list endpoint — not an error. + +**Ask:** add `bigint` to `expandTemplateVar`'s branch, matching `stringifyLeaf`. If a bigint path +param is genuinely unsupported, throw rather than emit an empty segment. + +## 2. Why this slot in particular + +`params` is where an ID goes. And IDs above 2⁵³ are exactly where `bigint` is the correct type — +so the one slot that drops bigints is the one that receives them. + +The end-to-end shape, measured in eight lines: read an id from a vendor response, hand it +straight back as a path param, and the request goes to `/v1/things/1234567890123456800`. That is +a **third** distinct digit string — not the `…789` the vendor sent, and not the `…768` a debugger +shows you — because `JSON.parse` rounds on the way in and `String(number)` renders the +shortest round-trip form on the way out. This is the well-known +[Discord `Unknown Channel`](https://github.com/openclaw/openclaw/issues/23170) shape, and nothing +in the event spine reports it. + +## 3. Smaller, same area + +- **`JSON.stringify(report)` throws under a bigint body** with `Do not know how to serialize a +BigInt`. `.report()` is documented as safe to log, so a diagnostic added while debugging a + precision problem is itself a crash. (`trace.ts` gets this right — it ships a `bigintSafe` + replacer explicitly so tracing cannot break the call it observes. `.report()` could borrow it.) +- **A JSON-serialising `store` throws on the write and the throw is fatal** (`ok: false`), not a + degraded cache miss. `memoryStore` survives, holding values by reference. +- **`wire.response` and `transform` are independent keys.** Setting `wire: { response: 'text' }` + and forgetting `transform` silently returns a **string** where an object is expected — no + throw. They are independent at the type level too: `transform` is `(body: unknown) => unknown`, + so a parser written `(text: string)` does not typecheck in the slot even though + `wire.response: 'text'` guarantees a string at runtime. A narrowed `transform` signature under + `wire.response: 'text'`, or a warning when one is set without the other, would close it. +- **`wire.response` has no guide page.** It is the one config key that recovers a corrupted ID, + and it appears in the docs only in passing — in the GraphQL guide's list of what `wire.body` + does _not_ do. Worth documenting on its own, with this scenario as the motivating example. + +## Not a bug, recorded so a fix doesn't chase it + +`JSON.parse` corrupting integers above 2⁵³ is JavaScript, not StitchAPI, and the library's own +default (`http-adapter.ts:135`) is the correct default. The finding worth acting on is only that +**nothing reports it** — the spine is four events with zero drift, zero error and zero info — and +that `wire: { response: 'text' }` already solves it for anyone who knows the key exists. + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +filing. Runnable proof scripts live under `docs/scenarios/proofs/precision-loss/` on the branch +`claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/body-verdict-footguns.md b/docs/scenarios/issue-drafts/body-verdict-footguns.md new file mode 100644 index 00000000..f5453958 --- /dev/null +++ b/docs/scenarios/issue-drafts/body-verdict-footguns.md @@ -0,0 +1,101 @@ +# Issue draft — three silent failures when the failure signal lives in the response body + +**Status:** ✅ **FILED** as [#651](https://github.com/rejifald/StitchAPI/issues/651) +**Scenario:** [`cost-based-rate-limits`](../cost-based-rate-limits.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `resilience`, `footgun` + +> The scenario came out **achievable** — a custom `Surface` closes it in 73 lines +> (`proofs/cost-based-rate-limits/c6`). These three findings are separate: each one +> typechecks, looks right, and silently does the wrong thing. Ordered by blast radius. + +All three were measured offline. Reproduce with: + +```bash +for f in docs/scenarios/proofs/cost-based-rate-limits/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `verdict.flag` returns `ok: true` and hands the caller an error envelope + +**Severity: high — silent data corruption, no error anywhere.** + +`verdict.flag` is the one built-in that reads the **body** to decide success, so it is the +natural reach when an API reports failure in a 200. On a payload that omits the flagged path +entirely, an absent path is treated as "no signal", the 200 stands, and the call succeeds. + +Against a Shopify `THROTTLED` response — HTTP 200, `{ errors: [{ extensions: { code: +'THROTTLED' } }] }`, no `data` key — a stitch with `verdict: { flag: 'data.ok' }` returned +**`ok: true`** and handed the caller the THROTTLED envelope as its result. Downstream code +processes an error object as a successful sync. There is no throw, no drift finding, and +nothing in the trace that reads as wrong. + +Measured in `c1-retry-on-200-throttled.ts` (e). + +**Ask:** absent-vs-falsy should not be the same verdict. Either treat a missing flag path as +a failure (or a distinct `unknown`), or emit a drift/health signal when the path a verdict +depends on is not present in the body at all. The current behaviour is the least safe of the +three options and is not stated in the guide. + +## 2. ~~`.safe()` downgrades `RateLimitError` and drops the body~~ — FIXED + +**Severity: medium — an outer rate-gate backs off blind.** +**Status: ✅ fixed by [#662](https://github.com/rejifald/StitchAPI/pull/662).** + +With `throttle: { delegate: true }`, `await call()` throws a real `RateLimitError` whose +`.body` carries the payload — for Shopify, `extensions.cost.throttleStatus`, i.e. exactly the +numbers an outer gate needs to pace itself. + +`call.safe()` returned a plain `StitchError` where **`error.body` was `undefined`**. +`asStitchError` copied only message/status/cause, so the payload survived only at +`error.cause.body`. + +The delegate-backoff guide's whole premise is handing back-pressure to something outside the +stitch. A consumer that follows the codebase's own preference for `.safe()` over `try/catch` +read "there was no body" and lost the pacing information. + +Measured in `c4-delegate-on-200.ts` (b2). + +**Ask:** preserve `body` (and `retryAfter`) across `asStitchError`, or document that +`delegate` requires `try/catch` rather than `.safe()`. + +**Resolution.** Neither — the coercion itself was the bug. `asStitchError` existed only because +`RateLimitError` and `StitchError` were **siblings** while `SafeResult.error` is typed +`StitchError`, so the safe path had no choice but to downgrade. #662 makes `RateLimitError` +extend `StitchError`, so `.safe()` returns the very instance `await` throws: `instanceof`, +`body`, `retryAfter` and `response` all behave identically on both paths, and nothing hides on +`.cause`. The `(b2)` proof block now measures that — 19/19. + +One migration note for anyone branching on both classes: **test `RateLimitError` first**, since +a leading generic `instanceof StitchError` arm now catches it. + +## 3. `backoff` as a function typechecks-then-vanishes + +**Severity: low — but the failure is silence, not an error.** + +`backoff` accepts `BackoffCurve | AtLeastOne` — no function form. Passing +`backoff: () => 6000` is correctly a **type error**. But casting past it (which people do when +they believe a feature exists) does not throw: the function is **never invoked**, and delays +silently fall back to the default curve — measured gaps of `100, 200` ms where the intended +wait was `6000`. + +Measured in `c2-computed-wait.ts` (a2). + +**Ask:** throw at construction on an unusable `backoff` value, the way an unparseable +`throttle.rate` already does (`bad rate: …` is thrown at construction — good precedent, worth +matching). Silently degrading a resilience policy is the one place a fallback is worse than a +crash. + +--- + +## Context worth keeping + +The scenario that surfaced these is a body-reported quota, and the correct answer turned out +to be a custom `Surface` — `interpret` + `SurfaceOutcome.after` (`surface.ts:34-37`, honoured +at `engine.ts:785-805`). That path works well and is public. The gap is that **nothing points +there**: `throttle`'s own doc comment (`types.ts:1018-1019`) sends readers to `delegate` for +vendor-accounted quotas, and `delegate` is status-keyed, so it does not reach this case. Three +of the four wrong turns above are what a reader tries _before_ finding the surface seam. + +A pointer from the retry/throttle guides to "if the failure signal is in the body, write a +surface" would remove most of this class. diff --git a/docs/scenarios/issue-drafts/cache-cannot-revalidate.md b/docs/scenarios/issue-drafts/cache-cannot-revalidate.md new file mode 100644 index 00000000..e0d910f8 --- /dev/null +++ b/docs/scenarios/issue-drafts/cache-cannot-revalidate.md @@ -0,0 +1,106 @@ +# Issue draft — `cache` cannot do conditional requests, and a surface cannot key a store by principal + +**Status:** DRAFT — not filed. Raised by the scenario pass on 2026-08-05. +**Scenario:** [`conditional-requests-304`](../conditional-requests-304.md) +**Suggested template:** feature_request.yml · **Suggested labels:** `cache`, `enhancement` + +> Not a bug — a capability gap with a sharp edge. The scenario came out **achievable** (87 +> lines on `Surface.execute`), but the primitive that _looks_ like it should carry it cannot, +> and the safe way to write the replacement depends on information a surface isn't given. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/conditional-requests-304/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `cache` is a value store, so an ETag can never reach it + +`CacheEntry` is `{ v, s, vary }` (`cache.ts:300-304`) and what gets written is `out.value` +(`engine.ts:1624`) — the post-`interpret`, post-`transform`, post-validation value. No response +header survives that far, so there is nowhere to put an `ETag`, and no way to send +`If-None-Match` on the next call. + +Three consequences, all measured: + +- **A hit short-circuits everything below the lookup.** Over 3 calls: hooks fired once, + `interpret` ran once, one request reached the server. The hit spine is + `[start, cache:hit, result, done]` — no `request` phase (`engine.ts:1613-1617` returns before + `runFrom`). So revalidation cannot run _under_ a hit even if you wrote it. +- **The cache cannot store a 304.** Forced into its own key via `vary`, three conditional calls + measured `[undefined, undefined, undefined]` with statuses `[200,304,304,304]` and 4 network + requests — `op.set` writes `{ v: undefined }` and `cache.ts:482` reads that as a permanent miss. +- **The one workaround disables caching.** Folding `{ etag, body }` into the value via + `transform` makes the stitch un-fingerprintable, so ADR 0004 fails closed: + `bypass: opaque transform without cache.transformVersion or trustTransform`. + +**`revalidateOnHit` is a name collision worth fixing.** It re-checks the stored value against +the `output` **schema** (`engine.ts:1604`), never the network. A reader looking for conditional +requests finds this option first and it does something else entirely. + +**Ask:** either a `cache.revalidate` mode that stores the validator with the entry and issues +`If-None-Match` on a stale hit, or — cheaper — a documented statement that `cache` is a TTL +value cache and conditional requests belong in a surface, with a pointer to the pattern. +Renaming or aliasing `revalidateOnHit` would remove the collision either way. + +## 2. A surface cannot see the bound principal, which is what makes a user-written ETag store leak + +The correct key for an ETag store is _credential-scoped_ — GitHub caches ETags per token, so a +store keyed on URL alone replays one principal's validator for another. + +The information needed to do that is not available where you'd write it: + +- `ResolvedStitchConfig` carries **no `principal`** — it lives on `AuthContext` (`engine.ts:1032`). +- `Surface.buildRequest` runs at `engine.ts:253`, **before** `cfg.auth.apply` at `:649`, so it + cannot even read the credential _header_. Measured `absent` in both positions. +- Only `hooks.onRequest` (`:652`) and `Surface.execute` (`:666`) are downstream of auth — both + measured `Bearer tok-alice`. + +So the only safe key is parsed back out of the `Authorization` header by hand. + +**Measured leak:** against a server with content-derived validators, a store keyed on +`METHOD URL` produced one store entry and **bob receiving `viewer: tok-alice`** — +`[tok-alice|(none)→200, tok-bob|"v1"→304]`. `cache.tenancy: 'principal'` protects the built-in +cache (verified: 2 requests, 2 distinct keys) but knows nothing about a user-written store. + +The reason this is worth flagging rather than filing under "user error": **the rate-limit +metrics improve while it happens.** A 50% 304 rate is exactly what a correctly working +revalidator looks like, so the leak is invisible in exactly the dashboard you'd check. + +**Ask:** expose the bound principal to a surface (on `ResolvedStitchConfig`, or as a field on +whatever context `execute` receives). One term in a key expression is the whole fix; today it +requires re-parsing a header the engine already resolved. + +## 3. Smaller edges from the same verification + +- **`interpret` DOES run on non-2xx** — measured with a counter across `[200, 304, 404]`. Worth + stating in the surfaces reference, because the neighbouring streaming path does _not_ call it + (see [`sse-reconnect-replays-completed-streams`](sse-reconnect-replays-completed-streams.md)), + and the asymmetry is currently undocumented. +- **`buildRequest` runs once per run, not per attempt** (`engine.ts:253`, outside `attemptLoop`). + A validator set there is baked into every `cloneReq` — measured 3 identical validators across + 3 attempts and a run that then failed, where `hooks.onRequest` recovers by dropping the header + (`["v1.t1","(none)"]` → `[304,200]`). +- **Request headers are never case-folded** (`engine.ts:232`, a plain spread). Measured: + `delete headers['If-None-Match']` does not remove a header set as `'if-none-match'`, and the + stale validator still went out. +- **`verdict.flag` on a 304 emits a spurious `info` drift finding on every unchanged poll** — + noise on the hot path. +- **Adding an `output` schema breaks a working bare conditional poll**: the empty body fails the + contract (`ok: false`, `contract violation (drift)`). Correct once substitution is in place, + but the failure mode is "adding validation broke my polling". + +## 4. A second clock finding, companion to the existing one + +**`cache.ttl` does not honour an injected `clock`** — `memoryStore` reads `Date.now()` +(`store.ts:16,45`, `util.ts:4`). Measured: one request after advancing a `manualClock` by a +virtual hour against a 1-second TTL, so a cache-expiry test on virtual time passes vacuously. + +This is the same shape as the `timeout.total` finding in +[`clock-and-diagnostic-side-effects`](clock-and-diagnostic-side-effects.md), and the two +together suggest a general audit is worth more than two point fixes: **which time-driven +features read the injected clock, and which read `Date.now()`?** Whatever the answer, the +testing guide should list it, because right now a `manualClock` test of either feature is green +and meaningless. diff --git a/docs/scenarios/issue-drafts/clock-and-diagnostic-side-effects.md b/docs/scenarios/issue-drafts/clock-and-diagnostic-side-effects.md new file mode 100644 index 00000000..2477a390 --- /dev/null +++ b/docs/scenarios/issue-drafts/clock-and-diagnostic-side-effects.md @@ -0,0 +1,91 @@ +# Issue draft — `timeout.total` is untestable with `manualClock`, and `.inspect()`/`.report()` re-issue the request + +**Status:** ✅ **FILED** as [#652](https://github.com/rejifald/StitchAPI/issues/652) +**Scenario:** [`async-job-polling`](../async-job-polling.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `bug`, `testing`, `dx` + +> Two independent findings, both in the "the tool tells you something untrue" class. The first +> makes a green test meaningless; the second gives a diagnostic method a side effect. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/async-job-polling/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `timeout.total` goes silent under an injected clock — so its own test passes vacuously + +**Severity: high — a green test proving nothing.** + +`timeout.total` is compared against **wall-clock** time (`engine.ts:453-488`, and the comment +at `:482` says so explicitly: _"`timeout.total` stays on wall-clock"_), while every sleep in +the attempt loop runs on the **injected clock**. + +Inject a `manualClock()` and the two disagree completely. Measured: **60 polls across 59 +virtual seconds under `timeout: { total: '10s' }` never tripped the budget.** No error, no +event — the deadline simply never fires, because no wall-clock time passed. + +This matters because [the testing guide](https://stitchapi.dev/docs/guides/testing/mocking) +recommends `manualClock` for exactly this class of behaviour: _"Retry backoff, throttle +pacing, and the per-attempt timeout are time-driven, so testing them used to mean real +waiting."_ `timeout.total` is time-driven and is **not** in that list — and a reader who +assumes it is writes a test for "give up after an hour" that passes without ever exercising +the deadline. + +The scenario that surfaced it is the honest motivation: an hour-long job budget can only be +tested on a virtual clock. Today that test is guaranteed green and guaranteed meaningless. + +**Ask:** put `timeout.total` on the injected clock, the way the sleeps already are. If +wall-clock is deliberate (a defensible choice — a virtual clock shouldn't let a real request +hang forever), then say so in the timeout guide _and_ in the testing guide's list, and +consider a warning when a `manualClock` and a `timeout.total` are configured on the same +stitch. Silence is the one option that can't be right. + +## 2. `.inspect()` and `.report()` issue a fresh request — so they duplicate side effects + +**Severity: high on non-idempotent stitches — measured, duplicate jobs.** + +`.inspect()` and `.report()` read as diagnostics over a call that already happened. They are +not: each one performs the request again. + +On a `GET` that is wasteful. On the `POST /jobs` that starts an async job it is a bug you pay +for in someone else's system. Measured: **one `.safe()` plus one `.inspect()` plus one +`.report()` submitted three jobs.** Two of them are orphans — nothing polls them, and they run +to completion server-side, consuming quota. + +The naming is the whole problem. Nothing about "inspect" or "report" suggests a network call, +and the obvious debugging move on a failing submit — call `.inspect()` to see what came back — +is precisely the move that submits another job. + +**Ask:** at minimum, document it prominently on both methods and in the errors/pitfalls page. +Better: have them replay the _last_ result when one exists, or refuse on a non-`GET` stitch +unless explicitly opted in (`inspect({ reissue: true })`). + +--- + +## Smaller findings from the same verification + +- **`after: res.headers['retry-after']` polls 1000× too fast.** `Retry-After` is + delta-**seconds**; `SurfaceOutcome.after` reads a bare numeric string as **milliseconds**. + Measured gaps of `30 ms` against a requested `30 s`. The correct spelling is + `after: \`${raw}s\``. Two units, one field, no type friction — worth a line in the surface +docs, since a surface author reading `Retry-After` is the _expected_ use. +- **`parseRetryAfter` is not exported**, so every surface author must re-implement HTTP-date + parsing. Without it the date form silently falls through to the computed curve — measured + `7,7,7` ms where the server asked for 30 s. The engine already has this parser for the + status-driven path; exporting it would make the body-driven path correct by default. +- **`retry.respect` does not reach the body-driven retry path at all** (`engine.ts:797-801` + reads only `parseDuration(outcome.after)`; `:749-751` is where the header is read). Arguably + by design, but a user who sets `respect: true` and gets the computed backoff has no signal + that the setting is inert on that path. +- **`cache: { methods: ['POST'] }` as restart-safety orphans the job.** It does prevent the + duplicate POST (measured: 1 submit across two processes), but the cached value of a `202` is + `{}` and **a cache hit never fires `onResponse`** — so the `Location` is unrecoverable and + the job is lost. This is a plausible thing to try; it half-works, which is the dangerous + amount. +- **A surface that omits `verdictOf` returns HTTP errors as successes** — measured: a 404 came + back `ok: true` with `data: { message: 'unknown job …' }`. Every scenario in this pass that + wrote a surface had to remember `verdictOf` first. If composing the declarative verdict is + mandatory for correctness, consider doing it in the engine rather than by convention. diff --git a/docs/scenarios/issue-drafts/coalescing-does-not-share-failures.md b/docs/scenarios/issue-drafts/coalescing-does-not-share-failures.md new file mode 100644 index 00000000..fd5a0cd4 --- /dev/null +++ b/docs/scenarios/issue-drafts/coalescing-does-not-share-failures.md @@ -0,0 +1,126 @@ +# Issue draft — a coalesced failure is not shared, and a `store` silently un-pools `pool: 'host'` + +**Status:** ✅ **FILED** as [#653](https://github.com/rejifald/StitchAPI/issues/653) +**Scenario:** [`n-plus-one-fanout`](../n-plus-one-fanout.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `cache`, `throttle`, `enhancement` + +> **Leading with the strongest positive result of the pass.** `cache.coalesce` genuinely +> collapses **in-flight** duplicates: 100 concurrent calls over 30 distinct ids made **30 +> requests** — one per id, exactly the floor — with every call in flight and **no response +> landed**, from a single `cache: { ttl }` block. 70 callers were served without a request of +> their own; `coalesce: false` on the same cache made 100. Most clients do not have this, and the +> N+1 fan-out is precisely the shape it fixes. +> +> Two findings sit beside it, and one is a silent un-fix. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/n-plus-one-fanout/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. A coalesced failure releases every joiner to re-run + +**Severity: medium — it costs a wasted request per duplicate reference to a broken id.** + +A leader that _succeeds_ serves its joiners. A leader that _fails_ releases them +(`engine.ts:1646-1649`, `:1656-1659`), and each re-runs the whole chain independently. + +Measured: 100 concurrent calls for one id that 404s made **100 requests in two waves** — 1 +leader, then 99 followers. Every joiner got its own honest `HTTP 404` (never a leader artefact), +which is the right _error_ at the wrong _price_: the correct diagnosis is bought by asking the +vendor 100 times for a resource that does not exist. + +End to end this is the only place the hand-rolled control beats the library — **44 customer +requests against 32**, and a deleted customer cost **4 requests against 1**. A plain +`Map` shares the rejection. This is exactly the shape a dead foreign key takes, and +it multiplies with how many rows reference it. + +**And coalescing does not protect a herd.** With 100 calls over 20 ids and only the 20 leaders +429'd, the run made **100 requests** — 80 followers re-fanned at full width. The cohort that +just tripped the limit is exactly the cohort that fans back out. + +**Ask:** share the leader's rejection with its joiners, at least for a short window, or offer +`coalesce: { shareFailures: true }`. If failures must not be shared (a defensible position — a +transient failure shouldn't be broadcast), a negative-cache window would cover the two cases that +actually recur: a 404 that will stay a 404, and a 429 the whole cohort just caused. + +## 2. A `store` silently un-pools `pool: 'host'` concurrency + +**Severity: medium-high — a declared bound of 8 measured a peak of 100, config unchanged.** + +Three constructions, one declared budget of `concurrency: 8`: + +| construction | peak in-flight | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| one stitch called 100 times | **8** ✅ | +| 100 separate stitches, each declaring 8 | **100** — the limiter is per stitch (`stitch.ts:985-988` over closure-local state, `resilience.ts:107`) | +| 100 stitches + `pool: 'host'` | **8** ✅ (module-level `hostStates`, `resilience.ts:84`) | +| 100 stitches + `pool: 'host'` **+ a `store`** | **100** ❌ | + +`createStoreThrottle` never reads `opts.pool` and keeps `inFlight` in a closure-local Map +(`store.ts:137-151`). The store is exactly what you add to make the **rate** budget +cross-process, and it un-pools the **concurrency** on the way past. + +The 100-stitch shape isn't hypothetical: it is the only construction that lets `all()` express a +per-id fan-out at all (see §4), so the two findings compose into "the way you're pushed to write +it is the way the bound stops working." + +**Ask:** honour `pool` in the store-backed throttle, or reject `pool: 'host'` + `store` at +construction rather than accepting it and ignoring it. A seam-level `concurrency` does survive +both (`seam.ts:51-69`) and is worth documenting as the answer for a stitch-per-item shape. + +## 3. Two backoff/slot interactions worth documenting + +- **A backing-off call holds its concurrency slot.** The retry sleep sits inside the `try` the + release `finally` guards (`engine.ts:760-765`, `:834-837`). Measured with a bound of 4, a + 429'd first wave and a 1 s backoff: the fifth call left at **t=1050** rather than t=50, with + ~95% of the declared budget occupied by calls that were asleep and issuing nothing. The retry + then re-queues at the **back** of the FIFO — the first call's retry left at t=4200, behind + every other call's first attempt. +- **`Retry-After` defeats `expo-jitter` by default.** The default `expo-jitter` genuinely + de-clusters — 100 calls 429'd in one instant retried across **~98 distinct milliseconds**, + where `'fixed'` and `'expo'` both put all 100 in **one millisecond**. But `retry.respect` + defaults on and takes the header verbatim (`engine.ts:748-761`), so `Retry-After: 2` put all + 100 back into one millisecond at t=2000 with jitter still declared. `respect: false` restores + the spread and is all-or-nothing — there is no "honour the header, then jitter around it", + which is the behaviour a herd wants. + +## 4. The combinators can't express a per-id fan-out, and not for the obvious reason + +The runtime length is fine — `all(ids.map(…))` compiles, because `membersFrom` takes a plain +array (`pipe.ts:226-229`). **The input is the wall**: `runMember` spreads one `StitchInput` into +every member (`pipe.ts:75-86`), so 100 members called with one input made **100 requests for one +distinct id**, 99 of them waste. + +Of seven candidate spellings, **two** compile (`all(array)`, `all(a, b, c)`); `all(one, inputs)`, +`all.map`, `allSettled`, `.safe()` members and `all(members, { concurrency })` are all compile +errors. `Member` is brand-gated on `__stitch` (`pipe.ts:188-189`), so the suggestion to compose +`.safe()` members by hand does not typecheck. + +This is the third scenario to land on the input broadcast (7, 10, 16) and the second to want +`allSettled`. **Ask:** a per-member input — `all([{ node, input }])` or a mapping function — +would close all three at once. It is the same ask as +[`any-is-priced-as-a-hedge`](any-is-priced-as-a-hedge.md) §1, now with a second motivating shape. + +## 5. Footguns + +1. ‼ **`cache: { ttl: 0 }` caches forever.** It is the obvious spelling for "dedupe but don't + cache", and `expires === 0` reads as live (`store.ts:15-16,45`) — a later fan-out added **0 + requests**. There is no coalesce-only spelling. +2. ‼ **Coalesced and cached callers share one object by reference** (`engine.ts:1645`, `:1662`). + 20 rows over 5 customers gave **5 distinct objects**; mutating row 0 changed row 5, and a + cache hit aliases the same way for the whole TTL. Any normalise/enrich step that writes onto a + joined record writes onto every row sharing it — and the aliasing arrives _with_ the + optimisation. +3. **`sensitive: true` silently disables coalescing** (`engine.ts:1020`) — back to 100 requests + for 30 ids with the `cache` block still reading as if it coalesces. +4. **Coalescing is GET/HEAD only** — a POST lookup deduped nothing until `methods: 'POST'`. +5. **`cluster` silently degrades to `process`** (`cache.ts:396-397`). +6. **`verdict: { accept: [404] }` succeeds on a missing record**, handing the error envelope back + as `data` so the join writes a row with no name. **Fifth sighting** of this shape. +7. **`StitchError.url` comes from the transport** — an adapter that doesn't echo it leaves the + failing id unidentifiable from the error alone. The array index is the only identifier that + always holds. diff --git a/docs/scenarios/issue-drafts/diff-primitives-are-unreachable.md b/docs/scenarios/issue-drafts/diff-primitives-are-unreachable.md new file mode 100644 index 00000000..4d56e900 --- /dev/null +++ b/docs/scenarios/issue-drafts/diff-primitives-are-unreachable.md @@ -0,0 +1,107 @@ +# `diff` and `classifyDiff` exist and are unreachable, and `pool: 'host'` re-keys the breaker + +**Status:** drafted, not filed +**Scenario:** [dual-run-migration](../dual-run-migration.md) +**Proofs:** `docs/scenarios/proofs/dual-run-migration/` (8 scripts, 142 checks, offline) + +## 1. Two value-vs-value comparators ship in the tree and neither is exported + +Of the 33 root exports, exactly one is comparison-shaped: `drift()` — which takes a **schema** +and performs no value-vs-value comparison at all. Meanwhile the tree contains two functions that +do exactly that: + +- `diff(before, after)` — `packages/core/src/diff.ts:94` +- `classifyDiff(a, b, opts)` — `packages/core/src/drift.ts:113` + +Neither is reachable from any of the **17 subpaths** in the package `exports` map. + +Any user comparing two responses — a v1/v2 migration diff, a cache-vs-live check, a +before-and-after on a write — has to hand-write a walker that already exists twice. + +**Ask:** export one of them, most usefully `diff`. It is a small surface addition and it removes +a whole category of hand-rolled walkers. + +**One caveat so a fix aims correctly:** for this use case `classifyDiff` is the _worse_ of the +two, because it is built for schema drift and reports **kinds, not values**. It renders a planted +regression (`balance_cents` 1200 → 1500) as the detail `"number -> number"` — a type delta with +no numbers in it. Measured, a 23-line hand-written comparator was strictly better precisely +because it carries both values. `diff` is the one worth exporting. + +## 2. `DriftOptions.ignore` is suppression, not relevancy + +`ignore` (path, `*`, prefix, `[]` for array elements) is the only declarative filtering surface, +and it is reachable only through the source-only `classifyDiff`. On a realistic v1→v2 change — a +rename, a retype, an array reorder, a new field, and one planted regression — it took 7 diff ops +to 1 with four clauses. + +But it silences by **path**, not by reason. Measured, both ways: + +- the clause silencing a benign **tag reorder** also silences a real tag **change**; +- the clause silencing the `created` → `created_at` **rename** also silences a v2 reporting the + **wrong instant**. + +There is no field aliasing, no unordered-array comparison, no coercion hook and no numeric +tolerance anywhere in the tree. So the relevancy model that the shadow-testing literature calls +the core of the technique is 24 lines of user code, and the declarative option that looks like it +does the job quietly hides real regressions. + +**Ask:** either document `ignore` as _suppression by path_ with that caveat stated, or grow the +comparison surface an alias/unordered/tolerance clause. + +## 3. `pool: 'host'` fixes cost accounting and silently re-keys the breaker + +Two stitches against the same vendor, each with `throttle: '20/s'`, put **4 requests through in +one gap** — the vendor saw 40/s. Two constructions fix it, and they are **not** equivalent: + +| construction | throttle | circuit | +| ------------------- | ------------- | ---------------------------------- | +| seam-level throttle | one bucket ✅ | left keyed per path ✅ | +| `pool: 'host'` | one bucket ✅ | **also re-keyed onto the host** ⚠️ | + +`seamBucket` (`seam.ts:58`) returns the inner throttle unchanged when `pool: 'host'` is set — +`// host key already pools across the seam` — and the host key then applies to the circuit too. +Measured: under `pool: 'host'` a failing shadow stitch fast-failed the **primary**, which got +**0** requests. + +So the one setting a reader reaches for to make two stitches share a rate budget also makes them +share a breaker, and nothing says so. + +**Ask:** note the coupling in the `throttle.pool` docs, or let `circuit.key` override it +independently. + +## 4. A construction-time method gate is bypassable by a surface + +Not a library bug so much as a trap worth documenting, found while building a guard that prevents +shadowing a write. + +A gate written against `__config.method` refuses a plain `POST` stitch — and the `llm` surface +reports `__config.method === undefined`, passes the gate, and then POSTs. Measured. The same +holds for any surface that supplies its own method. + +The construction that actually works is an `Adapter` wrapper, which sits below every authoring +surface: 3 shadow write attempts (a plain POST, an `llm`-surface call, a `.with()`-bound handle) +reached the wire **0 times**. + +**Ask:** a line in the surfaces reference noting that `__config.method` is not a reliable +predicate for "what HTTP method will this send", and that the `Adapter` is the seam below all +surfaces. + +## 5. Smaller + +- **`linked` returns a `Promise`, not a `Composable`**, so it cannot be a member of `all`/`any`. + That is a reasonable design, and it is not stated anywhere the combinator docs would show it. +- **`.with()` binds a constant through the broadcast.** A bound partial _does_ survive + `runMember` — but a group built once and called twice sent the shadow to the customer from the + **first** call. This is the near-miss workaround for + [#643](https://github.com/rejifald/StitchAPI/issues/643) and it fails silently. +- **A bare `void stitch(input)` sends zero requests**, because `StitchResult` extends + `PromiseLike` (`types.ts:1962`) and nothing runs until `.then`. Already filed as + [#660](https://github.com/rejifald/StitchAPI/issues/660); recording the second sighting because + a fire-and-forget shadow is exactly the shape that hits it, and there the symptom is "the + comparison silently never ran." + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +drafting. Runnable proof scripts live under `docs/scenarios/proofs/dual-run-migration/` on the +branch `claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/drift-cannot-grade-a-coercion.md b/docs/scenarios/issue-drafts/drift-cannot-grade-a-coercion.md new file mode 100644 index 00000000..0bb76ca4 --- /dev/null +++ b/docs/scenarios/issue-drafts/drift-cannot-grade-a-coercion.md @@ -0,0 +1,123 @@ +# Issue draft — a `coerced` finding can't say whether the coercion was destructive, and the nullable class has no level + +**Status:** ✅ **FILED** as [#654](https://github.com/rejifald/StitchAPI/issues/654) +**Scenario:** [`intermittent-drift`](../intermittent-drift.md) +**Suggested template:** feature_request.yml · **Suggested labels:** `drift`, `validation`, `enhancement` + +> **Leading with what held up**, because this is the flagship feature and it tested well. The +> default is safe: `z.number()` on `"12345"` fails the call, and `z.coerce.number()` on `"abc"` +> fails too because Zod rejects `NaN`. StitchAPI does not manufacture a $0 charge on its own, +> and the capture's fear that it might is **refuted**. Precision is excellent — 5 findings on +> exactly the 5 drifting calls out of 100, each naming the field. And aggregation, which the +> capture predicted would be missing, **works**: `trace` + `ctx.spanId` measured a canary +> widening 5.0% → 25.0%. +> +> Two gaps survive that, and one footgun sits next to them. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/intermittent-drift/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. A `coerced` finding cannot distinguish a harmless coercion from a destructive one + +**Severity: medium-high — the finding is emitted, and it is not enough to act on.** + +`detail` is built from `kindOf(old) -> kindOf(new)` (`drift.ts:77-83`), so the values never +appear. Measured, on the same field: + +| wire value | caller receives | finding | +| ------------------------ | --------------------- | ------------------------------------------------------- | +| `"12345"` | `12345` — correct | `warn \| coerced \| transaction_id \| string -> number` | +| `"abc"` with `.catch(0)` | **`0`** — a $0 charge | `warn \| coerced \| transaction_id \| string -> number` | + +**Byte-identical.** A sink, an alert rule, or a human reading the trace cannot tell the benign +row from the catastrophic one. The only way to separate them measured in this scenario is to +join the `drift` event to the `result` event on `ctx.spanId` inside a custom sink — which is +exactly the work the finding was supposed to save. + +**Ask:** carry the values (or a redacted/typed summary of them) on a `coerced` finding, or add a +sub-kind distinguishing a _lossless_ coercion (`"12345" → 12345`, round-trips) from a _lossy_ +one (`null → 0`, `"abc" → 0` via `.catch`). The information exists at the moment the finding is +built; it just isn't kept. + +## 2. "Nullable is a warning, value intact" has no spelling + +**Severity: medium — the fourth industry change class is inexpressible.** + +The published taxonomies all treat a field _becoming nullable_ as warning-level: not breaking, +but worth knowing. Measured against one null, four declarations, four answers — and none of them +is that: + +| declaration | result | +| ---------------------------- | ----------------------------------------------------------------------------------------------- | +| required | `error \| invalid` + **failed call** (and the four fields that were fine are discarded with it) | +| `.nullable()` / `.nullish()` | **nothing at all** — declared variance, so the 5% rollout is completely invisible | +| `.catch('')` | `warn \| coerced` + a **fabricated `""`** the vendor never sent | + +`.nullable()` is the schema a team actually ships for availability, and it is the one that hides +the rollout. The hand-rolled classifier written for the comparison **beats `DriftOptions` on +exactly this row** — it levels a null `warn` and passes the value through. + +**Ask:** a `nullable` change kind (or a `DriftOptions` mode) that reports a newly-null field at +`warn` while passing the value through unchanged. + +## 3. `severity` is keyed by mechanism, not by change class + +`DriftSeverity` takes `undeclared` / `coerced` / `defaulted` (`types.ts:71-74`). Of the four +industry change classes, only **addition** maps 1:1 (`undeclared`). Removal, type change and +nullability each land on a kind decided by _your schema_, so their loudness is a schema decision +rather than a severity one — which makes "removal is fatal" something you have to have already +declared rather than something you can configure. + +Two smaller limits alongside it, both measured: + +- **No per-path severity.** `ignore` is the only path-aware lever and it is on/off + (`drift.ts:90-101` never sees the path). "Coercion on `transaction_id` pages, coercion on + `description` doesn't" has no spelling. +- **A soft finding can't be promoted to fatal through the type** — `error` isn't in + `DriftSeverity` (machine-checked with `@ts-expect-error`), though a cast past it does fail the + call at runtime, which is an odd pairing. + +## 4. Soft findings are invisible on the awaited path + +`await` and `.safe()` carry **nothing** for a soft finding — the caller gets `{ok, data, error}` +and, in the bad case, a `0`. `StitchError` has no `findings`, so even a hard failure gives only +the generic `contract violation (drift)`. The trace sink for the _same run_ named the field and +both types. + +So **`drift()` configured on a stitch that is only ever `.safe()`-ed does nothing for you** — +which is a plausible way to use it, and there is no signal that the feature is inert. + +**Ask:** put `findings` on `StitchError`, and/or document that `drift()` requires `trace`, +`.stream()` or `.inspect()` to be observable at all. + +## 5. Footguns (the plausible-but-wrong ones first) + +- ‼ **`z.coerce.number()` maps `null` → `0`** with no `.catch()` involved, because + `Number(null) === 0`. Measured: `null`, `""`, `" "`, `false` and `[]` all coerce to exactly + `0`; only `"abc"` rejects. On money this is the $0 transaction arriving through the front door. +- ‼ **`.catch(0)` is a $0-transaction generator** — it hands the caller `0` for anything. +- ‼ **`.default('usd')` on a _removed_ field fabricates a value** the vendor never sent, at + `verbose` — the quietest level. +- **A tolerant schema is a blind one.** `z.union([number, string])` and `z.unknown()` pass the + raw string through with **zero** findings. `.optional()` lets a field be deleted in total + silence. +- **Caching divides your drift rate by the miss ratio.** A cache hit emits `start` + `result` and + no drift (`engine.ts:1605-1613`), so 5 calls against a vendor drifting on **100%** of responses + measured **20%**. +- **Findings are not calls** — 2 findings on one response reads as 200% unless you collapse on + `ctx.spanId`. +- **`.report()` / `.inspect()` are fresh probes** — fourth sighting in this pass (see + [`clock-and-diagnostic-side-effects`](clock-and-diagnostic-side-effects.md)). Here `.report()` + called immediately after a drifting call reported **zero** findings, because the probe hit a + clean response — _and_ it added a request and a tick to the rate's denominator. +- **`severity` filtering deletes the data from the sink too** — it is an emission-time allowlist + (`drift.ts:147`), not a display filter, so a filtered finding never reaches a sink that might + have counted it. +- **A schema strips what it doesn't declare.** The engine serves the _validated_ value + (`engine.ts:1224`), so an added field is `undefined` on `data` — reachable only via + `.inspect().raw`, a fresh request. diff --git a/docs/scenarios/issue-drafts/hooks-can-rewrite-the-call.md b/docs/scenarios/issue-drafts/hooks-can-rewrite-the-call.md new file mode 100644 index 00000000..f9bae533 --- /dev/null +++ b/docs/scenarios/issue-drafts/hooks-can-rewrite-the-call.md @@ -0,0 +1,124 @@ +# Issue draft — `hooks.onResponse` can rewrite the call, and a seam-level `kind` is a compile error that works + +**Status:** ✅ **FILED** as [#655](https://github.com/rejifald/StitchAPI/issues/655) +**Scenario:** [`deprecation-headers`](../deprecation-headers.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `docs`, `hooks`, `types` + +> This scenario was run as a consolidation test — three earlier ones each hit "the header isn't +> on this accessor", and the goal was a definitive answer. **The answer is a positive**: +> response headers _are_ reachable on a successful call, in exactly three places, and +> `Surface.interpret` is the one seat where a header and the returned value are in scope +> together. The full table is in §4 and is worth putting in the surfaces reference. +> +> Three findings sit beside it. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/deprecation-headers/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `hooks.onResponse` can change what a stitch returns + +**Severity: medium-high — the guide states the opposite.** + +The hooks guide says hooks _"never change what a stitch returns"_. Measured, that is true of the +hook's **return value** and not of `ctx.res`, which is the engine's live `AdapterResponse` +(handed over at `engine.ts:705`, read again by `interpret` at `:775`): + +| mutation in `onResponse` | measured effect | +| ------------------------ | -------------------------------------------------------------------------- | +| `ctx.res.body.x = …` | the key appeared in the value the caller received | +| `ctx.res.status = 503` | the vendor's **200 became a thrown `HTTP 503`** | +| `ctx.res.headers[…] = …` | a downstream surface read `"REWRITTEN BY HOOK"` instead of the real header | + +Either the guide should say `ctx.res` is live and mutable — with the status case called out, +because turning a 200 into a throw is a big lever to find by accident — or `ctx.res` should be +frozen/cloned for hooks. The current pairing (documented as inert, actually a write channel) is +the worst of the two. + +Three further limits worth documenting alongside, all measured: `onResponse` fires **once per +attempt** (3 firings for one retried call, each carrying the same notice); `HookContext` has no +`emit`/`run`/`findings`, so nothing a hook learns can reach the event stream or the drift report; +and the only way out is a closure with no de-duplication of its own. + +## 2. A seam-level `kind` is a compile error that works perfectly at runtime + +**Severity: medium — it costs 40× the code for a capability that already works.** + +`seam({ kind: mySurface })` fails to typecheck — `TS2353: 'kind' does not exist in type +'SeamOptions'` — and the engine honours it anyway: members inherited the surface and behaved +correctly in the measurement. + +So a typed codebase writes the surface onto all forty members to get something one seam-level +declaration already delivers. + +**Ask:** either add `kind` to `SeamOptions` (it works), or make the runtime reject it so the +types and the behaviour agree. This is the same shape as scenario 13's +`stream({ kind })` **silently dropping** the surface — the two are opposite failures of the same +seam/`kind` relationship, and are probably worth fixing together. + +## 3. `parseRetryAfter` is exactly the function needed, and is unreachable + +**Severity: low-medium — the second sighting, and worse than the first.** + +`parseRetryAfter` (in `resilience.ts`) parses delta-seconds **or** an HTTP-date against an +injectable clock and returns ms-until. That is precisely the `Sunset` requirement — pointed at +the fleet's three real `Sunset` values it returned the right ms every time. + +It is not reachable: `resilience.ts` is not among the 17 published export subpaths, and +`index.ts` re-exports only `RateLimitError` from it. The barrel's three `parse*` helpers +(`parseDuration`, `parseBytes`, `parseRate`) do not parse dates —`parseDuration` returns +`undefined` for both an HTTP-date and a structured-field date. + +[Scenario 14](sigv4-ignores-the-injected-clock.md) raised this for a surface author reading +`Retry-After`; here the unexported helper isn't merely _similar_ to what's needed, it is +identical. + +**And the naive substitute is silently wrong:** `Date.parse("@1735689600")` is `NaN`, so a +client reaching for `Date.parse` reads `Sunset` correctly and reports **no deprecation** for the +format RFC 9745 actually mandates. + +**Ask:** export `parseRetryAfter` (or a `parseHttpDate`) from the barrel. + +## 4. The accessor → headers table, for the surfaces reference + +Measured across every accessor on a **successful** call. Worth publishing, because three +scenarios in this pass each rediscovered one row of it: + +| | carries response headers | +| -------------------------------------------- | ---------------------------------------------------- | +| `adapter` | yes — knows no stitch name, cannot change the result | +| `hooks.onResponse` | yes — full `AdapterResponse` + `ctx.name` | +| **`Surface.interpret(res, cfg)`** | **yes — and it returns the resolved value** | +| `await` / `.unwrap()` / `.safe()` | no | +| `.inspect()` (5 keys) / `.report()` (9 keys) | no | +| `StitchError` (5 keys) | no | +| `transform` | no — its one parameter is the body | +| the event spine — 4 events, 15 distinct keys | **no** | + +The last row has the consequence: **no event carries a header, so a `TraceSink` can only +aggregate what a `Surface` folded into the value.** Worth stating explicitly next to the trace +docs. + +## 5. Smaller findings + +- **No API mints a levelled finding.** A `Validator` returns a value or _issues_, and an issue is + `error | invalid` that fails the call. The only door onto the drift channel is folding a field + into the value and letting an undeclaring `output` report it `info | undeclared` — non-fatal + and re-levellable, but it carries **neither the value nor the endpoint**, and re-levelling is + per-_kind_, so raising a deprecation notice to `warn` also raised an unrelated new vendor field. +- **An `output` contract deletes a folded field** before the `result` event fires, with no + warning — so the surface-folds-it/sink-reads-it path breaks the moment someone adds a schema. +- **Nothing de-duplicates an observation.** 600 calls → **360 lines carrying 3 facts**; + `loggerSink` is louder at 2400. `levelOf` can drop an event but is a pure function of one + event, so it can reach 600 and not 3. `cache.coalesce` de-duplicates _requests_ and `throttle` + paces the _wire_; neither has anything to say about a repeated report. +- **`ctx.name` defaults to the literal `'stitch'`**, so two unnamed endpoints merge into one row + at a sink. (Third sighting of the `'stitch'` default causing a collision — see + [`resilience-has-no-tenancy`](resilience-has-no-tenancy.md) and + [`any-is-priced-as-a-hedge`](any-is-priced-as-a-hedge.md) §4.) +- **A cache hit re-serves a header captured on the one wire response** — 9 of 10 rows were a + replayed notice, so a long TTL will report a passed sunset as "in 12 days". diff --git a/docs/scenarios/issue-drafts/idempotency-default-is-not-restart-safe.md b/docs/scenarios/issue-drafts/idempotency-default-is-not-restart-safe.md new file mode 100644 index 00000000..427d1b67 --- /dev/null +++ b/docs/scenarios/issue-drafts/idempotency-default-is-not-restart-safe.md @@ -0,0 +1,128 @@ +# Issue draft — `idempotency: true` double-charges a re-driven job, and adding `retry` silences the warning + +**Status:** ✅ **FILED** as [#642](https://github.com/rejifald/StitchAPI/issues/642). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`unconfirmed-write`](../unconfirmed-write.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `idempotency`, `dx`, `money` + +> The highest-stakes finding of the pass, because the measurement is charges. Everything here is +> counted against a ledger the fake vendor owns, not inferred. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/unconfirmed-write/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. The default key is per call, so a re-driven job charges twice + +**Severity: high — measured duplicate charges with correct-looking config.** + +`engine.ts:172` mints `randomUUID()` when no `keyOf` is given, inside `buildRequest` +(`engine.ts:257`) — i.e. once per **call**, not per **intent**. + +Measured, simulating a queue re-driving a job after a crash with identical input: + +| config | distinct keys | charges | intended | +| ----------------------------- | ------------- | ------- | -------- | +| `idempotency: true` + `retry` | **2** | **2** | 1 | +| `idempotency: { keyOf }` | 1 | **1** | 1 | + +End to end across six workloads (crash-and-re-drive, lost response, TTL expiry, two concurrent +runs, a decline): the default produced **8 charges for 6 intended payments**, two of them +duplicates. A derived key plus a query-first recovery produced **5** (the sixth legitimately +declined). + +**And the guard is scoped narrower than readers will assume.** `stitch.ts:386` is: + +```ts +// A derived `keyOf` dedupes resubmissions on its own, so only the random default with no +// retry is the inert case. +if (idem.keyOf || cfg.retry) return; +``` + +The reasoning is sound on its own terms — a random key _does_ protect the retries inside one +call, which is what `retry` adds. But the practical effect is that **following the warning's own +advice ("add `retry`") silences it**, while the restart case it doesn't cover is the one that +costs money. The message says the config "only dedupes its own retries"; that sentence is +accurate and is exactly the limitation, so the fix may be as small as re-wording it. + +**Ask:** either warn whenever `keyOf` is absent regardless of `retry`, or re-word to name the +restart case explicitly — something like _"…only dedupes retries **within one process**; a +re-driven job will charge again. Set `idempotency.keyOf` to derive the key from the business +reference."_ Documenting `keyOf` as the default recommendation for writes would also do it. + +## 2. A dropped request and a lost response are indistinguishable + +**Severity: medium — inherent to timeouts, but the library discards what it does know.** + +The two cases with opposite ledger outcomes produced **field-for-field identical** errors: + +| | dropped request | lost response | +| ------------------------------ | -------------------------------------- | ------------- | +| charges on the vendor | **0** | **1** | +| `name` / `status` / `attempts` | `StitchError` / `undefined` / `1` | identical | +| `message` / `body` | `timed out after 5000ms` / `undefined` | identical | + +Nothing can be inferred from the error. Three things the library could keep and doesn't: + +- **`TimeoutError` is flattened and unexported.** The class survives only in `hooks.onError` + (`resilience.ts:17,233` never sets `.name`; `errEvt` at `engine.ts:346-355` flattens it; + `index.ts:86` exports only `RateLimitError`). A caller cannot type-test for a timeout. +- **No event carries the idempotency key.** So with the random default, the standard recovery — + query the vendor by key — is impossible: you never learned the key that was sent. +- **`StitchError` has no `headers`**, so a vendor's replay marker (the one signal that would say + "this was a replay, not a fresh charge") is unreachable from the error. It _is_ visible to + `hooks.onResponse`. + +**Ask:** export `TimeoutError` and preserve the class through `errEvt`; put the idempotency key +on the `start` event; consider `headers` on `StitchError`. + +## 3. `retry.on` cannot exclude a timeout, and cannot see a replay + +- **A transport failure is retried unconditionally** — measured **3 requests with + `retry: { on: [] }`** (`engine.ts:675-703`). For a non-idempotent write with no key, "retry + statuses but never a timeout" has no spelling; the only lever is `attempts: 1`. +- **A replayed failure cannot be excluded.** If 500 is added to `retry.on`, all 4 attempts burn + against the vendor's recording (3 replays). `Surface.interpret` cannot veto it — it runs + _after_ the retry check (`engine.ts:743` vs `:775`) — and the `retry.on` predicate receives + only the status, measured `[[500],[500],[500]]`. A `hooks.onResponse` status rewrite cuts 4 + requests to 2 at the cost of lying about the status. + +**Good news worth keeping:** the default `retry.on` (`[429, 502, 503, 504]`) means a cached 500 +is **not** retried out of the box — 1 request under `attempts: 4`. The capture predicted this +would burn the budget; it does not. + +## 4. TTL expiry has no client-side signal + +A prune turns a correct, stable key into a second charge: measured **2 charges at a 25 h delay +against a 24 h TTL**, 1 charge at 23 h. The duplicate arrives as a clean `200` with **no replay +marker**, so nothing distinguishes it from the original. `timeout.total` is per call and cannot +bound the gap. + +This is genuinely the vendor's semantics, not a library defect — but it is worth a line in the +idempotency guide, because the natural mental model ("the key makes it safe") has an expiry date +that is usually shorter than a dead-letter queue's. + +## 5. Footguns + +1. ‼ **`keyOf: (i) => JSON.stringify(i.body)`** — the obvious spelling — **fails by charging + twice, not by erroring.** Key order alone moved the hash: **2 keys, 2 charges, statuses + `[200, 200, 200]`**, no `409` anywhere, because the vendor never saw the same key twice. + `refKeyOf` and a canonical sha256 held at 1 key / 1 charge across all three body variants. + Worth an explicit "derive from a business reference, don't stringify the body" in the guide. + Note an input **schema does not canonicalise** what `keyOf` sees. +2. ‼ **A `__config` JSON round-trip drops `keyOf`** and leaves `idempotency: {}` — truthy, so the + random default is silently restored. Measured **2 charges, nothing warned**. Nothing shipped + rebuilds from `__config`, but it is documented as round-tripping as JSON, so this is a trap + for anyone who does. +3. **Pagination mints a key per page** — 3 pages, 3 distinct keys (`engine.ts:936,940` rebuild + per page). Correct for reads; a hazard if anyone paginates a write. +4. **`verdict: { accept: [409], flag: 'ok' }` swallows an idempotency conflict** — `ok: true`, + error payload returned as data. **Fourth sighting** of the absent-flag rule producing a silent + success (scenarios 2, 11, 14, 15); see the standing note in the [ledger](../LEDGER.md). +5. **Nothing expresses "sticky for a decline, fresh for a blip."** A stable key makes a recorded + failure sticky for the whole TTL — measured, a declined card stayed declined — while the + random key never reaches the record and charges again. Both are defensible; neither is + selectable, and the choice is currently a side effect of which key strategy you picked. diff --git a/docs/scenarios/issue-drafts/input-schemas-check-but-never-filter.md b/docs/scenarios/issue-drafts/input-schemas-check-but-never-filter.md new file mode 100644 index 00000000..55a86741 --- /dev/null +++ b/docs/scenarios/issue-drafts/input-schemas-check-but-never-filter.md @@ -0,0 +1,86 @@ +# `input` schemas check but never filter — the parsed value is discarded + +**Status:** ✅ **FILED** as [#648](https://github.com/rejifald/StitchAPI/issues/648) +**Scenario:** [agent-holds-the-tool](../agent-holds-the-tool.md) +**Proofs:** `docs/scenarios/proofs/agent-holds-the-tool/c7-schema.ts` + +## What happens + +`validateInput` (`packages/core/src/engine.ts:384-409`) awaits the validator, checks `r.ok`, +throws on failure — and **drops `r.value` on the floor**. The original, unparsed input goes to +the transport. + +`validateOutput` (`engine.ts:437`) does the opposite, and its comment says so explicitly: +_"On success returns the PARSED value"_. So the two halves of the same feature behave in +opposite ways, and only one of them is documented as doing so. + +This matters because **stripping unknown keys is the default behaviour of Zod, Valibot and +ArkType alike**. A reader who declares an input schema reasonably believes the request is now +shaped by it. It is not — the schema is a gate, not a filter. + +Measured: a `query` validator that returned `{ limit: 10 }` still put +`?tenant=globex&limit=10` on the wire, overwriting a `tenant=acme` pinned in the configured +path. + +## Minimal reproduction + +```ts +const getOrder = stitch({ + url: 'https://api.vendor.test/v1/orders?tenant=acme', + input: { query: z.object({ limit: z.number() }) }, // strips by default +}); + +await getOrder({ + query: { limit: 10, tenant: 'globex', include: 'internal_notes' }, +}); +// wire: /v1/orders?tenant=globex&limit=10&include=internal_notes +// the validator returned { limit: 10 }; nothing used it +``` + +## Why it is worth more than a doc note + +Two properties compound: + +1. **A schema constrains one slot.** Declaring `params` does nothing about `query`, so an + undeclared slot is a full passthrough. That is defensible on its own. +2. **A pinned query parameter is a default, not a pin.** `{ ...predefined, ...input.query }` + means `?tenant=acme` written into the configured path is overwritable by caller input — and + in our fixture the vendor duly returned the other tenant's data. + +Together, the one mechanism a reader would use to close (2) does not close it, because of (1) +and the discarded parsed value. The natural fix — "declare a strict schema" — silently does +nothing. + +This shows up hardest on the MCP surface, where the caller is a model and `run_stitch` forwards +its argument object, but nothing about it is MCP-specific: it is true of every call. + +## The ask + +Use the parsed value, as `validateOutput` already does: + +```ts +(input as Record)[part] = r.value; +``` + +If that is too breaking, then either: + +- an opt-in (`input: { strict: true }`), or +- a documented statement, at the `input` reference and in the MCP surface page, that **an input + schema validates and does not filter**, and that untrusted callers need the input rebuilt + before the call. + +The workaround today is a `Proxy` apply-trap that rebuilds the input from an explicit key list +before the engine sees it. That works and is about 15 lines, but every user exposing a stitch to +an untrusted caller has to invent it. + +## Related, same area + +`describe_stitch` reports a declared contract as `"params": true` — a stitch whose `params` +schema is `{ id: }` tells a model only that the slot exists. A model learns the shape by +failing. Worth surfacing the actual schema if it is JSON-Schema-representable. + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +filing. Runnable proof scripts live under `docs/scenarios/proofs/agent-holds-the-tool/` on the +branch `claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/mcp-error-channel-leaks-a-query-credential.md b/docs/scenarios/issue-drafts/mcp-error-channel-leaks-a-query-credential.md new file mode 100644 index 00000000..146c6ce9 --- /dev/null +++ b/docs/scenarios/issue-drafts/mcp-error-channel-leaks-a-query-credential.md @@ -0,0 +1,122 @@ +# MCP: the error channel is an unfiltered pass-through, and two smaller agent-boundary traps + +**Status:** drafted, not filed +**Scenario:** [agent-holds-the-tool](../agent-holds-the-tool.md) +**Proofs:** `docs/scenarios/proofs/agent-holds-the-tool/` (8 scripts, 181 checks, offline) + +## First, the part that held + +Worth stating up front because it is the headline claim and it survived the sharpest test we +could build. Across **34 JSON-RPC exchanges and 30 payload scans (14,529 bytes)** — `initialize`, +`tools/list`, `describe_stitch` on ten stitches, successful calls under `bearer` / `apiKey` +(header, query, cookie) / `cookieSession`, a vendor 401 whose **body held a credential-shaped +string**, a validation failure, an unknown stitch, an unknown tool, a bad JSON-RPC method, and +the same run over stdio — **not one of the five held credentials appeared by value anywhere**. +Controls confirm the wire carried them and the vendor authenticated every call. + +`sanitizeAgentInput` (`packages/core/src/mcp.ts:125-130`) is doing real work: six model-supplied +headers including `authorization`, `cookie` and `host` reached the vendor as **zero** headers. +The comment above it is accurate. + +The findings below are the edges around that. + +## 1. A transport error message reaches the model verbatim + +`packages/core/src/mcp.ts:184`: + +```ts +} catch (e) { + return errorResult((e as Error).message); +} +``` + +StitchAPI's **own** errors are terse and request-free — a vendor 500 whose body held an internal +hostname, a stack frame and a `postgres://vendor:hunter2@…` DSN reached the model as the four +characters `HTTP 500`. That part is good, and deliberate-looking. + +But the channel is unfiltered, so anything the **transport** writes passes straight through. On +the **default `fetchAdapter`**, with `apiKey({ in: 'query' })` and a mistyped port, the model +received: + +``` +Failed to parse URL from http://api.vendor.test:99999/v1/metrics?api_key=ak_live_qry_8899aabbccddeeff +``` + +With a `node-fetch`-shaped adapter the same thing happens on **any DNS failure** — a routine +production event, not a typo: + +``` +request to https://api.vendor.test/v1/metrics?api_key=ak_live_… failed, reason: getaddrinfo ENOTFOUND … +``` + +Zero lines of user code. The control pins the cause: the identical failure under `bearer` +disclosed the URL and no secret. + +This is `apiKey({ in: 'query' })` leaking where URLs go, which the auth guide already warns +about. The reason to file it anyway: **the model's context is a uniquely bad destination** — it +flows to the model's output, its logs, and any downstream tool it calls — and it is not a place +a reader thinks of as "where URLs go". + +**Ask:** redact the credential from messages crossing the MCP boundary (the auth strategy knows +its own parameter name), or — cheaper — have `describe_stitch`/the MCP docs warn when a +registered stitch uses `apiKey({ in: 'query' })`. + +## 2. A renamed stitch is callable but invisible + +`selectStitch` falls back from the registry key to each stitch's configured `name` +(`registry.ts:71-74`). So a filtered registry that **renames** a stitch to hide it still answers +to the original name — reachable and absent from `list_stitches` at the same time. + +That inverts the one seam the library gives for an allow-list. The registry object handed to +`createMcpServer` is otherwise a genuinely good boundary (we built a working allow-list on it in +47 lines). + +Compounding it: the documented starter `stitch mcp --module ./stitches.ts` builds that object +with `collectStitches`, which sweeps up **every exported stitch** — in our fixture a write and a +login stitch alongside the intended read. + +**Ask:** resolve MCP tool calls by registry key only, and document that `--module` exposes +everything. + +## 3. `cookieSession` joins where `apiKey({ in: 'cookie' })` replaces + +Only reachable when the operator declares an `input.headers` schema, so it is the narrowest of +the three — but the outcome is session fixation. Measured: + +``` +Cookie: tracking=xyz; SESSION=attacker; SESSION=sess_live_cookie_abcdef0123456789 +``` + +The model's pair is sent **first**. A vendor that reads the first occurrence runs the call as the +model's session. `auth.ts:918-921` joins; `auth.ts:228-245` (`apiKey`) replaces. + +**Ask:** have `cookieSession.apply` drop a pre-existing pair with the same cookie name, as +`apiKey` does. + +## 4. No tool annotations, so a host cannot prompt + +All three tool descriptors carry no `annotations`, so `readOnlyHint`/`destructiveHint` — the +fields an MCP host reads to decide whether to ask a human — are absent. And because code-mode +puts every endpoint behind one tool name, reading an order and issuing a 25,000 refund arrive at +the host as the same `run_stitch` call. + +`list_stitches` does report `POST /v1/refunds`, but a host would have to call a tool to learn +that, and annotations are fixed at `tools/list` time. + +User code can **refuse** — `hooks.onRequest` throwing gave the vendor zero requests, the model a +readable reason, and (nicely) was asked exactly once despite `retry: { attempts: 3 }`, because a +refusal is not a retryable failure. But nothing in the process can **ask**. + +**Ask:** emit `annotations` per tool, at minimum `readOnlyHint` on `list_stitches` and +`describe_stitch`. + +## Not filed here + +`validateInput` discarding its parsed value is the other finding from this scenario and it is +**not MCP-specific** — see the companion issue. + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +filing. Runnable proof scripts live under `docs/scenarios/proofs/agent-holds-the-tool/` on the +branch `claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/no-compensation-seam.md b/docs/scenarios/issue-drafts/no-compensation-seam.md new file mode 100644 index 00000000..e7fb49bc --- /dev/null +++ b/docs/scenarios/issue-drafts/no-compensation-seam.md @@ -0,0 +1,109 @@ +# Issue draft — there is no seam that runs on failure, so mandatory cleanup can't be expressed + +**Status:** ✅ **FILED** as [#656](https://github.com/rejifald/StitchAPI/issues/656) +**Scenario:** [`multipart-upload`](../multipart-upload.md) +**Suggested template:** feature_request.yml · **Suggested labels:** `enhancement`, `hooks` + +> The scenario came out achievable — a `try/finally` in user orchestration gets to **0 orphans +> on every exit path**. But this is the first scenario in the pass where the library contributes +> _nothing at all_ to the requirement the scenario exists for, and two of the ways people will +> write that `try/finally` are measurably wrong while looking right. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/multipart-upload/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. Nothing runs on failure + +Some APIs impose a **compensating action**: if a multi-step operation fails partway, you must +call a _different_ endpoint to undo it. S3 multipart is the canonical case — abandon an upload +and every part already sent bills as storage indefinitely, invisible to `aws s3 ls`. AWS puts +this at [up to 20% of an S3 bill](https://aws.amazon.com/blogs/aws-cloud-financial-management/discovering-and-deleting-incomplete-multipart-uploads-to-lower-amazon-s3-costs/). + +Nothing in the library can express it: + +- `Hooks` is exactly `{ onRequest, onResponse, onError, onRetry }` (`types.ts:1285-1290`). +- **`onError` is not a failure hook.** It fires from the `catch` around the transport + (`engine.ts:668-680`). Measured on an HTTP 500: hook sequence `[onRequest, onResponse]`, + **0** `onError` calls. On a stalled socket it fired 3 times — once per attempt, not once per + failure. +- `HookContext` (`types.ts:1279-1284`) is `{ name, attempt, req, res, error }` — **no + run-scoped slot**, so even if a hook did fire there is nowhere to keep the `UploadId` it + would need. +- `linked()` is `Promise.resolve(body(run))` (`pipe.ts:357-369`) — no `finally`. +- A trace sink sees the terminal `done` (measured 4 events, one `ok: false`) but it is a _log_ + seam: per stitch call, no control flow, and no `UploadId` either. +- Inventing `onFinally` is **accepted at runtime**, lands on `__config`, and never runs. + +**Measured orphans** with a part failing and no user-side cleanup: **3 parts / 15 MiB / 1 +dangling UploadId / 0 DELETEs**. Cancelling via `AbortSignal`: **2 orphans, 0 DELETEs**. A +`timeout.total` expiry: **3 orphans, 0 DELETEs** — cancellation cancels in-flight work and +forgets the work that landed. + +**Ask:** a per-**run** terminal seam with control and a scratch slot — `onSettle(ctx)` carrying +`ok` plus somewhere to have stashed the `UploadId`. Today the only per-run terminal signal is +the `done` trace event, which has neither. This is the whole fix; everything below is the +consequence of its absence. + +## 2. Two ways the `try/finally` people write instead is wrong while looking right + +**Counts double — `.safe()` on the cleanup call cannot throw.** In a correct-looking +`try/finally`, pointed at a wrong `UploadId`: **0 accepted DELETEs, 3 orphaned parts, and +nothing thrown anywhere.** The `finally` ran; the review passes; the bill is permanent and +invisible. The codebase's own preference for `.safe()` over `try/catch` leads directly here. + +**Counts double — cleanup inside `Surface.execute` runs after the caller returns.** The +tempting design is "make the whole upload one stitch so the engine owns the lifecycle". +Measured at the instant the caller's promise settled on a timeout: **3 orphans, 0 DELETEs**; +the DELETE landed several turns later, because `withTimeout` (`resilience.ts:230-244`) rejects +the caller and lets `fn` run on. In a lambda, or any process that exits on the error, the later +half never happens. + +**Ask:** if `onSettle` lands, document that cleanup must be loud. If it doesn't, the pitfalls +page should carry both of these — they are not obvious and both are silent. + +## 3. Contributing hazards measured alongside + +- **`all()` bounds nothing.** Peak in-flight measured **8** over 8 members (`pipe.ts:122-136` + maps straight into `Promise.all`). It also hands every member the **same `StitchInput`** + (`pipe.ts:75-76`) — measured: one stitch × 8 members produced 8 PUTs all carrying + `partNumber=1` and stored **one** part. +- **`throttle.concurrency` defaults to `pool: 'stitch'`**, so 8 stitches at `concurrency: 3` + each measured a peak of **8** (`resilience.ts:103-109`). `pool: 'host'` or a seam bucket + gives 3. The combination that reads correct — `all()` plus per-member `concurrency` — bounds + nothing. +- **`all()` discards partial results on fail-fast.** 2 parts stored server-side, **0 nameable** + by the client for the abort. No `allSettled` (`pipe.ts:20` says the omission is deliberate); + only an `onResponse` side channel recovers them. Using `.safe()` members keeps the values but + disables fail-fast — measured part 4 uploading _in full_ into an already-doomed upload. +- **The default `retry.on` excludes 500** (`engine.ts:612`), which is S3's own transient error + (`500 InternalError`). Measured with the default set: 4 PUTs, 1 failed part, **3 orphans**. + Widening to include 500 restored a clean run. +- **Whole-upload retry is neither flagged nor prevented.** `retry: { attempts: 3 }` on an outer + orchestration stitch measured **3 initiates, 12 PUTs, 3 dangling UploadIds, 9 orphaned parts, + 45 MiB**. +- **A progress tick has no identity.** `AdapterProgress` is `{ direction, loaded, total }` + (`types.ts:858-867`) — no part number, request or run id — so a shared `onProgress` across a + concurrent fan is unattributable. Ticks are cumulative _within_ a part, so the natural + `Σ loaded` overshoots: measured **400** for a 160-byte file, and **300** when a retry replays + a part's ticks from zero. A per-part high-water map gives the right answer. +- **`.inspect()` carries no response headers** (`types.ts:1736-1766`), so on the awaited path a + part's `ETag` is unrecoverable without a custom surface. +- **A custom `interpret` that omits `verdictOf` turns an HTTP 500 part into `ok: true` / + `data: undefined`** — the failure then surfaces only when `complete` rejects the list. This is + the fourth scenario in which `verdictOf` had to be remembered; see the standing note in the + [ledger](../LEDGER.md). + +## 4. Two incidental corrections, unrelated to the scenario + +- **`backoff` has no `delay` field.** It is `{ curve, base, max }`; `delay` is a compile error + and a runtime no-op. +- **The docs reference a `pipe()` combinator that does not exist.** + `apps/docs/content/docs/concepts/run-identity.mdx:26` and `:33` describe "each step of a + `pipe()`" and label a diagram `a pipe(): step 1`. Verified: `stitchapi/pipe` exports exactly + `all, any, linked, race`. The construct the passage describes is `linked()`. Left unfixed + here to keep the scenario commits scoped — it is a two-line docs edit. diff --git a/docs/scenarios/issue-drafts/oauth2-params-rotation-footgun.md b/docs/scenarios/issue-drafts/oauth2-params-rotation-footgun.md new file mode 100644 index 00000000..fe01b0c4 --- /dev/null +++ b/docs/scenarios/issue-drafts/oauth2-params-rotation-footgun.md @@ -0,0 +1,97 @@ +# Issue draft — `params` lets you configure a rotating refresh grant that works once, then revokes the account + +**Status:** ✅ **FILED** as [#657](https://github.com/rejifald/StitchAPI/issues/657) +**Scenario:** [`oauth2-refresh-token-rotation`](../oauth2-refresh-token-rotation.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `auth`, `footgun`, `docs` + +> The scenario itself came out **achievable** (a custom `AuthStrategy` does it — proven by +> `docs/scenarios/proofs/oauth2-refresh-token-rotation/c5`/`c6`), so no escalation was needed +> for achievability. This draft is a _separate_ finding surfaced by the same verification: a +> silent-failure path that the current docs actively point readers toward. + +## Summary + +`OAuth2Options.params` merges arbitrary fields into the token-request body and can override +`grant_type` (`packages/core/src/auth.ts:393`, merged at `:511-518`). That makes this compile, +typecheck, and **succeed on the first call**: + +```ts +oauth2({ + tokenUrl, + clientId: env('ID'), + clientSecret: env('SECRET'), + params: { grant_type: 'refresh_token', refresh_token: storedRefreshToken }, +}); +``` + +But `oauth2()` reads only `access_token` and `expires_in` from the token response +(`auth.ts:549-568`) — the vault entry is exactly `{ token, expiresAt }`. A **rotated** +`refresh_token` in that response is silently discarded. + +Against a provider that rotates (Atlassian, Asana — RFC 6819 §5.2.2.3 replay detection), the +second redemption therefore presents the **already-consumed** token. That is not a failed +request: the provider treats it as token theft and **revokes the entire token family**. The +connected account is dead until the user re-authorizes in a browser. + +## Why this is worth fixing rather than documenting away + +The failure has every property that makes a footgun expensive: + +- **No signal at authoring time.** No type error, no runtime warning, no lint. +- **No signal on first use.** Redemption #1 returns 200 and the call succeeds. +- **The docs point at it.** [`guides/auth/oauth2.mdx:78-82`](../../../apps/docs/content/docs/guides/auth/oauth2.mdx) + says `params` "merges arbitrary fields into the token-request body (e.g. `resource`, or a + custom `grant_type`)" — naming a custom `grant_type` as an intended use, with no caveat. +- **The blast radius is the account, not the request.** + +## Reproduction + +Measured, offline, no network: + +```bash +pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c4-params-escape-hatch.ts +``` + +Observed (C4a): redemption #1 sends `RT-0`, provider rotates to `RT-1`, vault stores only +`{token, expiresAt}`. Redemption #2 sends **`RT-0` again** ⇒ 1 replay ⇒ family revoked ⇒ the +call throws. `RT-1` never appears on the wire. + +The proof also closes off the obvious workaround. A `params` **getter** (legal — an object with +a getter satisfies `Record` and `Object.assign` invokes it per request) plus a +capturing adapter does rotate correctly in one process (C4b/C4c: 11 redemptions, 0 replays). +It still revokes the family across two workers (C4d), and it **cannot be repaired**, because a +getter must return synchronously while every `StitchStore` read is async — C4e measures +`[object Promise]` arriving at the token endpoint. + +## Options (in increasing order of work) + +1. **Docs-only.** Add an explicit "not for rotating refresh tokens" warning to the `params` + paragraph and to the `oauth2` reference. Cheapest; leaves the silent-failure path in place. +2. **Fail loudly.** Throw at strategy construction when `params.grant_type === 'refresh_token'`, + pointing at the custom-strategy path. Turns a revoked account into a startup error. Narrow, + but only catches the literal spelling. +3. **Preserve the rotated token.** When the token response carries a `refresh_token`, persist it + to the vault alongside the access token. Small change, and it makes a correct + `refresh_token` grant expressible — but only meaningful together with (4). +4. **A first-class rotating grant.** `oauth2({ grant: 'refresh_token', … })` that rotates, + persists write-before-use, and coordinates redemption per account across workers. This is + the real fix and the largest; it needs a cross-process lock the store does not currently + offer (see the companion note below). + +## Companion finding (separate, smaller) + +`StitchStore` is `get`/`set`/`increment`/`close` with no compare-and-set and no blocking wait +(`types.ts:1965-1976`). A mutex can be built from `increment(key, ttl) === 1` — proven in +`c6-cross-process-lock.ts`, 3 workers × 10 callers ⇒ 1 redemption — but its correctness depends +on `increment` being atomic **across processes**, while the store contract only specifies +atomicity _within_ a process (`testing.ts:160-162`). Either the contract should be tightened for +backends that can honor it, or a lock primitive should be offered directly. + +## Also worth a docs correction + +[`guides/auth/oauth2.mdx:50-52`](../../../apps/docs/content/docs/guides/auth/oauth2.mdx) — +"one token serves them all — across stitches and across workers" is true for _sharing_ a token +once a write has landed, but not for _coordinating the fetch_. Measured: two cold workers on a +shared `store` + `key` fire one token request **each** (`c3-two-workers-shared-store.ts`); the +count scales with workers, not callers. Harmless for `client_credentials` (a wasted request); +fatal under rotation. diff --git a/docs/scenarios/issue-drafts/paginate-cannot-report-a-partial-run.md b/docs/scenarios/issue-drafts/paginate-cannot-report-a-partial-run.md new file mode 100644 index 00000000..9e01e23e --- /dev/null +++ b/docs/scenarios/issue-drafts/paginate-cannot-report-a-partial-run.md @@ -0,0 +1,104 @@ +# Issue draft — four different endings share one `break`, and the natural dedupe causes data loss + +**Status:** ✅ **FILED** as [#645](https://github.com/rejifald/StitchAPI/issues/645). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`unstable-pagination`](../unstable-pagination.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `bug`, `paginate`, `data-loss` +**Companion to:** [`paginate-silent-data-loss`](paginate-silent-data-loss.md) — same root cause, +two more consequences + +> `paginate`'s third appearance in this pass, and the first from the _data-correctness_ angle. +> The earlier draft asked for a stop reason; this one shows two further things that go wrong +> without one, including a case where the standard fix **causes** the loss. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/unstable-pagination/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. Deduping in `items`/`transform` can end the run early and lose rows + +**Severity: high — the recommended mitigation causes the damage it prevents.** + +Client-side dedupe by id is the standard advice for offset drift, and `items`/`transform` are +where a reader will put it — they're the per-page hooks. + +They run **above** the `break` at `engine.ts:984`, which fires on `items.length === 0`. So on a +workload where page 2 repeats page 1 verbatim (four inserts behind the cursor), the deduper +emptied that page, the loop read zero items as "the collection ended", and the run finished +**`ok`** having **skipped `["r05".."r10"]` — 6 rows lost by the fix**, against a declared total +of 14. Without the dedupe the same run returned all 10 rows it could see. + +Related and equally quiet: **a deduping `items` on a reused stitch returns `data: []`, +successfully, on every call after the first**, because the `seen` set outlives the call. Defining +a stitch once and calling it many times is the normal shape. + +**Ask:** this is the `items.length === 0` break again (see the companion draft). If it stays, +the pagination guide should say explicitly that dedupe belongs in `output` — after the loop — +and never in `items`/`transform`. `output` works correctly and is measured doing so. + +## 2. Four different endings are one `break` and one successful result + +Measured, all four ending `ok: true` with `error: null` and no distinguishing event: + +| ending | measured | +| ---------------------------------- | --------------------------------------------------------------------------------------- | +| the collection ended | correct | +| a page came back **empty mid-run** | 8 deletes left 4 rows; the offset-4 window was empty; **skipped `["r10","r11","r12"]`** | +| the **page cap** was hit | `pages: 50` default → **200 of 220 rows**, skipped `["r201".."r220"]` | +| a **deduper** emptied a page | §1 above | + +The empty-page case is the nastiest, because the shrunken `total` makes a reconciler _agree_: +4 collected against a declared total of 4, while 3 rows were never read. + +**Ask:** the stop reason asked for in the companion draft would separate all four. `pages` being +a _silent_ terminus is arguably its own bug — a cap that truncates should say so. + +## 3. The check everyone writes doesn't catch the case that matters + +`length === total` is the standard reconciliation. Measured over clean / insert / delete / ties, +it fired **0 of 4** times. The deduped variant fired on the insert (a false alarm — nothing was +lost) and on the ties, and **missed the delete entirely** while `r05` was gone. + +The reason is exact: a delete behind the cursor removes one row from the result **and** one from +`total`, at the same instant. The arithmetic balances perfectly. + +The signal that does carry it is that **the declared `total` moved between pages** (10 → 9) — +a check none of the standard write-ups name. Over 8 workloads a detector built on it had **zero +false negatives** and 3 false alarms. + +**Ask:** surface `total`-like fields, or at least document the moved-total check in the +pagination guide. Today `total` is reachable only from `transform` or `hooks.onResponse` — +`next` never sees the terminal page's body, so the obvious place to look is the one place it +isn't. + +## 4. Smaller findings from the same verification + +- **The default `items` wrap inverts the safety.** An envelope is always one item, so an empty + page never breaks the loop — but then `data.length` was **2 for a 12-row collection**, and + every downstream count measures _pages_. `pick: 'rows'` truncates identically. +- **`drift()` cannot express a duplicate.** Deduping an array re-indexes it, so the findings come + back as 3 × `coerced` and 1 × `undeclared` on element paths (`drift.ts:59-69`) — none saying + "duplicate". Reasonable given what drift is for; worth knowing it is not the tool here. +- **`.report()` is a fresh probe** (third sighting in this pass — see + [`clock-and-diagnostic-side-effects`](clock-and-diagnostic-side-effects.md)). Here it + re-paginated the collection, made 3 more requests, and **did not reproduce the duplicate at + all** — it describes a run it just made, never the run you made. +- **A rejecting `output` gives `.safe()` a generic message and `data: null`** — the offending + ids and the partial rows exist only on `.report()`. Same shape as the + `.safe()`-drops-the-body finding in [`body-verdict-footguns`](body-verdict-footguns.md). +- **A custom `Surface.interpret` is the only seam that stops _at_ the drifted page** with the id + named — at the cost of discarding every row already collected, and emitting no `result` event. + +## 5. What works, and is worth documenting as the pattern + +**Keyset via `next` is four lines and correct.** `next` receives the previous page's raw body +(`engine.ts:985`), so a composite `(created_at, id)` cursor is trivial, and against a real seek +endpoint it measured `skipped []` / `duplicated []` on every workload that broke offset. The +zero-item break is _right_ for this case — it's the natural terminus of a cursor walk. + +One caveat worth a line in the guide: **sending a composite cursor does not make an endpoint a +seek endpoint.** The same four lines against a vendor that accepts `(after_ts, after_id)` but +orders by `created_at` alone lost `["r03"]` and duplicated `["r13"]`, with no writes. diff --git a/docs/scenarios/issue-drafts/paginate-silent-data-loss.md b/docs/scenarios/issue-drafts/paginate-silent-data-loss.md new file mode 100644 index 00000000..df30f9b3 --- /dev/null +++ b/docs/scenarios/issue-drafts/paginate-silent-data-loss.md @@ -0,0 +1,111 @@ +# Issue draft — `paginate` ends successfully and drops data when a page returns zero items + +**Status:** ✅ **FILED** as [#644](https://github.com/rejifald/StitchAPI/issues/644). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`batch-partial-failure`](../batch-partial-failure.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `bug`, `data-loss`, `paginate` + +> This is the strongest finding of the scenario pass so far. Unlike the earlier drafts, which +> report footguns, **this one is a plain bug**: a correct-looking `paginate` config loses data +> on an ordinary upstream response and reports success. + +Reproduce (all measurements below are from these scripts): + +```bash +for f in docs/scenarios/proofs/batch-partial-failure/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. A zero-item page ends the loop successfully — with the remainder unfetched + +**Severity: high — silent data loss.** + +`paginated` breaks out of the loop when a page aggregates zero items: + +``` +engine.ts:984 if (items.length === 0 || page >= max) break +``` + +The break happens **before `next` is called** (`engine.ts:985-986`), so the loop cannot ask +whether there was more to fetch. The call then resolves through the normal path +(`engine.ts:1005-1010`) with `ok: true` and `error: null`. + +That is fine for a cursor API, where an empty page really does mean the end. It is wrong for +any endpoint where an empty page is a **transient** condition. Measured, on a batch-write loop +against a table that is out of write capacity — the single most ordinary response DynamoDB +gives under load — a round landed zero items, the loop stopped, and **4 of 6 rows were never +written. `ok: true`. No error. Nothing in the event stream.** + +`items.length === 0` is not the termination contract. `next() === undefined` already is, and +it is the one the docs describe: _"Return `undefined` to stop."_ The zero-item break is a +second, undocumented termination condition that the user cannot override or opt out of. + +**Ask:** drop `items.length === 0` as a termination condition and let `next` decide. If it must +stay for compatibility, make it opt-out (`paginate: { stopOnEmpty: false }`). + +## 2. "Finished" and "gave up" are the same return value + +**Severity: high — an unbounded run looks identical to a complete one.** + +Hitting the `pages` cap (default 50) also breaks at `engine.ts:984` and resolves `ok: true`. +So three genuinely different outcomes are indistinguishable to the caller: + +- the cursor ran out — complete; +- the page cap was hit — **incomplete**; +- a page landed nothing — **incomplete**. + +Measured: cap hit at 3 rounds with a true residue of `def`, and the caller received +`ok: true`, `error: null`, `data: abc`. `.inspect().raw` was `abc`, `.report()` reported no +error and `attempts: 1`, and the event stream's last `paginate` detail was +`page 3 (+1, total 3)` — accurate, and no help. A `trace` sink carried nothing either. + +**Ask:** put the stop reason on the result and in `.report()` — `'exhausted' | 'page-cap' | +'empty-page'`. A caller cannot currently write a correct completeness check at all. + +## 3. A residue ledger built in `paginate.next` is stale by one round — and names the wrong items + +**Severity: medium — it typechecks, reads correctly, and is wrong.** + +Because `next` is only invoked when the loop _continues_, it never sees the final page. A user +tracking "what is still outstanding" inside `next` — the obvious place — is always one round +behind. + +Measured: with a true residue of `def`, the ledger built in `next` reported **`cdef`**. It +names row `c`, which **had already landed**. Acting on that ledger re-writes `c`. + +Only a `hooks.onResponse` closure reported the residue correctly (3 calls, correct answer), or +reconstructing it as a set-difference from the aggregated successes. + +**Ask:** call `next` on the terminal round too (with a flag), or add +`onStop(prevBody, reason)`. Either gives the user one correct place to read the tail state. + +--- + +## Why this cluster matters together + +Individually each is arguable. Together they mean **a `paginate` user cannot detect an +incomplete run**: the result says success, the error is null, the events look normal, the +report is clean, and the one place they'd naturally track progress lies to them. The Logstash +bug this scenario is drawn from +([`elastic/logstash#1631`](https://github.com/elastic/logstash/issues/1631)) is exactly this +shape, and it took a long time to find precisely because nothing reported it. + +The scenario itself came out **achievable** on a different seam (`Surface.interpret` + +`hooks.onRequest`, ~50 lines — see the published page), so this is not a gap in what the +library can do. It is that the primitive that _looks_ like the answer fails quietly. + +## Smaller notes from the same verification + +- **`verdict: { flag: 'UnprocessedItems' }`** reads as "fail when there are unprocessed items" + and is **inert** — arrays are truthy whether empty or not. (Related to the `verdict.flag` + finding in [`body-verdict-footguns`](body-verdict-footguns.md); same root cause, different + surface.) +- **A backoff sleep in `onRequest` is invisible to the engine** — 2.5 s of real waiting + produced 0 `throttled` events and no `waited` in the run report. If user-space waiting is + the sanctioned pattern for a growing backoff, the engine should account for it. +- **`SurfaceOutcome` has no attempt number** (`surface.ts:61-64`), so a surface that needs to + know it is on its last round must count its own invocations and keep that count in sync with + `retry.attempts` by hand — an easy thing to get out of step. +- **`cloneReq` shares `body` by reference** (`engine.ts:261-264`). Mutating `ctx.req.body` in + place instead of assigning rewrites _the caller's own object_ — measured: a caller passed 6 + items and got their array back holding 2. Worth a line in the hooks guide. diff --git a/docs/scenarios/issue-drafts/resilience-has-no-tenancy.md b/docs/scenarios/issue-drafts/resilience-has-no-tenancy.md new file mode 100644 index 00000000..5d599b90 --- /dev/null +++ b/docs/scenarios/issue-drafts/resilience-has-no-tenancy.md @@ -0,0 +1,130 @@ +# Issue draft — `throttle` and `circuit` have no tenancy axis, so one tenant can fail all of them + +**Status:** ✅ **FILED** as [#641](https://github.com/rejifald/StitchAPI/issues/641). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`multi-tenant-blast-radius`](../multi-tenant-blast-radius.md) +**Suggested template:** feature_request.yml · **Suggested labels:** `enhancement`, `resilience`, `multi-tenant` + +> The highest production-impact finding of the pass. It is not a bug — every piece behaves as +> documented — but the composition has a 100% blast radius, and the fix is one option name on +> two interfaces, on an axis the codebase already has. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/multi-tenant-blast-radius/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. The measurement + +A shared seam, `circuit: { failures: 3, cooldown: '30s' }`, ten customers each bound with +`seam.as(id)`. One customer's refresh token has been revoked, so their calls 401. + +**Measured: 9 of 9 healthy customers failed**, with `StitchError` status 503 `circuit open`, and +**zero** of their requests reached the vendor. + +Worse, it does not self-heal. Half-open admits exactly one trial call +(`resilience.ts:375-379`), and the broken tenant is the one retrying hardest — so across **four +full cooldown windows (120 virtual seconds)** the healthy tenant measured `503, 503, 503, 503`. +Recovery happens only if a healthy tenant wins the probe race. + +## 2. Why — the principal stops at the auth boundary + +`AuthContext.principal` reaches the auth strategies (`auth.ts:483-499`) and the cache-key builder. +It does not reach the resilience layer at all. The breaker keys on +`opts.key ?? hostKey(req, cfg)` (`resilience.ts:353`, `engine.ts:860`), and `hostKey` is +`cfg.name ?? cfg.path ?? 'stitch'` (`engine.ts:140,265-274`) — no principal anywhere. + +The consequence is a split that maps exactly onto the auth/resilience line: + +| resource | isolated by | fails | +| --------------- | ---------------------------------- | --------------------------------------------- | +| Token | `oauth2({ tenancy: 'principal' })` | **closed** (errors without a bound principal) | +| Cache | `tenancy`, default `'principal'` | **closed** | +| Rate budget | a limiter key you hand-write | **open, silently** | +| Circuit breaker | a `circuit.key` you hand-write | **open, silently** | + +The two whose isolation is a **security** property fail closed. The two whose isolation is an +**availability** property fail open, with no diagnostic. + +## 3. Isolation is a property of the key string, never of the object graph + +This is what makes it hard to get right by intuition. All three of these look isolated and are +not — measured: + +- **10 distinct `.as()`-bound stitch objects** on the same `path` → **1** breaker key, 9/9 down. +- **10 distinct seams** sharing one store → same. +- A **`url`-only stitch** keys its breaker on the literal string `'stitch'`, so every such stitch + sharing a store shares one process-wide breaker — across tenants _and_ endpoints. + +And one that actively un-isolates: **`throttle: { pool: 'host' }` silently re-keys the circuit** +onto the host (`engine.ts:265-274` feeding `:860`). A per-tenant `name` partition evaporates, and +an unrelated endpoint for an unrelated tenant measured `503`. + +Sharpest of all: **a per-tenant seam isolates the rate budget and shares the breaker.** Measured +in a single run — quiet tenant at t=0 (rate isolated), 3 of 3 healthy tenants `503` (breaker +shared). The two resources are keyed by different rules, so no construction can be reasoned +about as a whole. + +## 4. The ask + +Add `tenancy?: 'principal' | 'app'` to `ThrottleOptions` and `CircuitOptions`, and thread the +principal into `hostKey`/`createCircuit`. It is the **same axis `CacheOptions` and +`OAuth2Options` already carry** (`types.ts:1136-1147`), so it needs no new concept — only the +existing one extended to the two interfaces that lack it. + +`'app'` should stay the default for compatibility, but a seam that has `auth` with +`tenancy: 'principal'` and a `circuit` with `tenancy: 'app'` is almost always a mistake, and is +worth a construction-time warning. + +The workaround works and is cheap — a per-tenant `name` and `circuit.key`, ~3 strings per tenant; +100 keyed stitches built in under 100 ms with zero timers. But it has to be _known_, and nothing +in the type system, the docs, or the runtime points at it. + +## 5. A second, independent ask: don't count credential failures as dependency failures + +A `401` means the credential is bad, not that the vendor is down. Counting it toward a breaker is +what turns a per-tenant credential problem into a dependency-wide one. + +Measured: three 401s recorded `failures: 3` and tripped the breaker, because a bad status reaches +`attemptWithCircuit` as a **throw** (`engine.ts:824-831, 879-890`). + +The good news is that the engine already routes on _what_ failed, so the fix is pure config: +`verdict: { accept: [401], flag: 'ok' }` gave the bad tenant a real `StitchError` 401, **0** +circuit failures, and 0 of 9 healthy tenants affected — while a genuine 500 still tripped the +breaker as designed (`500, 500, 500, 503`). + +**Ask:** document this pattern in the circuit-breaker guide. `verdict: { accept: [401] }` _alone_ +is the trap — measured, it swallows the failure entirely and hands the caller +`{"error":"invalid_token"}` as its **data**. The `flag` is what makes it correct, and the pairing +is not obvious. + +## 6. Smaller findings from the same verification + +- **`oauth2` defaults to `tenancy: 'app'`** (`auth.ts:483-486`). Measured: 3 different customers, + **1** token fetch, one shared `Authorization` header. For an app-level credential that is + right; for a per-customer integration it is a silent credential bleed, and nothing at the call + site hints at which one you have. +- **`tenancy` partitions the token cache, not the credential.** All tenants' tokens were minted + from one `client_id`, because `Secret = string | (() => string)` (`auth.ts:47`) is niladic. A + custom `AuthStrategy.apply(req, ctx)` reading `ctx.principal` works (measured + `Bearer cred-for-t1` / `cred-for-t2`) and is the only user-reachable hook that sees the bound + principal at call time — worth documenting as the per-customer-credential pattern. +- **Global quota + per-tenant fairness is not expressible.** A member throttle stacks + tighten-only on the seam bucket (`seam.ts:94-106`), so adding the vendor's global cap + re-instates the noisy neighbour (quiet tenant back to t=2000). +- **Breaker records have no TTL** (`resilience.ts:382-403` writes with no `ttl`; `store.ts:45` + treats that as live-forever). A churned tenant's key was still resident after a virtual year, + and nothing sweeps them — while the rate counter beside it does expire. At 4,000 connections + that is 4,000 immortal keys. +- **`seam.stitch()` pins every stitch it creates.** With `WeakRef` after a forced GC: **200/200** + root-created still reachable, **0/200** created through `seam.as(p).stitch()` + (`seam.ts:136-141` — `runtime.register` is set only when `principal === undefined`). The + per-request shape is the one that doesn't leak; the only release for the other is + `seam.close()`, which also closes the store. +- **Seam ids are a module-level creation-order counter** (`seam.ts:38,233`), so two workers each + hand out `s1, s2, s3`. Per-tenant seams over a shared durable store therefore collide across + processes non-deterministically — worker A's tenant-1 bucket is worker B's tenant-7 bucket. +- **`CircuitOpenError` carries nothing identifying the tripping tenant**, so a shared-breaker + outage cannot be attributed from the error alone. diff --git a/docs/scenarios/issue-drafts/sigv4-ignores-the-injected-clock.md b/docs/scenarios/issue-drafts/sigv4-ignores-the-injected-clock.md new file mode 100644 index 00000000..5c6ef4f9 --- /dev/null +++ b/docs/scenarios/issue-drafts/sigv4-ignores-the-injected-clock.md @@ -0,0 +1,108 @@ +# Issue draft — SigV4 signs with `new Date()`, and a skew 403 opens the dependency's breaker + +**Status:** ✅ **FILED** as [#658](https://github.com/rejifald/StitchAPI/issues/658) +**Scenario:** [`expiring-signatures`](../expiring-signatures.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `aws-sigv4`, `testing`, `resilience` + +> **Leading with what the library gets right**, because it is the headline of this scenario and +> it is a genuine design property: **a StitchAPI throttle cannot expire a signature.** +> `acquireWithin` (`engine.ts:629`) sits _above_ `cfg.auth.apply` (`:649`) inside the attempt +> loop, and `cloneReq` gives each attempt fresh headers off the unsigned base. Measured: 4 calls +> behind `rate: '1/2m'`, granted at 0/2/4/6 virtual minutes, signature ages **0, 0, 0, 0 ms**, +> all `200` — where the same calls pre-signed (the botocore#149 shape) aged 0/2/4/**6 min** and +> the last got a `403`. The breaker doesn't queue a signed request either: 3 blocked calls, **0** +> signings. [botocore#149](https://github.com/boto/botocore/issues/149) cannot happen here. +> +> Three findings sit beside that. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/expiring-signatures/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `hooks.onRequest` runs after signing, so a hand-rolled gate there re-creates the bug + +**Severity: medium-high — it re-introduces a defect the library is otherwise immune to.** + +`onRequest` (`engine.ts:652`) is the only user-code seam that runs **after** `auth.apply` +(`:649`). Anyone who paces calls with a sleep in `onRequest` — a plausible thing to write, and +the obvious place to put a custom gate — signs first and waits second. + +Measured: a 6-minute wait in `onRequest` aged the signature **6 minutes** and the request came +back `403`. The identical wait expressed as `throttle` aged it **0 ms**. + +**Ask:** a line in the hooks guide saying `onRequest` runs post-auth and must not block, with a +pointer to `throttle`. Better: emit an `info` when an `onRequest` hook's duration exceeds some +threshold on a stitch carrying `auth` — the engine already has the precedent of warning about an +undrawable upload-progress bar (`engine.ts:1694-1698`). + +## 2. SigV4 signs with `new Date()` rather than the injected clock + +**Severity: medium — a testability defect, not a wire defect.** + +`aws-sigv4/src/index.ts:244-249, 301` calls `amzDateOf(new Date())`. On real time the stamp is +correct, so nothing is wrong on the wire. But it means **SigV4 behaviour cannot be tested on a +virtual clock**: + +- 600 virtual seconds moved the shipped stamp **0 seconds** (a clock-reading signer moved 600). +- Under a default `manualClock()`, **0 of 3** calls were accepted — ~20,670 days of apparent + skew, because the virtual clock starts at epoch while the server's validation reads real time. + +Every proof in this scenario had to inject its own clock-reading signer to measure anything. + +**This is the third instance of one inconsistency.** `timeout.total` +([scenario 4](clock-and-diagnostic-side-effects.md)) and `cache.ttl` +([scenario 6](cache-cannot-revalidate.md)) also read wall-clock while their neighbours use the +injected `clock`. + +**Ask:** thread the stitch's `clock` into the signer. And — the standing request from the earlier +two drafts — **audit which time-driven features read the injected clock and which read +`Date.now()`, and state the answer in the testing guide.** Three point fixes are worth less than +one documented rule. + +## 3. A skew 403 counts as a circuit failure + +**Severity: medium — a fault inside your process opens the dependency's breaker.** + +A bad status reaches `attemptWithCircuit` as a throw (`engine.ts:824-831`), so +`RequestTimeTooSkewed` — which means _your clock is wrong_ — is recorded against the vendor. +Measured with `circuit` configured: `["403", "403", "503", "503"]`. The half-open probe then +surfaces `RequestTimeTooSkewed` rather than the fault that opened the breaker, so the trace +misattributes a local problem to the remote one. + +**And the config recipe that fixes the analogous 401 case does not transfer.** +`verdict: { accept: [403], flag: 'ok' }` — which [scenario 9](resilience-has-no-tenancy.md) +measured working for a credential 401 — **swallows** the skew error here: `ok: true`, with +`RequestTimeTooSkewed` handed to the caller as data. The reason is `verdict.flag`'s three-state +absent rule (`surface.ts:174-191`): AWS error bodies carry no flag, and an absent flag is "no +signal". + +That is the **third sighting** of the absent-flag rule producing a silent success — see +[`body-verdict-footguns`](body-verdict-footguns.md) (a THROTTLED envelope returned as data) and +[`paginate-cannot-report-a-partial-run`](paginate-cannot-report-a-partial-run.md) +(`flag: 'UnprocessedItems'` inert because arrays are truthy). Six lines of `Surface.interpret` +fix it, as they did in both earlier cases. + +**Ask:** either a way to mark a status as "client fault, don't count it against the dependency", +or documentation that credential/clock 4xxs should be excluded from the breaker — the same ask as +scenario 9's §5, now with a second instance. + +## 4. Smaller findings + +- **`refresh` cannot see the response that triggered it.** Skew correction works — + `shouldRefresh`/`refresh` learned a 600,000 ms offset from the `Date` header and re-signed the + **same attempt** to a `200`, costing no retry budget — but the offset has to be smuggled out of + `shouldRefresh` through a closure, because `refresh(ctx)` receives no response. Passing the + triggering response to `refresh` would make this a clean 10 lines instead of 26. +- **A breaker does not shed a burst already queued behind a throttle.** The circuit phase is read + at `engine.ts:863`, before the throttle wait at `:629`. Measured: 4 concurrent calls all reached + the wire over 6 minutes _after_ the breaker opened; the same 4 issued sequentially stopped after 2. Defensible, and worth documenting — "the breaker gates entry, not the queue". +- **`backoff.max` defaults to 10 s**, so `base: '6m'` silently waits 10 seconds + (`resilience.ts:47,56`). Protective in this scenario — it cost the proofs a false negative — but + this is the **fourth** silent-clamp/silent-ignore in the pass (see + [`void-call-drops-work`](void-call-drops-work.md) §2). +- **A skew 403 is _not_ retried by default** — measured 1 request with `attempts: 4`, because 403 + isn't in the default `on` set. That is the correct behaviour and worth keeping. diff --git a/docs/scenarios/issue-drafts/sse-reconnect-replays-completed-streams.md b/docs/scenarios/issue-drafts/sse-reconnect-replays-completed-streams.md new file mode 100644 index 00000000..00cbcb50 --- /dev/null +++ b/docs/scenarios/issue-drafts/sse-reconnect-replays-completed-streams.md @@ -0,0 +1,113 @@ +# Issue draft — `sse: { reconnect: true }` replays a **completed** stream and delivers duplicated content + +**Status:** ✅ **FILED** as [#640](https://github.com/rejifald/StitchAPI/issues/640). Raised by the scenario pass on 2026-08-05. +**Scenario:** [`mid-stream-failure`](../mid-stream-failure.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `bug`, `sse`, `data-integrity` +**Affects:** resumable SSE as shipped in **#622** (two commits before this branch) + +> Highest-severity finding of the scenario pass. A single documented flag delivers **duplicated +> content to the end user** on a stream that never failed, and the run ends `ok: true`. It is +> in code that shipped days ago, so it is worth confirming before the next release rather than +> after. + +## Reproduce + +```bash +pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts +``` + +Block (a). No network — a fake adapter serving an OpenAI-shaped `text/event-stream`. + +## What happens + +A stitch with `sse: { reconnect: true }` against a **cleanly completing** OpenAI-shaped stream +(`data: {...}` frames with **no `id:`**, terminated by `data: [DONE]`): + +| | measured | +| -------------------------- | ---------------------------------------------- | +| opens | **4** | +| `[DONE]` sentinels seen | **4** | +| deltas delivered | **24** | +| text the consumer received | **`ABCDEABCDEABCDEABCDE`** | +| terminal event | **`done(ok: true)`** | +| `Last-Event-ID` sent | **never** — `[(none), (none), (none), (none)]` | + +The stream completed correctly on the first open. The library then requested the whole +completion three more times and concatenated the results into one uninterrupted delta stream, +which a UI renders as the answer repeating four times. Against a model API, those are three +billed completions nobody asked for. + +`sse: true` is the same flag (`stitch.ts:241-243`), so the shorthand carries it too. + +## Why — two independent defects that compose + +**1. `resumable` is decided from surface _capability_, before any frame is read.** + +``` +engine.ts:1288 resumable = policy.enabled && !!resumeToken && !!applyResume +``` + +`sseSurface` exposes both hooks unconditionally (`sse.ts:164-184`), so an **id-less** stream is +classified resumable. At reopen `lastToken` is `undefined`, the guard at `engine.ts:1344-1345` +skips `applyResume`, and the request goes out with no `Last-Event-ID` — i.e. a request for the +entire completion rather than a resumption. + +**2. A clean close is treated as a drop.** + +``` +engine.ts:1466 // 'closed' and 'error' take the same path +``` + +There is no "this stream is complete" signal, so `[DONE]` terminates nothing and the reconnect +budget is always spent in full. This is why a stream that never failed is reopened at all. + +Either defect alone is survivable. Together they turn a successful stream into four. + +## Suggested fixes + +- **`reconnect.requireToken`** (smallest useful fix): refuse to reconnect when no resume token + has ever been seen. This alone turns the id-less case from silent replay into a refusal, and + makes the resumable case unaffected. +- **`reconnect.onlyOnDrop`**: distinguish a clean body close from a transport drop, and stop + reconnecting once the stream has ended. This fixes the wasted-round-trip half, which also + affects the _resumable_ path — measured: a feed that never drops still opens **4×**, each + reopen replaying `Last-Event-ID: t5` (`c3-resumable-reconnect.ts`). +- Both spellings are machine-checked absent today (`@ts-expect-error` in `c8-connect-vs-body.ts` (f)). + +Longer term, a surface-declared "the stream is finished" signal would let `[DONE]` mean what it +says, rather than the engine inferring completion from a closed socket. + +## Docs that need a caveat regardless of the fix + +`apps/docs/content/docs/reference/surfaces.mdx:83-104` says a dropped stream is reopened, and +never mentions that a **finished** one is too, or that the feature requires the server to emit +`id:`. As written, a reader with an OpenAI-shaped stream has every reason to turn it on. + +--- + +## Related findings from the same verification (separate asks) + +These are not the bug above, but they surfaced alongside it and shape the same scenario. + +- **`retry` does not run on a streaming stitch at all.** Measured: `retry: { attempts: 4 }` + against an always-503 server → **4 requests on a buffered stitch, 1 on an `sse` one**, + `error.attempts: 1`. `runStreaming` (`engine.ts:1248`) has no attempt loop. That means the + one replay that is unambiguously safe — the connect phase, before any byte has flowed — is + the one case the retry config cannot express. +- **`retry.attempts` is inert while `retry.backoff` is live** on the same stitch: `backoff` + supplies the _reconnect_ curve. `retry: { attempts: 1 }` still produced 4 opens. Two knobs + under one name capping different things is worth either separating or documenting. +- **The default reconnect backoff is ~50 ms** (`expo-jitter` off base 100, `resilience.ts:39-56`), + so a dropped stream replays almost immediately unless a `retry.backoff` is authored. +- **`interpret` and `verdict.flag` are dead code on streaming surfaces.** A custom surface's + `interpret` ran **zero** times; `runStreaming` calls only `classifyStatus` (`engine.ts:1371`). + Both typecheck. The hook whose stated job is "this 200 is really a failure" is unavailable in + the one place where every failure is a 200. +- **`hooks.onError` never fires for a post-200 stream failure** — measured `[onRequest, +onResponse]` on a failing run. Any hook-based error pipeline is blind to mid-stream failure. +- **The partial is discarded one line before it could be returned.** `engine.ts:1467-1472` + returns before reaching `resultEvt(chunks, …)` at `:1492`, so the buffered accessors get + nothing: `.safe().data` null, `StitchError.body/data/partial/chunks` all undefined, and + `.inspect()` reports `status: 0` — it does not even record that a 200 arrived. +- **`verdict: { accept: [503] }` + `reconnect` resolves a permanently-503 server successfully** + with `data: []` (`c8-connect-vs-body.ts` (d)). diff --git a/docs/scenarios/issue-drafts/streaming-is-not-memory-bounded.md b/docs/scenarios/issue-drafts/streaming-is-not-memory-bounded.md new file mode 100644 index 00000000..c3de03ef --- /dev/null +++ b/docs/scenarios/issue-drafts/streaming-is-not-memory-bounded.md @@ -0,0 +1,129 @@ +# Issue draft — `.stream()` is not memory-bounded, and `decode: 'json'` buffers the array it streams + +**Status:** ✅ **FILED** as [#659](https://github.com/rejifald/StitchAPI/issues/659) +**Scenario:** [`large-response-memory`](../large-response-memory.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `bug`, `stream`, `memory` + +> Two defects, both located to a specific line, both with a control proving the surrounding code +> is fine. Together they mean the library ships an O(1) NDJSON decoder that no configuration can +> actually benefit from. + +Reproduce (needs `--expose-gc`): + +```bash +for f in docs/scenarios/proofs/large-response-memory/c[0-9]*.ts; do pnpm exec tsx --expose-gc "$f"; done +``` + +--- + +## 1. The engine retains every chunk, so no accessor is memory-bounded + +**Severity: high — it defeats the entire purpose of the streaming path.** + +`engine.ts:1443` pushes every delta onto a `chunks` array unconditionally, so the terminal +`result` can mirror the whole spine. The line is honest about it — there is a MEMORY NOTE at +`:1437-1442` — but the mitigation it recommends does not work: + +> _"A consumer of an unbounded stream should read the `delta` events incrementally (via +> `.stream()`)…"_ + +Measured: **`.stream()` 30.2 MB vs `await` 33.5 MB** for the same 100k rows. The accumulator is +_inside_ the generator both accessors drain, so iterating escapes nothing, and no option +disables it. + +The waste is precise, because the decoder underneath is excellent: + +| | retained heap | +| ------------------------------------------------------------------ | ------------------------------------------------------- | +| `ndjson` decoder driven **directly**, 1,000,000 rows / 214 MB wire | **0.8 MB** (<15% movement over a 1000× workload change) | +| the same decoder **through the engine**, 10k → 100k rows | 3.5 MB → **30.2 MB** (linear) | + +**Ask:** make retention opt-out, or automatic when the caller uses `.stream()`. Something like +`stream: { retain: false }`, or simply not accumulating when the consumer is iterating rather +than awaiting. Today the O(1) decoder is unreachable from any configuration — the only way to +get a bounded export is a custom `Surface.stream` that yields a _receipt_ per batch instead of a +row per row (75 lines; measured **1.3 MB flat for 100,000 rows**, a 40× cut). + +At minimum the MEMORY NOTE should be corrected — it currently points readers at a mitigation +that measures identically to the thing it's mitigating — and the claim at `engine.ts:1242-1247` +that `.stream()` "buffers nothing" should be qualified: true about latency, false about memory. + +## 2. `decode: 'json'` buffers the whole array it is streaming + +**Severity: high — it fails on its own default, and the message blames the vendor.** + +Emission is **correct**, and worth saying so: one delta per top-level element, holding up under +`,` `]` `}` inside string values, escaped quotes, embedded newlines, pretty-printed multi-line +records, deep nesting, and 1-character chunk boundaries. The parser is good. + +Memory is not. Retained heap tracks the **whole array text** at 0.88× the wire, with 34× growth +over a 100× workload, and time is **quadratic** (28× for 10× the rows). + +**Root cause, one branch:** `compact()` floors on `valueStart`, and for a top-level array +`valueStart` is set to the opening `[` and only reset at the closing `]` +(`json-stream.ts:157-171`, `:173-195`, `:227-238`). So `compact(live)` is a no-op for the entire +array and nothing is ever released. + +**The control proves it is not a design limit:** the same 100,000 records as **concatenated +top-level values** — the other thing this decoder accepts — run **flat at 0.9 MB**. + +**The consequence is a silent truncation.** The decoder trips its own 8,388,608-char default +(`json-stream.ts:20-27`, `:89-97`): 37,000 rows decode, 38,000 fail. On a 60,000-row array the +consumer receives **37,312 rows, then `error` / `done(ok: false)`** — invisible to any loop that +only matches `delta`. And the message, _"a malformed or never-closing value was streamed"_, +accuses the vendor of something it did not do. + +**Ask:** reset `valueStart` per top-level element inside an array, the way the concatenated-value +branch already does. Failing that, the cap message should distinguish "this value is too large" +from "this array is too long", because they need opposite fixes — and raising the cap converts +the failure into the memory profile the user was trying to avoid (18.8 MB retained, **405 MB peak +allocation** for a 21 MB body). + +## 3. Buffered/streaming path asymmetries, all silent + +Scenario 5 found `retry` inert and `interpret` never called on `runStreaming`. Three more: + +- **`pick` and `transform` do not run on a stream.** `transform` called **zero** times over 200 + deltas; `pick: 'id'` left the whole row. Both behave normally on the buffered path. No `info`, + no drift finding, no throw — and the static delta type derives from `output`, never `pick`, so + the call site doesn't catch it either. A stitch carrying them that is later switched to + `kind: stream` keeps compiling and quietly stops reshaping. +- **`output` on a stream validates but does not transform.** `engine.ts:1419` keeps only + `{ errors }` where `engine.ts:1223` on the buffered path serves the validated value. A coercing + schema reshapes your data on `await` and silently does not on `.stream()`. +- **`stream({ kind: mySurface })` silently drops the surface** — `stream()` overwrites `kind` + after spreading the caller's config (`stream.ts:138-146`; `sse.ts:205-212` has the same shape). + Measured: 1,000 raw rows, **0** through the surface, no error. It must be `stitch({ kind })`. + +The engine already has a precedent for warning about an ignored config slot — an upload-progress +bar the transport can't draw emits an `info` event (`engine.ts:1694-1698`). These four slots get +nothing. + +## 4. Nothing guards the buffered path + +A 21 MB response emits exactly `start`, `progress:request`, `result`, `done` — the same four +events a 40-byte one emits. No threshold, no `info`, no drift finding. + +When the wall arrives it is V8's, not the library's: 400,000 rows under a 96 MB heap gave +`FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory`, **exit 134 +(SIGABRT)** — no catchable error, no `error` event, no `finally`. The same rows batched over +`ndjson` under the same ceiling: **1.3 MB, every row processed.** + +And the one config slot with `buffer` in its name is **accepted on a buffered stitch and does +nothing** — `stream: { buffer: { chars: 1_000 } }` typechecked, composed, and delivered all +50,000 rows of an 11-million-character body. + +**Ask:** a response-size threshold that emits an `info`/drift finding on the buffered path would +turn an unattributable 3am `SIGABRT` into a signal, without changing any behaviour. + +## 5. Two things that are fine, recorded so the fix doesn't chase them + +- **`output` is innocent.** It runs **per delta** — 500 calls for 500 records, each carrying one + object, `sawArrayOfLength: 0`, even when the wire is literally one array. Heap cost within 1%. + The capture predicted it would re-buffer; it does not. +- **Backpressure propagates properly.** A consumer awaiting a macrotask per row kept the producer + within 8 chunks of 32; a lazy producer's queue stayed at 1.3% of the body. The chain is + pull-based end to end. (Note for anyone measuring: a backlogged socket is invisible to + `heapUsed` — it's external memory, visible only in `arrayBuffers`.) +- **The buffered multiplier is `JSON.parse`'s, not the library's** — measured 2.5×, and + StitchAPI's overhead over a bare `JSON.parse` of the same bytes was **0.2 MB**. diff --git a/docs/scenarios/issue-drafts/testing-kit-clock-gaps-and-two-bugs.md b/docs/scenarios/issue-drafts/testing-kit-clock-gaps-and-two-bugs.md new file mode 100644 index 00000000..036caeea --- /dev/null +++ b/docs/scenarios/issue-drafts/testing-kit-clock-gaps-and-two-bugs.md @@ -0,0 +1,141 @@ +# The testing kit's clock covers half its own features — plus two bugs + +**Status:** ✅ **FILED** as [#650](https://github.com/rejifald/StitchAPI/issues/650) +**Scenario:** [stale-fixture](../stale-fixture.md) +**Proofs:** `docs/scenarios/proofs/stale-fixture/` (8 scripts, 168 checks, offline) + +## First, what works + +Worth stating because a fix shouldn't disturb it. Resilience is **fully** testable with no vendor +and no real waiting: attempt counts assertable three ways, the whole circuit +closed→open→half-open→closed trace readable from `callCount()` as `1, 2, 2 (blocked), 3, 4`, +throttle spacing exact and self-reporting (`waited` = `500` then `1000` for `'2/s'`), backoff +curves exact at virtual `0 / 1000 / 3000`. Streams are byte-identical across five repeat runs. +That is a genuinely good testing story. + +The findings below are the edges. + +## 1. `manualClock` drives six time-driven features and not the other six + +| Feature | Driven by | Measured | +| ----------------------------- | -------------- | --------------------------------------------------------------- | +| `retry` backoff | `manualClock` | `advance(5000)` → 3 calls; `advance(0)` → 1, pending | +| `throttle` rate | `manualClock` | `advance(3000)` → 3 calls | +| `throttle` concurrency | `manualClock` | holder releases on virtual time | +| `circuit.cooldown` | `manualClock` | `advance(60_000)` past a 30s cooldown → half-open probe | +| `timeout` (per-attempt) | `manualClock` | `advance(2000)` past a 1s timeout → error | +| `Retry-After` | `manualClock` | honours it — **and that is a trap**, see §2 | +| **`timeout.total`** | **wall clock** | a 1000ms budget survived **2700 virtual ms**, `ok: true` | +| **`cache.ttl`** | **wall clock** | `advance(600_000)` past a 60s TTL still served the cached entry | +| **`memoryStore` TTL** | **wall clock** | a 1s entry survived 60,000 virtual ms | +| **event `at`/`done.elapsed`** | **wall clock** | `at = 1785941469178` while `clock.now() = 0` | +| **OAuth2 token expiry** | **wall clock** | 600,000 virtual ms past a 60s `expires_in` refetched nothing | +| **AWS SigV4 signing date** | **wall clock** | `new Date()`, no `clock` option exists | +| `paginate` | no time | nothing to drive | + +ADR 0010 §4 documents **four** of these as deliberate — `timeout.total`, event `at`/`done.ms` +and `memoryStore`/cache TTL — and `types.ts:1476` repeats the event one in JSDoc. **SigV4 and +OAuth2 token expiry are documented nowhere.** OAuth2 is the one worth acting on, because it is in +core and looks unintentional rather than scoped out: +`packages/core/src/auth.ts:502` is + +```ts +const isFresh = (t: CachedToken | undefined): boolean => + !!t && (t.expiresAt === 0 || now() < t.expiresAt - skew); +``` + +where `now` is a module-level import. `auth.ts` contains **zero** occurrences of `clock`, and +`AuthContext` carries none — so this is not one line reaching for the wrong function, the +plumbing to do otherwise doesn't exist. "Does my client refresh the token before it expires" is a +thing people write tests for, and today that test cannot be written on a virtual clock. + +**One correction to how this is usually described**, including in our own capture: `timeout.total` +does not _ignore_ the clock. The per-attempt clamp fires on virtual time +(`withTimeout(..., rt.clock)`, `engine.ts:669-674`); the wall-anchored part is the **deadline** +(`wallT0 + total`). Virtual sleeps never drain it, so the budget **resets** each attempt rather +than being ignored. On a real clock it behaves correctly — this is specifically a testing +unsoundness. + +ADR 0010 also closes with _"Follow-ups (out of scope here): driving `timeout.total`, event +timestamps, and store/cache TTL off the clock, **should a concrete need arise**."_ The measured +vacuous pass is that concrete need. + +**Ask:** route OAuth2 expiry through the injected clock; and put the table in the mocking guide — +an ADR section and a `types.ts` JSDoc are not where someone writing a test will look. + +## 2. `Retry-After` as an HTTP-date is a trap _because_ it honours the clock + +`parseRetryAfter` computes `httpDateEpoch - clock.now()` (`resilience.ts:70`), and +`manualClock()` starts at `0`. So a fixture carrying a normal dated `Retry-After` meaning "5 +seconds" becomes a wait of roughly **20,000 days**. + +This is the only row where doing the right thing produces the worse outcome, and it will read as +a hang rather than as a bug. + +**Ask:** default `manualClock()` to a realistic epoch, or warn when a parsed `Retry-After` exceeds +some sane ceiling. + +## 3. BUG — `stubStitch(...).safe()` throws on a synchronous throw + +``` +stub, impl throws synchronously -> THREW boom +stub, impl rejects asynchronously -> ok=false +REAL stitch, adapter throws sync -> ok=false / "transport boom" +``` + +`.safe()` is the never-throws accessor, and here it throws — diverging from both its async twin +and the real stitch. `resolve()` (`packages/core/src/test-stub.ts:59-63`) evaluates `impl(input)` +as an **argument** to `Promise.resolve`, so the throw escapes before there is a chain to catch it: + +```ts +Promise.resolve( + typeof impl === 'function' ? (impl as ...)(input) : impl, +); +``` + +`.stream()` on the same stub is unaffected. A one-line fix (`async` wrapper, or move the call +inside a `.then`). + +## 4. BUG — `mockAdapter` fails the library's own adapter contract + +Running `verifyAdapterContract` against `mockAdapter` passes 8 of 9 rules and violates +**`abort: a pre-aborted signal rejects`** — _"adapter resolved although the signal was already +aborted"_. + +Cause: `req.signal` is consulted only inside the `delay` branch +(`packages/core/src/test-mock.ts:188-189`), so any route **without** a `delay` ignores an aborted +signal that every real transport honours. Any test of cancellation behaviour against a +delay-less route is therefore testing the opposite of production. + +## 5. Smaller, same area + +- **`mockAdapter` validates almost nothing about a fixture.** It served nonsense statuses + (`999`, `-1`, `0`, fractional) and bodies of type `Date`, `Map`, class instance, `undefined`, + `function`, `bigint` and Symbol-keyed, all verbatim through the full engine. The sharp consequence: a + fixture built from `new Invoice(...)` gives the caller `data.total === 42` from a **prototype + getter**, where the same object over a JSON wire is `{"id":"inv_1"}` and `data.total` is + `undefined`. A green test for code that cannot work. A `wireShape` opt-in would cost little. +- **`stubStitch` runs none of the `input` schemas.** `{ params: { id: 42 } }` against + `z.string()`: the real stitch errors with no request sent; the stub resolves. There is no slot + to give it the contract — `StubStitchOptions.config` is `Partial`, which + carries no schema. +- **Retry backoff delays are absent from the event stream.** `progress{phase:"retry"}` carries + `waited: undefined`, while the throttle (`engine.ts:641`) and reconnect (`:1486`) paths set it. +- **`done.elapsed` is wall-clock**, so it reads `0` after any virtual time — the kit's own + duration field cannot see the kit's own clock. +- **`stubStitch(...).with()` returns a new stub with a fresh spy**, so the parent's `callCount()` + reads `0` after a bound call. +- **The circuit breaker has no distinct error name** — a plain `StitchError` with message + `"circuit open"`, so assertions have to match on the string. + +## Not asked for here + +The scenario's own question — detecting that a _fixture_ has gone stale while the _vendor_ moved — +is a capability gap rather than a bug, and ADR 0015 removed snapshot drift deliberately ("a single +snapshot is one observation"). Recording it separately. + +--- + +_Found by an automated scenario pass. Line references verified against `main` at the time of +filing. Runnable proof scripts live under `docs/scenarios/proofs/stale-fixture/` on the branch +`claude/api-integration-scenarios-436a38`._ diff --git a/docs/scenarios/issue-drafts/void-call-drops-work.md b/docs/scenarios/issue-drafts/void-call-drops-work.md new file mode 100644 index 00000000..3dbfda45 --- /dev/null +++ b/docs/scenarios/issue-drafts/void-call-drops-work.md @@ -0,0 +1,93 @@ +# Issue draft — `void call(input)` silently drops the work, and `backoff.base` is clamped without warning + +**Status:** ✅ **FILED** as [#660](https://github.com/rejifald/StitchAPI/issues/660) +**Scenario:** [`webhook-receipt`](../webhook-receipt.md) +**Suggested template:** bug_report.yml · **Suggested labels:** `dx`, `footgun` + +> The scenario itself resolved cleanly as a **documented boundary** — inbound receipt is out of +> scope by design, and [the-stitch.mdx:44](../../apps/docs/content/docs/concepts/the-stitch.mdx) +> already says so. These two findings are independent of that and apply to any stitch. + +Reproduce: + +```bash +for f in docs/scenarios/proofs/webhook-receipt/c[0-9]*.ts; do pnpm exec tsx "$f"; done +``` + +--- + +## 1. `void call(input)` makes no request and reports nothing + +**Severity: high — silent data loss, and the spelling is the idiomatic one.** + +A stitch call is a **lazy thenable**: the run starts on `.then` (`stitch.ts:729,781`). So the +natural fire-and-forget spelling does nothing at all. + +Measured: `void call(input)` produced **0 HTTP calls and 0 errors**. No request, no event, no +rejection, no trace — the call simply never happened. + +This matters most in exactly the place people write it. In a webhook handler the shape is "ack +the provider fast, do the work after", so `void call(input)` goes in right after +`res.writeHead(200)` — and the work is dropped _after_ the sender has been told it succeeded. +The provider will not retry, because you said 200. + +Adjacent behaviours measured alongside: + +- `void call(input).then(…)` **does** run it (1 call), and is then unsupervised: unhandled, it + surfaced as **1 `unhandledRejection`**. +- `call.safe()` in that position produced **0 rejections** and reported the failure **nowhere** — + no throw, no event, no sink. + +**Ask:** the lazy thenable is a deliberate and defensible design (it is what lets `.stream()`, +`.safe()` and `.inspect()` branch off one expression), so the fix is probably not to make it +eager. But _something_ should mark the discard: + +- a lint rule or a `no-floating-stitch`-style type trick, or +- a `.detach()` / `.start()` that makes "run it and don't await" explicit and supervised, or +- at minimum, a prominent line in the pitfalls page. `void x()` is a well-known idiom for + "deliberately not awaiting"; here it means "deliberately not running", which is the opposite. + +## 2. `backoff.base` is silently clamped by `backoff.max` + +**Severity: low — but it is a silent policy downgrade, and the third of its kind in this pass.** + +`backoff: { base: 30_000 }` measured an actual sleep of **10,000 ms** — `max` defaults to 10 s +(`types.ts:972-973`) and silently wins over an explicitly authored `base`. + +A user who writes `base: 30_000` has clearly stated an intent; getting a third of it with no +diagnostic is the same class as the two already filed — a `backoff` function that +[vanishes when cast past](body-verdict-footguns.md), and `retry.attempts` +[being inert on a stream while `retry.backoff` is live](sse-reconnect-replays-completed-streams.md). + +**Ask:** either raise `max` implicitly when `base` exceeds it, or warn at construction. The +precedent is already in the codebase: an unparseable `throttle.rate` **throws** at construction +rather than degrading. + +--- + +## 3. Smaller notes from the same verification + +- **`serve` is unauthenticated by design** (`serve.ts:28,59`) and this is worth stating louder + in the surfaces docs than it currently is. Measured: an **unsigned, forged body** under the + size cap ran the stitch and returned 200. It is a local front door; anyone who can reach the + port can run any registered stitch. The failure mode of someone mistaking it for a webhook + endpoint is not a 404 — it is an open endpoint. +- **`engine.ts:287` exports `RAW_BODY`**, which is the raw **response** body of an **outbound** + call. Anyone grepping "raw body" while debugging an inbound signature failure lands on the + exact opposite thing. Worth a doc comment noting the direction. +- **`xxh128` (`hash.ts:110-113`) is unkeyed and non-cryptographic** — measured arity 1, same + digest with no secret. It is correctly documented, but it is the nearest-looking primitive to + "hash the payload", and a verification built on it authenticates nobody while appearing to + work. +- **`idempotency` is outbound-only** — measured putting `Idempotency-Key` on an outgoing POST. + It shares a word with inbound event dedup, which is a different problem at the other end of + the pipe. A cross-reference from the idempotency guide to the store would help. +- **`memoryStore.close()` is `data.clear()`** (`store.ts:59-61`), so any ledger built on the + default store does not survive a restart — and a deploy inside a provider's retry window + re-processes everything still in flight. The swap to a durable store is genuinely one line; + the default just needs to be understood as ephemeral. +- **`get`-then-`set` is not a claim.** Measured with 3 concurrent deliveries of one id: + `get`+`set` → `[true, true, true]` (three side effects); `increment(key, ttl)` → + `[true, false, false]`. `increment` is the atomic primitive and deserves to be the documented + way to build a dedup ledger — this is the same "the store has the right primitive but nothing + points at it" shape as [`cache-cannot-revalidate`](cache-cannot-revalidate.md). diff --git a/docs/scenarios/large-response-memory.md b/docs/scenarios/large-response-memory.md new file mode 100644 index 00000000..d9846e11 --- /dev/null +++ b/docs/scenarios/large-response-memory.md @@ -0,0 +1,144 @@ +# Scenario: the 84 MB response that took 2.1 GB of heap + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `large-response-memory` + +**Verification:** 8 proof scripts (97 checks), run offline, one process per measurement, +`peakLive` = high-water `heapUsed` after a forced GC. Requires `--expose-gc`. In +[`proofs/large-response-memory/`](proofs/large-response-memory/). Published page: +[`scenarios/large-response-memory.mdx`](../../apps/docs/content/docs/scenarios/large-response-memory.mdx). +Escalated: [`issue-drafts/streaming-is-not-memory-bounded.md`](issue-drafts/streaming-is-not-memory-bounded.md). + +| Claim | Verdict | Measured | +| ----------------------------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — buffered baseline | linear, and **fair to the library** | 21.4 MB wire → **53.8 MB** retained (2.5×), matching a bare `JSON.parse` to within **0.2 MB**. The multiplier is `JSON.parse`'s | +| C2 — `ndjson` flat? | **NO — and the two halves are in one call** | decoder driven directly: **0.8 MB for 1,000,000 rows**. Through the engine: **3.5 → 30.2 MB** (linear). `.stream()` 30.2 vs `await` 33.5 — the same number twice | +| C3 — `decode: 'json'` on one array | streams the parse, **buffers the text** | emission correct under every adversarial case; heap tracks the whole array at **0.88× wire**, time quadratic; trips its own cap at **37,000 rows** and blames the vendor | +| C4 — does `output` re-buffer | **capture REFUTED — innocent** | 500 calls for 500 records, each one object, `sawArrayOfLength: 0`; heap within 1% | +| C5 — `pick`/`transform` on a stream | neither runs | `transform` called **0** times over 200 deltas; completely silent | +| C6 — backpressure | propagates; the cap throws | producer stayed within 8 chunks; 20,000 rows streamed through a **1,000-char** cap while the engine kept all 20,000 | +| C7 — guard on the buffered path | **none** | 400,000 rows under a 96 MB ceiling → `FATAL ERROR … heap out of memory`, **exit 134**, no catchable error, no `finally` | +| C8 — assembled | PASS | **1.3 MB flat vs 53.8 MB** — a 40× cut — via `Surface.stream` yielding a receipt per batch. 75 lines + 4 config vs 67 hand-rolled | + +**One hypothesis refuted in the library's favour, one confirmed worse than feared.** `output` +does _not_ re-buffer a stream — it validates per delta and costs nothing. But the memory the +capture went looking for is spent somewhere it didn't think to look: `engine.ts:1443` retains +every chunk unconditionally, so **the library owns a genuinely O(1) NDJSON decoder and spends +the win one line later** — and `.stream()`, the mitigation the engine's own MEMORY NOTE +recommends, measures identically to `await`. + +**The `decode: 'json'` defect is one branch, not a design limit.** `compact()` floors on +`valueStart`, which for a top-level array is the opening `[`, so nothing is released until `]`. +Control: the same 100,000 records as _concatenated top-level values_ run **flat at 0.9 MB**. + +--- + +## The use case + +You call an export endpoint — a product catalog, a transaction ledger, a bulk query result. +The vendor hands back **one JSON array** with tens of thousands of rows. You `await` it, parse +it, and iterate. + +Then one day the catalog grows, and the process dies. + +## Why it is not straightforward + +**Parsing costs several times the wire size.** `JSON.parse` has to hold the whole string _and_ +build the complete object tree, and the tree is commonly 2–5× the raw bytes. In practice it is +worse than the rule of thumb: a documented sync of **22,000 products in an 84 MB response +exhausted ~2.1 GB** on a 4 GB VPS — roughly **25×**. After restructuring to a streaming parse +with batching, the same sync ran in 4 minutes at **180 MB peak**. + +This failure mode is different from every other scenario in this section: nothing returns wrong +data. The process simply dies, usually in the middle of the night, usually after the dataset +crossed a threshold nobody was watching. + +The awkward parts: + +- **You can't size it in advance.** A chunked response has no `Content-Length`, so "check the + size first" isn't available. And the threshold moves as the customer's data grows. +- **A single top-level JSON array is the hard case.** NDJSON streams trivially — one line, one + record, discard, repeat, ~1 MB peak for a 1 GB file. A single `[ {...}, {...}, … ]` cannot be + split on newlines; it needs a _structural_ parser that emits each top-level element. +- **Validation quietly re-buffers.** Streaming the rows and then validating "the result" puts + the whole array back in memory. Any per-response contract defeats the streaming it sits above. +- **Backpressure is the second collapse.** If the consumer is slower than the producer and the + code ignores the signal to slow down, Node buffers chunks until the process falls over — the + memory arrives from the _other_ direction. +- **The fix changes the shape of your code.** Buffered code says `const rows = await get()`. + Streaming code says `for await (const row of get.stream())`, and everything downstream — + batching, transactions, error handling — has to change with it. + +## Evidence this bites real projects + +- **The 84 MB → 2.1 GB incident** and its 180 MB streaming fix are documented in + [Handling massive JSON payloads without crashing your workflow runner](https://triumphoid.com/handle-massive-json-payloads-without-crashing-workflow-runner/). +- **The parse multiplier** — an object tree 2–5× the raw string — is the consistent number in + [Memory-safe large JSON streaming](https://www.technetexperts.com/memory-safe-json-streaming-node-bun/) + and [parsing large JSON in Node](https://salivity.github.io/node.js/article/parsing-large-json-in-node-js-performance-impacts). +- **NDJSON's O(1) property** — ~1 MB peak for a 1 GB, million-record file — is the standard + contrast ([Jsonic on JSON streaming](https://jsonic.io/guides/json-streaming)). +- **Backpressure blindness** is its own documented collapse: + [your Node.js streams aren't backpressuring, they're silently eating your memory](https://frontendmasters.com/blog/your-node-js-streams-arent-backpressuring-theyre-silently-eating-your-memory/). +- **JSONStream #101** — ["Node process goes out of memory while parsing large JSON files"](https://github.com/dominictarr/JSONStream/issues/101) — + is the same complaint against the streaming library itself. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ----------------------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| **`await res.json()`** | The default. | Dies above some size you cannot predict, and the multiplier is ~25× in the field, not the 2–5× folklore. | +| **Ask the vendor for NDJSON** | Newline-delimited, streams trivially. | The correct fix where offered — Shopify bulk results, some exports. Most REST endpoints don't. | +| **Structural streaming parser** (`stream-json`) | Emits each top-level array element. | The real answer for a single giant array. A dependency, and the pipeline is fiddly to assemble. | +| **Paginate instead of exporting** | Ask for pages, not the whole thing. | Bounded memory — and inherits every problem in [the pagination scenario](unstable-pagination.md), plus N× the requests. | +| **Raise the heap** (`--max-old-space-size`) | Buy headroom. | Moves the cliff without removing it, and the cliff moves toward you as data grows. | +| **Batch + null out references** | Process N at a time, release. | Necessary alongside streaming; useless on its own if the parse already buffered. | + +**Summary of the state of the art:** stream structurally, batch the consumer, keep validation +per-record rather than per-response, and honour backpressure. The distinguishing property is +that the _default_ path is the dangerous one, and it works fine until it doesn't. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- The `stream` surface takes `decode: 'bytes' | 'lines' | 'ndjson' | 'json'`, and `'json'` is + documented as _"the structural, unframed streaming-JSON decoder (issue #111): one `delta` per + complete value / top-level array element, tolerant of internal newlines and concatenated + values"_ (`types.ts:1441-1450`). **If that holds, the hard case — one giant array — is a + config value**, which would make this one of the few scenarios with a genuine built-in answer. +- `stream.buffer.chars` exists as a cap. What happens _at_ the cap is the open question: does it + throw, truncate, or apply backpressure? +- **The validation question is the sharp one.** Scenario 12 measured the engine serving the + _validated_ value, and scenario 11 measured `output` running over the whole aggregated array. + If `output` on a streaming stitch buffers every delta to validate the aggregate, the streaming + is undone — and the config would look correct. +- Scenario 5 measured the streaming path is a different engine (`runStreaming`) with different + rules — `retry` inert, `interpret` never called. Expect more asymmetries here. + +**Claims to test with runnable offline code:** + +1. **C1** — measure the baseline. `await` a large JSON body and record the **heap high-water + mark** against the wire size. Confirm the multiplier on this runtime. +2. **C2** — `stream` with `decode: 'ndjson'`: does heap stay flat as the body grows? Measure + peak against 1×, 10× and 100× row counts. +3. **C3** — **DECIDING CLAIM.** `decode: 'json'` against **one single top-level array**. Does it + emit one delta per element without buffering the whole array? Measure peak heap, and confirm + the element count and boundaries are right (including elements containing internal newlines). +4. **C4** — **DECIDING CLAIM.** Add an `output` schema to a streaming stitch. Does validation run + **per delta** or over the **aggregate**? Measure peak heap with and without it. If it + re-buffers, that is the finding. +5. **C5** — `pick` / `transform` on a streaming stitch: per-delta or buffering? +6. **C6** — backpressure: a slow consumer against a fast producer. Does the stream buffer + unboundedly? What does `stream.buffer.chars` do at the cap — throw, truncate, or block? +7. **C7** — is there any guard on the **buffered** path? A response larger than some limit on a + plain `await` — does anything intervene, or is OOM the only signal? +8. **C8** — assemble the best available answer: stream, validate per record, batch the consumer. + Report the seam and line count, and measure peak heap against the buffered baseline. + +C3 and C4 decide this. A structural JSON decoder that genuinely streams a single array would be +a real capability few clients have — and an `output` schema that silently re-buffers it would +hand the memory straight back. diff --git a/docs/scenarios/mid-stream-failure.md b/docs/scenarios/mid-stream-failure.md new file mode 100644 index 00000000..3184c238 --- /dev/null +++ b/docs/scenarios/mid-stream-failure.md @@ -0,0 +1,148 @@ +# Scenario: a stream that fails after you've already shown the user 800 tokens + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `mid-stream-failure` + +**Verification:** 9 proof scripts, run offline (171 checks), in +[`proofs/mid-stream-failure/`](proofs/mid-stream-failure/). Published page: +[`scenarios/mid-stream-failure.mdx`](../../apps/docs/content/docs/scenarios/mid-stream-failure.mdx). +Escalated — **the pass's most severe finding, in code shipped in #622**: +[`issue-drafts/sse-reconnect-replays-completed-streams.md`](issue-drafts/sse-reconnect-replays-completed-streams.md). + +| Claim | Verdict | Measured | +| ------------------------------------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — drop mid-body, no `[DONE]` | both, and they disagree | `.stream()` sees `error` as an EVENT not a throw; `await` gives `ok:false, data:null`; a **clean** close mid-answer is identical in shape to success | +| C2 — does `retry` re-emit deltas | **no — refuted** | `ABC` once. `retry` never runs on a stream: 4 requests buffered vs **1** streaming, `attempts: 1` | +| C3 — `reconnect` on a resumable feed | **PASS — just works** | `ABCDE`, zero duplication, `Last-Event-ID` `(none) → t2 → t5`; server `retry: 9000` honored | +| C4 — `reconnect` on an id-less stream | **BUG** | a **completed** stream reopened **4×**, `ABCDEABCDEABCDEABCDE` delivered, `done(ok: true)` | +| C5 — in-band error frame at 200 | only via `output` | custom `interpret` ran **0 times**; `verdict.flag` inert; `onError` never fires post-200 | +| C6 — missing `[DONE]` | no built-in; 8 lines | truncated and complete streams share the terminal spine `result, done(ok:true)` | +| C7 — is the partial reachable | `.stream()` only | `.safe().data` null; `StitchError.body/data/partial/chunks` undefined; `.inspect()` says `status: 0` | +| C8 — connect-retry vs body-retry | not in config | one flag governs both; `Surface.execute` does it in 10 lines | +| C9 — assembled | PASS | **62 lines vs 83** hand-rolled — the first scenario where StitchAPI is _smaller_ | + +**Two hypotheses refuted, and the second one matters.** The capture nominated C2 as a deciding +claim — "does `retry` re-emit already-seen deltas into a downstream accumulator?" The answer is +no, and for a reason the capture didn't anticipate: `retry` doesn't run on streams at all. But +the duplication hazard the capture was hunting for **is real** — it just comes from +`sse.reconnect`, not `retry`, and it fires on streams that never failed. + +**The split verdict is worth keeping.** C3 alone is genuinely ACHIEVABLE — for a feed that +emits `id:` and honors `Last-Event-ID`, `reconnect: true` is one flag and it is correct. That +is the first clean built-in win in five scenarios. The overall verdict is only "with user code" +because the same flag is actively harmful everywhere else, and C5–C8 each need code. + +--- + +## The use case + +You stream an LLM completion to a user — OpenAI, Anthropic, OpenRouter, a self-hosted model. +Tokens arrive over SSE and render as they come. Eight hundred tokens in, the stream stops. + +Not "the request failed". The request **succeeded**: `200 OK` went out with the first token, +and the user is looking at most of an answer. + +## Why it is not straightforward + +**Once the first byte is written, the status line is spent.** HTTP 200 and the headers are +committed before anything goes wrong, so every failure after that point has to arrive +_in-band_ — as an SSE `error` frame, or as nothing at all when the connection simply drops. +Status-code logic is structurally unable to see it. (This is the fourth scenario in this +section where the failure signal lives below the status line.) + +**Retrying is not free, and not neutral.** Replaying the request re-runs the model: you pay +for the first 800 tokens _and_ the replacement, since tokens generated before a mid-stream +failure are still billed. Worse than the money, a retry **duplicates content the consumer has +already accumulated** — the user watches the answer restart, or the accumulator ends up with +800 tokens of prefix twice. Any retry policy applied to a stream has to answer "what happens +to the deltas already emitted?", and most just don't. + +**"Ended" and "ended early" look identical.** A cleanly finished OpenAI stream is terminated +by a `[DONE]` sentinel. A truncated one just... stops. Without checking for the sentinel there +is no way to distinguish a complete answer from a severed one — and the severed one arrives +with `ok`, because the transport was fine. + +**Resumption mostly isn't offered.** SSE has a resume mechanism — `Last-Event-ID` — but it +requires the server to put `id:` on each frame and to honour the header on reconnect. +OpenAI-style completion chunks carry no `id:` at all, so the standard mechanism does not apply +to the most common streaming API in the world. + +**The partial is often still valuable.** 800 of 1000 tokens is usually worth showing, saving, +or feeding to a repair prompt. Discarding it because the call "failed" throws away work that +was already paid for. + +## Evidence this bites real projects + +- **OpenRouter** — [error handling guide](https://openrouter.ai/docs/api_reference/errors-and-debugging): + once the first token is written the 200 is committed, so provider disconnects, timeouts, + content filters and overloads _must_ arrive in-band as SSE events. +- **openai-node** — [`#257`](https://github.com/openai/openai-node/issues/257): usage missing + on streamed responses, so a client cannot even reconcile what it was billed for. +- **openai-go** — [`#556`](https://github.com/openai/openai-go/issues/556). +- **Practitioner guidance** is unusually blunt about the retry hazard: replaying the request + can duplicate "model work, billing, tool intent, or a fragment already held by a downstream + accumulator", and partial output should never be treated as final. +- **OpenAI's own streaming guide** documents `[DONE]` as the completion signal — which is to + say, truncation detection is the client's job. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Retry the whole request** | Treat it like any failed call. | Pays twice, re-runs the model, and duplicates content in any accumulator that kept the first attempt's deltas. The user sees the answer restart. | +| **Never retry a stream** | Surface whatever arrived. | Safe and common. Turns every transient blip into a visible failure, even ones a single retry would have fixed. | +| **Checkpoint + resume via `Last-Event-ID`** | Reopen and replay from the last seen id. | The correct mechanism, and unavailable on OpenAI-style APIs — no `id:` on completion chunks. Works for event feeds that do emit ids. | +| **Continuation prompt** | Re-ask the model to continue from the partial text. | The pragmatic LLM-specific answer. Costs another call, and the seam is visible in the output. | +| **Buffer everything, emit at the end** | Don't stream to the user until `[DONE]`. | Makes truncation detectable and retry safe — and throws away the entire reason for streaming. | +| **Sentinel check** | Require `[DONE]`; treat its absence as failure. | Necessary in every one of the above. Cheap, and routinely forgotten. | + +**Summary of the state of the art:** detect truncation by the sentinel rather than the status, +never blind-retry a stream that has already emitted, resume only where the server supports it, +and keep the partial. The hard part is that the right policy differs per API — resumable feeds +and LLM completions want opposite behaviour from the same client. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **Resumable SSE is a real, shipped feature** and it is fresh (`#622`, two commits back). + `sse.reconnect` (`types.ts:847-851`) reopens a dropped body and resumes from the last `id:`; + `Surface.resumeToken` / `applyResume` (`surface.ts:83-107`) are the seam, and `sse` sets the + `Last-Event-ID` header. So for a **resumable** feed this may be genuinely one config flag — + which would make it the first scenario in this pass with a clean built-in answer. +- **The LLM case is the interesting one**, because `Last-Event-ID` cannot help: no `id:` on the + frames. What does `reconnect` do when there is no token to resume from — reopen from scratch + (duplicating everything) or refuse? +- **The retry question is the sharp one.** If `retry` is applied to a streaming stitch and the + body dies after N deltas have already been yielded to a consumer, does the consumer see those + N deltas **twice**? That is the "fragment already held by a downstream accumulator" hazard, + and it is a correctness question, not a policy one. +- **Is the partial preserved?** Scenario 3 found no channel for a batch residue. The same + question here: on a mid-stream failure, does the caller get the deltas that did arrive, or + only an error? + +**Claims to test with runnable offline code:** + +1. **C1** — a stream drops mid-body after N deltas. What does a consumer of `.stream()` see — + an error, a truncated-but-clean end, or nothing distinguishing it from success? +2. **C2** — with `retry` configured on a streaming stitch and a mid-body drop: are the + already-emitted deltas **re-emitted**? Measure the exact delta sequence the consumer sees. +3. **C3** — `sse.reconnect` against a server that **does** emit `id:` and honours + `Last-Event-ID`: does it resume without duplication? Measure deltas and the header sent. +4. **C4** — `sse.reconnect` against an **OpenAI-shaped** stream with no `id:` on any frame: + what happens? Duplication, refusal, or silent restart? +5. **C5** — an in-band SSE `error` frame arriving at HTTP 200: can it be made a real failure? + (`verdictOf`? a surface? a hook?) +6. **C6** — missing `[DONE]`: can "the stream ended early" be distinguished from "the stream + ended", and can that be a failure? +7. **C7** — is the **partial output** reachable on a mid-stream failure — from the error, the + event stream, or a hook — or is it lost? +8. **C8** — can retry be enabled for the _connect_ phase (a 503 before any byte) but disabled + once bytes have flowed? That is the policy every LLM client actually wants. +9. **C9** — assemble the best answer for the LLM case, run it, report the seam and line count. + +C2 and C8 decide this one. A client that re-emits already-seen deltas on retry is worse than +one that doesn't retry at all. diff --git a/docs/scenarios/multi-tenant-blast-radius.md b/docs/scenarios/multi-tenant-blast-radius.md new file mode 100644 index 00000000..fdfa1718 --- /dev/null +++ b/docs/scenarios/multi-tenant-blast-radius.md @@ -0,0 +1,151 @@ +# Scenario: one customer's bad token takes down all of them + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `multi-tenant-blast-radius` + +**Verification:** 8 proof scripts, run offline (152 checks), in +[`proofs/multi-tenant-blast-radius/`](proofs/multi-tenant-blast-radius/). Published page: +[`scenarios/multi-tenant-blast-radius.mdx`](../../apps/docs/content/docs/scenarios/multi-tenant-blast-radius.mdx). +Escalated — **the pass's highest production-impact finding**: +[`issue-drafts/resilience-has-no-tenancy.md`](issue-drafts/resilience-has-no-tenancy.md). + +| Claim | Verdict | Measured | +| -------------------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — shared breaker blast radius | **confirmed, worse than predicted** | **9 of 9** healthy tenants failed, `503 circuit open`, 0 requests reached the vendor — and it **never self-heals**: `503,503,503,503` across 4 cooldown windows | +| C2 — partition the breaker | yes, via the key **string** only | `circuit.key` is a static string; 10 `.as()`-bound stitches → **1** key; 10 separate **seams** → 1 key; per-tenant `key` → **0 of 9** failed | +| C3 — exclude a 401 | PASS, pure config | `accept: [401]` alone **swallows** it (caller got the error body as data); `{ accept: [401], flag: 'ok' }` → real 401, **0** circuit failures, a genuine 500 still trips | +| C4 — noisy neighbour | confirmed | one tenant's 20-call burst pushed a quiet tenant from t=0 to **t=2000 ms**; with `concurrency: 2` it was **last**, at t=5000 | +| C5 — partition the throttle | yes, two ways | per-tenant seam, or per-tenant `name` + member throttle — both t=0. `pool: 'host'` collapses **both** partitions _and_ re-keys the circuit | +| C6 — token isolation | confirmed, fails closed | 3 tenants → 3 tokens, 3 vault keys; errors without `.as()`. Default is `'app'` — 3 customers, **1** shared token | +| C7 — cost of isolation | cheap, leaky | 100 seams < 40 kb each, **0 timers, 0 pools** — but breaker keys are immortal and `seam.stitch()` pins 200/200 | +| C8 — the four resources | 2 isolated, 2 not | assembled blast radius **0 of 9** | + +**Three wrong hypotheses, and two were wrong in the optimistic direction — a first.** + +- "Per-tenant breakers mean one stitch per tenant" — **false**. 10 stitch objects → 1 breaker; + 10 _seams_ → 1 breaker. Isolation is the key string, never the object graph. +- "The rate bucket looks un-partitionable by tenant" — **wrong pessimistically**. It partitions + two ways; only the _declaration_ is missing. +- "One client per tenant doesn't scale: 4,000 pools and timers" — **wrong**. 100 seams cost + under 40 kb each with **zero** timers and **zero** pools, because a seam owns no transport. + +**The framing worth keeping:** the split falls exactly on the auth/resilience line. +`AuthContext.principal` reaches the auth strategies and the cache-key builder and nothing else. +So the two resources whose isolation is a **security** property fail closed, and the two whose +isolation is an **availability** property fail open, silently. + +--- + +## The use case + +You run a SaaS that integrates a vendor API — Jira, Salesforce, HubSpot, Shopify — **on behalf +of each of your customers**. Every customer connected their own account, so every call carries +that customer's credential. At 500 customers with 8 connections each you are managing +[4,000 token lifecycles](https://truto.one/blog/how-to-architect-a-scalable-oauth-token-management-system-for-saas-integrations/). + +The calls are ordinary. The failure mode is not. + +## Why it is not straightforward + +**The unit of failure is the tenant; the unit of protection usually isn't.** + +Resilience machinery — rate budgets, circuit breakers, connection pools — is normally scoped to +a _dependency_. But in a multi-tenant integration the thing that goes wrong is scoped to a +_customer_: their token was revoked, their admin changed a permission, their account hit its own +quota. When the protection is broader than the failure, one customer's problem becomes +everyone's: + +- **A shared circuit breaker is the sharpest edge.** One customer whose refresh token was + revoked produces a steady stream of `401`s. Those are failures. A breaker counting failures + across all tenants opens — and now every _healthy_ customer fails fast too. One revoked + token, total outage. +- **A shared rate budget is the noisy-neighbour classic.** One customer's batch job consumes + the bucket and every other customer's latency degrades. The + [documented shape](https://markheath.net/post/noisy-neighbour-multi-tenancy) is one badly + written job tripling everyone's p99 within minutes. +- **A shared credential is worse than a shared bucket.** As one write-up puts it: the upstream + 429s, _"and every other session behind that same credential inherits it… per-tenant buckets + contain the rate, but the credential model underneath is what decides the blast radius."_ +- **Token refresh is per-tenant and concurrent** — the thundering herd from + [scenario 1](oauth2-refresh-token-rotation.md), now multiplied by tenant count. +- **Isolation has to be cheap.** "One client instance per tenant" is correct and does not scale + to 4,000 of them: each carries its own connection pool, timers, and memory. + +The tell that this is hard: platforms keep adding _partitioned_ limiters to fix it — .NET 10 +shipped per-tenant rate limiters as a first-class feature precisely because the un-partitioned +kind is a known outage generator. + +## Evidence this bites real projects + +- **The noisy-neighbour writeups** are consistent: [Mark Heath](https://markheath.net/post/noisy-neighbour-multi-tenancy), + [OneUptime on per-tenant collector limits](https://oneuptime.com/blog/post/2026-02-06-otel-rate-limiting-per-tenant-noisy-neighbor/view), + [Gravitee on rate limiting at scale](https://www.gravitee.io/blog/rate-limiting-apis-scale-patterns-strategies). +- **Per-account circuit breakers** are named as the mitigation — open the breaker for _that + account_ so its doomed retries fail fast without slowing everyone else. +- **Token management at scale** — [Truto on architecting OAuth for B2B SaaS](https://truto.one/blog/how-to-architect-a-scalable-oauth-token-management-system-for-saas-integrations/) + puts the refresh race at the centre. +- **.NET 10 partitioned rate limiters** exist [for exactly this](https://blog.elmah.io/new-in-net-10-and-c-14-multi-tenant-rate-limiting/). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------ | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| **One client instance per tenant** | Construct the whole client per customer. | Correct isolation by construction, and it does not scale: 4,000 pools, timers and caches. | +| **Partitioned limiter** | One bucket per tenant key inside one client. | The right shape, and only helps if your client offers it — most don't. | +| **Per-tenant circuit breaker** | Key the breaker on the customer. | The named mitigation. Needs the breaker to accept a _per-call_ key, which most implementations don't. | +| **Global breaker, tuned high** | Raise the failure threshold so one tenant can't trip it. | Trades one failure mode for another: now a real outage takes far longer to trip. | +| **Exclude auth failures from the breaker** | Don't count `401`s as dependency failures. | Genuinely correct and often forgotten — a `401` says the _credential_ is bad, not the API. | +| **Sharded workers by tenant** | Route each customer to a worker. | Real isolation, at the cost of a routing tier and uneven load. | + +**Summary of the state of the art:** partition every piece of shared state by tenant — token +cache, rate budget, breaker — and don't count credential failures as dependency failures. The +first is what most clients get wrong, and the second is what most _teams_ get wrong. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **There is a real asymmetry in the isolation knobs.** `tenancy: 'principal' | 'app'` exists on + `OAuth2Options`, `CookieSessionOptions` and `CacheOptions` (`types.ts:1147`) — so **auth and + cache can be per-tenant**. But: + - `ThrottleOptions.pool` is `'stitch' | 'host'` (`types.ts:1039`) — **no `'principal'`**, so + the rate bucket looks un-partitionable by tenant. + - `CircuitOptions` is `{ failures, cooldown, key }` (`types.ts:1079-1088`) — a `key`, but **no + `tenancy`**. Whether `key` can vary _per call_ is the crux: if it is static config, + per-tenant breakers mean one stitch per tenant. +- `seam.as(principal)` binds a principal, and scenario 1 measured `oauth2({ tenancy: +'principal' })` isolating tokens correctly. The open question is whether that binding reaches + the _resilience_ layer at all. +- Scenario 6 measured that a surface cannot even see the bound principal, which suggests the + principal is auth/cache-scoped rather than run-scoped. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** One tenant with a permanently bad credential produces repeated + failures. Does a shared circuit breaker open and **fail healthy tenants**? Measure: how many + of N healthy tenants fail because of tenant X. +2. **C2** — can the breaker be partitioned per tenant? Is `circuit.key` static config or can it + vary per call? If static, measure the cost of the workaround (one stitch per tenant): what + does 100 tenants actually construct? +3. **C3** — is a `401` counted as a circuit failure? It shouldn't be — it says the credential is + bad, not the dependency. Measure whether `verdict`/`acceptStatus` can exclude it without + also swallowing the error. +4. **C4** — noisy neighbour: with `throttle: { rate }` on a shared seam, does one tenant's burst + consume other tenants' budget? Measure arrival times per tenant. +5. **C5** — can the throttle be partitioned per tenant at all? `pool` offers `'stitch' | 'host'`; + try `seam.as()`, a per-tenant seam, `key`. Measure what actually isolates. +6. **C6** — token isolation (the part scenario 1 suggests works): confirm `tenancy: 'principal'` + keeps tenant tokens separate, and that one tenant's refresh storm doesn't disturb another's + in-flight calls. +7. **C7** — the cost of the correct construction. If isolation requires per-tenant stitches or + seams, measure what 100 tenants costs: objects, timers, memory, and whether anything is + shared that shouldn't be. +8. **C8** — assemble the best available answer and state plainly which of the four shared + resources (token, cache, rate, breaker) end up isolated and which don't. + +C1 and C5 decide this. A shared breaker that one tenant can open is a total-outage bug, not an +ergonomics complaint — and if the rate bucket cannot be partitioned at all, the honest answer +may be that multi-tenant fan-out needs a seam per tenant. diff --git a/docs/scenarios/multipart-upload.md b/docs/scenarios/multipart-upload.md new file mode 100644 index 00000000..75e8d9dc --- /dev/null +++ b/docs/scenarios/multipart-upload.md @@ -0,0 +1,145 @@ +# Scenario: the upload you must clean up after + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `multipart-upload` + +**Verification:** 8 proof scripts, run offline (201 checks), in +[`proofs/multipart-upload/`](proofs/multipart-upload/). Published page: +[`scenarios/multipart-upload.mdx`](../../apps/docs/content/docs/scenarios/multipart-upload.mdx). +Escalated: [`issue-drafts/no-compensation-seam.md`](issue-drafts/no-compensation-seam.md). + +| Claim | Verdict | Measured | +| ------------------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — upload progress | PASS | `xhrAdapter` 4 upload ticks before the response existed; `fetchAdapter` **0**; `supports` readable with no call; an `info` event names `xhrAdapter()` — but only if the adapter declared capabilities | +| C2 — ETag header, in part order | PASS via `interpret` | server stored `[3,1,4,2]`, `Promise.all` resolved `[1,2,3,4]`; the same ETags in settle order → `400 InvalidPartOrder` + **4 orphans** | +| C3 — bounded concurrency | PASS, **not** via `all()` | peaks: no throttle 8, `all()` **8**, one stitch + `concurrency: 3` **3**, 8 stitches × `concurrency: 3` **8**, `pool: 'host'` **3** | +| C4 — compensation seam | **FAIL — none exists** | HTTP 500 → `[onRequest, onResponse]`, **0** `onError`. No cleanup: **3 parts / 15 MiB / 0 DELETEs**. Abort: 2 orphans. Timeout: 3 orphans | +| C5 — retry granularity | per-part PASS; whole-upload not prevented | `[1,2,3,4,3]`, 0 orphans — but the default `retry.on` excludes **500**, S3's own transient error; outer retry → **9 orphans, 45 MiB** | +| C6 — cancelled siblings | diverges | 2 parts stored, **0 nameable**; `all()` discards resolved values; no `allSettled` | +| C7 — progress aggregation | PASS, not the obvious way | naive `Σ loaded` **400** vs real **160**; per-part high-water gives 160 and survives a retry replay | +| C8 — assembled | PASS | 0 orphans on every exit path; **141 vs 163** lines | + +**The honest headline, and it is not flattering.** The library saves 22 lines — retry with +backoff, the concurrency pool, the retryable-status set, URL assembly, all config. But the +`try/finally`, the loud-cleanup rule, the per-part high-water progress map and the input-order +assembly are **byte-for-byte identical** on both sides. This is the first scenario in the pass +where the library contributes nothing to the requirement the scenario exists for. + +**Hypotheses: mostly right for once, and the one that was wrong matters.** The capture guessed +`all()` might need `throttle.concurrency` as its pool — in fact `all()` bounds nothing _and_ +hands every member the same input, so the combination that reads correct (`all()` + per-member +`concurrency`) measured a peak of 8 against a stated limit of 3. The capture also under-rated +C4: it asked whether cleanup was "entirely user-side", but the sharper finding is that the two +natural ways to write it are silently wrong — `.safe()` on the abort cannot throw, and cleanup +inside `Surface.execute` runs _after_ the caller returns. + +--- + +## The use case + +A user uploads a 5 GB video. You can't send it in one request, so you use S3-style multipart: +initiate the upload, send the file in parts, then tell the server to assemble them. + +1. `POST /uploads?uploads` → an `UploadId` +2. `PUT /uploads/{key}?partNumber=N&uploadId=…` × N → each returns an **`ETag` header** +3. `POST /uploads/{key}?uploadId=…` with the ordered `{ PartNumber, ETag }` list → the object +4. …and on **any** failure, `DELETE …?uploadId=…` — or you pay for the parts forever. + +## Why it is not straightforward + +**Step 4 is the one nobody models.** If you abandon a multipart upload, every part already +sent stays in the bucket and **bills as storage indefinitely** — while being invisible to +`aws s3 ls` and to the console's objects tab. AWS's own FinOps guidance puts incomplete +multipart uploads at **up to 20% of an S3 bill**. This is a _compensating action_: a failure in +step 2 or 3 obliges you to make a different API call, and no HTTP client models "on failure, +call this other endpoint". You write a `try/finally` and hope every exit path goes through it. + +The rest is a genuine orchestration problem: + +- **The result of each part is a response header.** `ETag` is not in the body, and the + `complete` call needs them **in part order**, not completion order. +- **Retry granularity is per part, not per upload.** Part 47 of 100 failing should re-send part + 47 — retrying the whole upload re-sends 5 GB. (Same shape as the batch-residue scenario, + arrived at from the opposite direction.) +- **Concurrency has to be bounded.** All 100 parts at once will exhaust sockets and memory; + one at a time wastes the bandwidth multipart exists to use. +- **Progress is not available over `fetch` at all.** The Fetch API cannot report bytes _sent_ — + upload progress requires `XMLHttpRequest`. Any progress bar therefore constrains the + transport, and a per-part byte count still has to be aggregated into one number. +- **Parts have a minimum size** (5 MB except the last), so chunking is not free-form. +- **A failure mid-flight leaves siblings in flight.** Cancelling them is right; but parts that + already landed still need the abort, so cancellation and cleanup are different concerns. + +## Evidence this bites real projects + +- **AWS's own cost guidance** — [Discovering and deleting incomplete multipart uploads](https://aws.amazon.com/blogs/aws-cloud-financial-management/discovering-and-deleting-incomplete-multipart-uploads-to-lower-amazon-s3-costs/), + and the [lifecycle rule](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html) + that exists purely to clean up after clients that didn't. +- **Cost writeups** — [Infracost](https://www.infracost.io/finops-policies/aws-s3-deleting-incomplete-multi-part-uploads/) + and [DoiT](https://www.doit.com/blog/aws-s3-multipart-uploads-avoiding-hidden-costs-from-unfinished-uploads/) + both lead with the same point: the parts are billed and invisible. +- **The progress constraint** is well documented: `fetch` cannot report upload progress, so + every progress bar in the browser is XHR-backed. +- **tus** exists as a protocol specifically because a single multipart POST throws the whole + upload away when the connection blinks. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| --------------------------------------- | -------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Single `PUT` of the whole file** | One request, no orchestration. | One blink and 5 GB is gone. Above 5 GB, S3 refuses outright. | +| **Vendor SDK** (`@aws-sdk/lib-storage`) | `Upload` class does parts, concurrency, and abort. | Correct, and the right answer _if you're on AWS_. Pulls in a large dependency, and the same problem recurs on every non-AWS API with the same shape. | +| **Hand-rolled loop + `try/finally`** | Chunk, `Promise.all` with a pool, collect ETags, abort in `finally`. | What most teams write. The abort is one missed early-`return` from being skipped, and nothing tells you when it was. | +| **tus / resumable protocol** | Offload resumability to a protocol. | Genuinely better where you control the server. Not an option against S3's own API. | +| **Lifecycle rule as the safety net** | Let S3 clean up after N days. | Necessary belt-and-braces, and not a fix: you still pay for N days of orphaned parts across every failed upload. | +| **Skip progress** | Avoid the XHR constraint. | Fine for a server-side job; unacceptable for a user watching a 5 GB upload. | + +**Summary of the state of the art:** bound the concurrency, retry per part, collect ETags in +order, aggregate progress, and — the part that actually bites — guarantee the abort runs on +every failure path. The first four are ordinary async work. The fifth is a _compensation_ +requirement, and it is the one no client library helps with. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **`xhrAdapter` exists precisely for this.** Its documented reason to exist is upload progress + ("`fetch` cannot report bytes sent"). Adapters also declare a `capabilities` descriptor — + `{ name?, supports }` over `'stream' | 'uploadProgress' | 'downloadProgress'` — so there may + be real capability negotiation, and possibly a diagnostic when you ask `fetch` for progress. +- **`all()` runs members as sibling child runs and auto-cancels them if one fails.** Useful for + the part fan — but auto-cancellation is not cleanup: parts that already _landed_ still need + the abort. Whether `all()` has any concurrency bound is open; `throttle.concurrency` on the + part stitch may be the pool. +- **There is no compensation hook anywhere in core.** No `onFinally`, no `compensate`, no + `onSettle` in `types.ts`. If that holds, the mandatory abort is entirely user-side, and the + interesting question becomes whether the library at least makes it _hard to skip_. +- The ETag-from-a-response-header mechanic already has two precedents in this section + (`Location` in the async triangle, `ETag` in conditional requests), so it is likely reachable + — the new question is collecting N of them **in part order** under concurrency. + +**Claims to test with runnable offline code:** + +1. **C1** — does `xhrAdapter` actually report upload progress ticks, and does `fetchAdapter` + silently report nothing? Is the capability difference detectable _before_ a call? +2. **C2** — can a part's `ETag` **response header** be captured, and N of them assembled in + **part order** (not completion order) for the complete call? +3. **C3** — can part uploads run with **bounded concurrency**? Try `all()`, `throttle.concurrency`. + Measure the actual peak in-flight count. +4. **C4** — **DECIDING CLAIM.** Is there any way to guarantee the **abort** runs on failure — + a compensation hook, a `finally` seam, anything? Or is it entirely `try/finally` in user + code? Measure the orphan: how many parts are left behind when a part fails and nothing + aborts. +5. **C5** — per-part retry: can one part retry without re-sending the others? And is the + _whole-upload_ retry prevented (it would re-initiate and orphan the first `UploadId`)? +6. **C6** — when `all()` auto-cancels siblings on a failure, what happens to parts that had + already completed? Are they visible for the abort, or lost? +7. **C7** — progress aggregation: can per-part byte counts become one number for a UI? +8. **C8** — assemble the best answer, run it, report seam(s) and line count, compare honestly + against the hand-rolled version and against `@aws-sdk/lib-storage`'s ergonomics. + +C4 is the one that matters. Every other part of this is ordinary orchestration; the abort is +the requirement that turns a working upload into a billing incident when it's missed. diff --git a/docs/scenarios/n-plus-one-fanout.md b/docs/scenarios/n-plus-one-fanout.md new file mode 100644 index 00000000..32064f44 --- /dev/null +++ b/docs/scenarios/n-plus-one-fanout.md @@ -0,0 +1,144 @@ +# Scenario: one list, a hundred follow-up calls + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `n-plus-one-fanout` + +**Verification:** 8 proof scripts (180 checks), run offline, in +[`proofs/n-plus-one-fanout/`](proofs/n-plus-one-fanout/). Published page: +[`scenarios/n-plus-one-fanout.mdx`](../../apps/docs/content/docs/scenarios/n-plus-one-fanout.mdx). +Escalated: [`issue-drafts/coalescing-does-not-share-failures.md`](issue-drafts/coalescing-does-not-share-failures.md). + +| Claim | Verdict | Measured | +| --------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — combinators can't express it | confirmed, **wrong reason** | runtime length is fine; the **input broadcast** is the wall — 100 members, 1 input → **100 requests for 1 id**; `all()` bounded nothing (peak 100) and discarded 99 successes | +| C2 — `cache.coalesce` | **PASS — strongest positive of the pass** | 100 concurrent calls / 30 ids → **30 requests**, no response landed. `coalesce: false` → 100. But a coalesced **failure is not shared**: 100 requests for one 404ing id | +| C3 — bounded concurrency | works on one stitch; silently multiplied otherwise | one stitch → **peak 8** exactly; 100 stitches → **peak 100**; `pool: 'host'` fixes it; **adding a `store` breaks the fix again** | +| C4 — partial failure | solved by `.safe()` | 99 rows kept, failure at index 49; bare `Promise.all` kept **0 rows and still spent all 100 requests** | +| C5 — thundering herd | **default works** | `expo-jitter` → **~98 distinct ms**; `'expo'` **and** `'fixed'` → **1 ms**. But `Retry-After` re-clusters all 100 into one ms by default | +| C6 — the trace | one tree, wrong shape | default 101 roots; `linked` gives 1 trace with per-call inputs — at **depth 101, fan-out 1** for calls that ran at peak 100 | +| C7 — ordering | positional and safe; **aliasing** is the finding | 20 rows / 5 customers → **5 distinct objects**; mutating row 0 changed row 5 | +| C8 — assembled | PASS | **45 lines vs 87** hand-rolled — but **44 requests vs 32**, the gap being failure dedupe | + +**The capture was right about C1 and wrong about why.** It said a runtime-length list rules out +the combinators; it doesn't — `all(ids.map(…))` compiles. The input broadcast is what makes +`all()` structurally unable to express a per-id fan-out. **Third scenario to land on that same +broadcast** (7, 10, 16). + +**And C2 is the strongest positive result of the pass.** In-flight coalescing is a real +capability most clients lack, it is one config field, and it is exactly the fix this shape +needs. The complement — a coalesced _failure_ releases every joiner — is the one place the +hand-rolled version wins, and it is precisely the shape a dead foreign key takes. + +--- + +## The use case + +You `GET /orders` and get 100 back. Each one carries a `customerId`, and you need the customer. +So you make 100 more calls. + +This is the most common composition shape in API integration, and every part of it is a +decision: how many at once, what to do when one fails, how to join the results back, and +whether you even need 100 calls. + +## Why it is not straightforward + +**N is unknown until runtime.** You cannot write the fan-out at authoring time — it comes from +the first response. That rules out any combinator that takes a fixed list of members, and it +means the per-call inputs all **differ**, which rules out anything that broadcasts one input. + +Then the four decisions: + +- **Concurrency.** All 100 at once will trip a rate limit; one at a time wastes the afternoon. + And the governance is often **concurrency-based rather than request-rate-based**, so a + requests-per-second cap doesn't protect you. +- **Partial failure.** `Promise.all` rejects on the first failure **and discards the results + that succeeded**. One deleted customer 404s and you lose 99 good rows. `allSettled` keeps them + and gives up fail-fast. +- **The thundering herd on retry.** If all 100 hit a 429 at the same instant and back off by the + same computed amount, they retry at the same instant. **Deterministic backoff re-clusters the + burst**; jitter is the only thing that breaks it — and it matters more here than anywhere, + precisely because the calls started together. +- **Duplicates.** 100 orders commonly reference far fewer distinct customers. Fetching the same + id 4 times is 4× the quota for one answer, and the fix — collapse in-flight duplicates — is + not something most clients offer. + +And the meta-point: the best fix is often **not to fan out at all**. One team cut 20k calls/day +to 800 by using a batch endpoint. A client library can't invent one, but it should not make the +fan-out so easy that nobody looks. + +## Evidence this bites real projects + +- **Concurrency vs rate** — [Truto on rate limits across third-party APIs](https://truto.one/blog/best-practices-for-handling-api-rate-limits-and-retries-across-multiple-third-party-apis/) + notes governance is often concurrency-based, and that a burst of parallel calls throttled + together will **re-cluster under deterministic backoff** unless there's full jitter. +- **`Promise.all` discards successes** — the standard warning + ([MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all), + [Beware of Promise.all](https://dev.to/jdorn/beware-of-promiseall-3pph), + [better handling with allSettled](https://www.coreycleary.me/better-handling-of-rejections-using-promise-allsettled)). +- **Concurrency control is a library** — `p-limit`, `Bottleneck` — because the platform doesn't + offer one. +- **Batching beats fanning out** — [the 20k → 800 calls/day case](https://truto.one/blog/best-practices-for-handling-api-rate-limits-and-retries-across-multiple-third-party-apis/). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| --------------------------------------------- | -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`Promise.all(ids.map(fetch))`** | The one-liner everyone writes. | Unbounded concurrency, and one failure discards every success. | +| **`Promise.allSettled` + a pool** (`p-limit`) | Bounded, keeps partial results. | Correct, and it is two dependencies and a hand-rolled join. | +| **Sequential loop** | One at a time. | Safe, and N× the latency. | +| **A batch endpoint** | `GET /customers?ids=…`. | Strictly best where offered — and then you inherit [partial-failure semantics](batch-partial-failure.md). | +| **Cache / dedupe by id** | Don't fetch the same id twice. | Free quota, and only if in-flight duplicates collapse too — a cache that only helps _after_ a response lands does nothing for a simultaneous fan-out. | +| **Prefetch / expand** | Ask the list endpoint to embed the customer. | The real fix where the vendor supports `?expand=`. Rarely does. | + +**Summary of the state of the art:** bound the concurrency, keep partial results, jitter the +backoff, collapse duplicate ids, and check whether a batch endpoint exists before writing any of +it. + +--- + +## What to verify against StitchAPI + +Two earlier scenarios bear directly on this. [Scenario 7](multipart-upload.md) measured `all()` +**bounding nothing** (peak 8 over 8 members) and handing **every member the same +`StitchInput`** — which, if it holds, means `all()` structurally _cannot_ express this scenario, +where every call needs a different id. [Scenario 10](provider-failover.md) measured the same +input-broadcast on `any`/`race`, and that `Composable` is not user-authorable. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **The combinators are out.** N is a runtime length and the inputs differ, so this is + `Promise.all(ids.map(…))` territory. The question is what the library contributes _around_ + each call. +- **`throttle: { concurrency }` on the per-item stitch is the pool.** Scenario 9 measured + `pool: 'stitch'` giving one stitch its own budget — which is exactly right when the same stitch + is called N times, and exactly wrong if someone builds N stitches. +- **`cache.coalesce` is the untested capability that matters most here.** It is documented as + `'process' | 'cluster' | false`, and in-flight coalescing is precisely the duplicate-fan-out + fix: 100 concurrent calls for 30 distinct ids should make 30 requests. Nothing in the pass has + exercised it. +- **`backoff: 'expo-jitter'` is the default**, which is the right default for this shape. Whether + it actually de-clusters a simultaneous burst — and how badly `'expo'` or `'fixed'` re-clusters + — is measurable. + +**Claims to test with runnable offline code:** + +1. **C1** — confirm the combinators can't do it: a runtime-length list of _different_ inputs. + Measure what `all()` actually sends. +2. **C2** — **DECIDING CLAIM.** `cache.coalesce`: 100 concurrent calls across 30 distinct ids. + How many requests reach the server? Does it collapse **in-flight** duplicates, or only serve + from a completed cache? +3. **C3** — bounded concurrency via `throttle: { concurrency }` on one stitch called N times. + Measure the peak in-flight. Then the trap: N _separate_ stitches. +4. **C4** — partial failure. One id 404s. With `.safe()` per member, do the other 99 survive, and + is the failing id identifiable? +5. **C5** — **the thundering herd.** 100 calls 429 simultaneously. Measure the retry arrival + spread under `'expo-jitter'` vs `'expo'` vs `'fixed'`. Does the default actually de-cluster? +6. **C6** — the trace. Is a 100-call fan-out one tree or 100 unrelated roots? Does `linked` help + when the members are created at runtime? +7. **C7** — ordering and joining: does the result order match the input order under concurrency? +8. **C8** — assemble the best available answer, run it, report the seam and line count against + `Promise.allSettled` + `p-limit`. + +C2 is the one that could make this scenario a genuine win: in-flight coalescing is a real +capability, most clients don't have it, and it is exactly the fix this shape needs. diff --git a/docs/scenarios/oauth2-refresh-token-rotation.md b/docs/scenarios/oauth2-refresh-token-rotation.md new file mode 100644 index 00000000..75c56143 --- /dev/null +++ b/docs/scenarios/oauth2-refresh-token-rotation.md @@ -0,0 +1,137 @@ +# Scenario: OAuth2 rotating refresh tokens under concurrent calls + +**Researched:** 2026-08-04 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `oauth2-refresh-token-rotation` + +**Verification:** 7 proof scripts, run offline, in +[`proofs/oauth2-refresh-token-rotation/`](proofs/oauth2-refresh-token-rotation/). Published +page: [`scenarios/oauth2-refresh-token-rotation.mdx`](../../apps/docs/content/docs/scenarios/oauth2-refresh-token-rotation.mdx). +Separate finding escalated to a draft: +[`issue-drafts/oauth2-params-rotation-footgun.md`](issue-drafts/oauth2-params-rotation-footgun.md). + +| Claim | Verdict | Measured | +| --------------------------------------------------------- | --------------------- | --------------------------------------------------------- | +| C1 — 20 concurrent cold callers, one `oauth2()` stitch | PASS | 1 token request, 1 distinct bearer | +| C2 — 20 concurrent 401s on a cached token | PASS, with a boundary | simultaneous ⇒ 1 refresh; staggered 8ms ⇒ **10** | +| C3 — 2 workers sharing `store` + `key`, cold | capability ABSENT | **2** token requests; scales with workers, not callers | +| C4 — `oauth2()` doing rotating `grant_type=refresh_token` | FAIL | redemption #2 replays the consumed token ⇒ family revoked | +| C5 — custom `AuthStrategy` doing rotation + single-flight | PASS | 1 redemption, 0 replays, **79 lines** | +| C6 — store-backed cross-process lock | PASS | 3 workers × 10 callers ⇒ 1 redemption, **+42 lines** | +| C7 — `cookieSession` as a richer seam | refuted | hook receives only `{ ok, status }` | + +**Hypotheses that were wrong.** Two of the pre-verification guesses below did not survive: + +- "`oauth2()` is `client_credentials`, so rotation is simply out of scope" — half wrong, and + the wrong half matters. `params` _can_ override `grant_type`, and the first redemption + genuinely **succeeds**. It is not rejected or unsupported; it works once, then kills the + account. That is worse than unsupported, and is why the issue draft exists. +- "`params` is static so a rotated value has no path back" — right conclusion, wrong mechanism. + A getter satisfies `Record` and is invoked per request, so the hack rotates + correctly in one process. It still dies across two workers, and cannot be repaired: a getter + must return synchronously while every `StitchStore` read is async. + +--- + +## The use case + +A backend integrates a third-party SaaS API on behalf of each of its users — Atlassian +(Jira/Confluence), Asana, Xero, QuickBooks, Slack, Google. The integration holds a +long-lived **refresh token** per connected account and exchanges it for a short-lived +access token as needed (`authorization_code` grant, then `grant_type=refresh_token`). + +The workload is ordinary: a sync job, a webhook handler, and a user-facing request path +all call the same vendor API for the same connected account, concurrently, from more than +one worker process. + +## Why it is not straightforward + +Two properties collide. + +**1. The refresh token is single-use and rotates.** Modern providers implement +[RFC 6819 §5.2.2.3](https://datatracker.ietf.org/doc/html/rfc6819#section-5.2.2.3) refresh +token replay detection: redeeming a refresh token returns _a new one_ and invalidates the +old. Presenting an already-redeemed refresh token is treated as evidence of theft — so the +provider does not merely reject that one call, it **revokes the entire token family**. The +user is silently disconnected and must re-authorize through the browser. + +**2. Expiry is discovered concurrently.** N in-flight requests all hit `401` at the same +instant, or all read the same "expires in 12 seconds" cached token. Each independently +decides to refresh. The first redemption succeeds and rotates; every other redemption +presents a consumed token and trips replay detection. + +The failure is therefore **not** "one request fails and retries." It is "the integration +loses the account," and it happens precisely under load, which is when it is hardest to +reproduce and most expensive. + +Three further wrinkles make the naive fixes insufficient: + +- **A mutex is not enough if it is in-process.** Two workers, two pods, or a serverless + fan-out each hold their own lock. The correct scope of mutual exclusion is _the connected + account_, which spans processes. +- **The rotated token must be durably persisted before it is used.** If worker A redeems, + writes the new refresh token to the database, and crashes between the HTTP response and + the commit, the stored token is now the consumed one — the account is dead on next + refresh. The write must land before the old token is considered spent. +- **Retry makes it worse.** A generic "retry on 401" wrapper turns one replay into several, + which is exactly the signal providers read as token theft. + +## Evidence this bites real projects + +- **OpenAI Codex** — [`openai/codex#10332`](https://github.com/openai/codex/issues/10332): + "refresh token was already used" when multiple app-server instances run concurrently. +- **MCP TypeScript SDK** — [`modelcontextprotocol/typescript-sdk#1760`](https://github.com/modelcontextprotocol/typescript-sdk/issues/1760): + a race in `auth()` invalidates the refresh token when rotation is on. +- **better-auth** — [GHSA-392p-2q2v-4372](https://github.com/better-auth/better-auth/security/advisories/GHSA-392p-2q2v-4372) + (CVE-2026-53517): concurrent redemption _forks the token family_, because the provider's + read → validate → revoke → mint sequence is non-atomic. +- **oauth2-proxy** — [`oauth2-proxy#1992`](https://github.com/oauth2-proxy/oauth2-proxy/issues/1992): + refresh session handling has a race condition. +- **Nango** — [_How to handle concurrency with OAuth token refreshes_](https://nango.dev/blog/concurrency-with-oauth-token-refreshes/): + names Atlassian and Asana as rotating providers where parallel 401s trip replay detection. + +Note the pattern: these are not application bugs by careless teams. They are races found in +_auth libraries and platforms_ — the layer whose whole job is this. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Refresh-on-401, no coordination** | Interceptor catches `401`, refreshes, replays the request. The default in most axios/fetch wrapper tutorials. | The baseline bug. N concurrent 401s ⇒ N redemptions ⇒ family revoked. | +| **In-process single-flight / promise memo** | Keep one in-flight refresh promise per account; concurrent callers await it. | Correct and cheap for **one** process. Silently insufficient the moment there are two workers — and it _looks_ fixed in dev, where there is only one. | +| **Distributed lock (Redis `SETNX`, advisory lock)** | Serialize refresh across workers. Losers wait and re-read the token. | Works, but is now a distributed-systems problem: lock TTL vs refresh latency, crash-while-holding, fencing tokens, and a hard dependency on Redis in the request path. | +| **Proactive refresh with a skew** | Refresh N seconds before expiry rather than on `401`. | Shrinks the window, does not close it — the skew boundary is itself a moment every worker crosses together. Best used _with_ coordination, not instead of it. | +| **Grace period / accept the old token briefly** | Provider-side: the consumed token stays valid for a few seconds. | Not the client's to choose. Auth0, Okta and others offer it; Atlassian and Asana notably do not — and [better-auth#8512](https://github.com/better-auth/better-auth/issues/8512) shows it is still a live design debate. | +| **Dedicated refresh worker** | One process owns refresh; everyone else reads the cached access token. | Clean and genuinely correct. Costs an extra deployable, and a cold access token now blocks on a queue round-trip. | + +**Summary of the state of the art:** there is no one-liner. The honest minimum is +_coordination scoped to the account and spanning processes_, plus _durable persistence of +the rotated token before the old one is treated as spent_. Everything cheaper is a +narrower race, not a fixed one. + +--- + +## What to verify against StitchAPI + +Read of the working tree (`packages/core/src/auth.ts`, ahead of the published docs bundle) +before verification — **hypotheses, to be confirmed or refuted by running code**: + +- `oauth2()` performs the **`client_credentials`** grant (`auth.ts:465`). That grant has no + refresh token at all, so rotation may simply be out of scope for it. +- There **is** an in-process `singleFlight` helper (`auth.ts:432`) keyed per token key, and + a shared `store` is documented to make one token serve many workers. +- `params?: Record` (`auth.ts:393`) can override `grant_type` — but it is a + **static** record, so a _rotated_ refresh token returned in the response has no path back + into the next token request. This is the suspected structural gap. +- `cookieSession` exposes `RefreshResult` / `CookieSessionRefreshOptions` — possibly a + richer seam for carrying rotating state. + +**Claims to test with runnable offline code:** + +1. **C1** — N concurrent calls needing a token fire exactly **one** token request (in-process). +2. **C2** — a `401` mid-flight triggers exactly **one** refresh + retry, not N. +3. **C3** — two independently constructed stitches sharing a `store` (a two-worker + simulation) fire **one** token request between them, or two. +4. **C4** — `oauth2()` can run `grant_type=refresh_token` where the response returns a + **new** `refresh_token` that must be used for the _next_ refresh. +5. **C5** — if C4 fails, can a custom `AuthStrategy` implement rotation + single-flight, and + how much user code does that take? (Decides "achievable but not simple" vs "not achievable".) diff --git a/docs/scenarios/pii-in-the-logs.md b/docs/scenarios/pii-in-the-logs.md new file mode 100644 index 00000000..fd667e13 --- /dev/null +++ b/docs/scenarios/pii-in-the-logs.md @@ -0,0 +1,156 @@ +# Scenario: the customer data you didn't mean to log + +**Researched:** 2026-08-05 · **Status:** ✅ verified (8 claims, 196 checks, offline) · page shipped +**Slug:** `pii-in-the-logs` + +--- + +## The use case + +You integrate a vendor API that returns customer records — names, emails, addresses, partial card +numbers, health or legal detail. You add tracing, because you are responsible about observability. + +Six months later someone greps your log aggregator and finds all of it, in plain text, replicated +across three regions and a backup retention policy you do not control. + +## Why it is not straightforward + +The generic advice is easy to state and hard to execute: _"never store raw PII in logs; use +hashed or tokenized values."_ The difficulty is that the leak is **structural**, not a mistake +anyone made: + +- **Nobody logs PII on purpose. Middleware does.** The two named causes in the field are a debug + endpoint returning full user objects and _"a logging middleware that captures raw request + bodies."_ The second is exactly what a well-instrumented client library is. +- **You cannot enumerate what is sensitive in advance.** A denylist of field names + (`email`, `ssn`) misses `contact`, `primaryEmail`, `user.profile.mail`, and anything nested + inside a free-text field. An allowlist inverts the problem correctly — and requires knowing + the whole response shape, which is the thing that changes. +- **The vendor adds a field and it starts flowing.** This is the property that makes it a + _drift_ problem, not a configuration problem. A response gains `taxId` in a minor release and + a denylist that was complete yesterday is silently incomplete today. +- **Every hop wants a copy.** The observability chain — traces, spans, error trackers, retry + logs, cache entries — _"none of these hops include a PII filter by default."_ +- **The regulation is about defaults.** GDPR Article 25 mandates data protection **by design and + by default**, so "we can turn redaction on" is not the same as compliant. + +## Evidence this bites real projects + +- **The named mechanism** — [Why PII leakage happens in APIs](https://hoop.dev/blog/why-pii-leakage-happens-in-apis/): + logging middleware capturing raw request bodies is one of the two most common causes. +- **Every hop, no filter** — [Is your AI agent leaking PII through LLM APIs?](https://airblackbox.ai/blog/ai-agent-pii-leaking): + LLM providers, vector DBs, logging systems, audit trails and dashboards, none filtering by + default. +- **Traces specifically** — [PII protection at the gateway](https://zuplo.com/learning-center/pii-protection-ai-apis-gateway): + debug logs and traces capture secrets unless scrubbed, and are an overlooked but rich target. +- **Log the detection, not the data** — the same source: record what was detected, when, on which + route, for which consumer, **without** putting the PII in the log that records it. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------- | --------------------------------- | --------------------------------------------------------------------------------------------- | +| **Denylist of field names** | Scrub `email`, `ssn`, `card`. | Misses renamed, nested and free-text fields. Silently incomplete the day the vendor adds one. | +| **Allowlist of safe fields** | Log only what you named. | Correct by construction, and needs the whole response shape — the thing that drifts. | +| **Don't log bodies at all** | Metadata only. | Safe and often unusable: the body is what you need when debugging an integration. | +| **Scrub at the log aggregator** | Filter on ingest. | The data already left your process and crossed a network. Too late for a cross-border flow. | +| **Gateway/sidecar redaction** | Strip in a proxy. | Another hop, and it cannot see which field is sensitive in _your_ domain. | +| **Tokenize before storing** | Replace with a reversible handle. | The right answer for data you keep. Heavy for a log line. | + +**Summary of the state of the art:** redact at the boundary, before the value is copied anywhere, +and prefer an allowlist — because the failure mode you cannot test for is the field that did not +exist when you wrote the list. + +--- + +## What to verify against StitchAPI + +The library has real redaction machinery, and read of the working tree suggests it points at +**config**, not at **response data**: + +- `redactConfig` and a per-slot drop-list (`config-anatomy.ts:50-62`) keep live handles and + secrets off `__config`. +- **Auth secrets are explicitly protected in traces** — `auth.ts:265` notes an OAuth key is + redacted from a sink's `url.full` and structured `input.query` "like `api_key`/… are". +- **ADR 0018 adds `redact` to `.inspect()` — and it is opt-in.** The ADR's own title is + _"an **opt-in** `redact` option for `raw`"_. + +So the hypothesis to test is a clean split: **credentials are protected by default; customer PII +is not.** If that holds, the library is safe against the leak it was designed for and open to the +one the regulation is about. + +**And there is a pre-registered suspicion.** [Scenario 18](agent-holds-the-tool.md) measured that +`sensitive: true` is a **cache** opt-out (`types.ts:1652-1658`) — a stitch carrying it was still +listed and still ran over MCP. It is also, by a distance, the nearest-looking key in the whole +config to "do not log this". Whether anything in the logging path reads it is worth settling +deliberately rather than assuming. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Where does a response body actually go by default? Enumerate + every destination: the event spine, a `TraceSink`, `.inspect().raw`, `.report()`, + `StitchError.body`, error messages, and a `cache` entry. For each: full body, redacted, or + absent? Use a payload with a name, an email, an SSN and a nested `profile.contact.mail`. +2. **C2** — **DECIDING CLAIM.** Does `sensitive: true` affect logging **at all**? Test it against + every destination from C1. If it only gates the cache, measure that and say so plainly. +3. **C3** — what does `.inspect({ redact })` actually redact (ADR 0018)? Nested fields? Renamed + ones? Array elements? Is it a denylist, an allowlist, or a shape? +4. **C4** — is the credential half genuinely safe? Put a bearer token, an `apiKey` in a query + string and a cookie through every C1 destination and scan for the literal values. +5. **C5** — can PII be stripped **at the boundary**, before anything copies it? Which seam runs + earliest — `Surface.interpret`, `transform`, `hooks.onResponse`? Does a value stripped there + stay out of the trace sink and the cache? +6. **C6** — is an **allowlist** expressible? Does an `output` schema that strips unknown keys + keep them out of the log? (Note scenario 20 measured `output` DOES use its parsed value, + unlike `input` — so this may work where the input side does not.) +7. **C7** — the drift angle: when a vendor **adds** a PII field, does anything notice? `drift()` + reports `undeclared` keys — is that a usable "new field appeared, check it" signal? +8. **C8** — assemble the best available "no customer data reaches a log" setup; report seams and + line count, and state what it costs. + +C1 and C2 decide this. C1 establishes the exposure; C2 settles whether the most plausibly-named +key does anything about it. + +--- + +## Verification result + +**All 8 claims verified**, 196 checks across 8 scripts, re-run by me before writing up. + +| Claim | Verdict | +| --------------------------------- | ---------------------------------------------------------------------------------------------------- | +| C1 — where does the body go? | **CONFIRMED, and binary** — 13 destinations carry all 7 sentinels, 11 carry 0, nothing between | +| C2 — does `sensitive: true` help? | **CONFIRMED** — 1 of 11 destinations (the cache); exactly **one** read in all of `packages/core/src` | +| C3 — `.inspect({ redact })` | Opt-in, name-based denylist over `raw` only; `redact: true` removes **0 of 7** | +| C4 — credentials safe? | **PARTIALLY REFUTED** — see below | +| C5 — boundary | `hooks.onResponse` is the earliest seam, and the **only** one covering the failure path | +| C6 — allowlist | **CONFIRMED** — `output` filters 7 → 0 without naming a single PII field | +| C7 — drift as a signal | **CONFIRMED**, and it **refutes ADR 0018 §4** | +| C8 — assembled | 42 lines, 2 seams, 0 of 9 on both paths | + +### Hypotheses that were wrong + +**My clean split was too clean.** I predicted "credentials are protected by default; customer PII +is not." The real line is **credentials the library _places_ vs credentials that ride the +payload**. A declarative `bearer`/`apiKey` never enters the event stream at all — 0 of 3 even for +a naive custom sink — but an `access_token` in a **response body** goes 3 of 3 into the JSONL, +because that sink's redactor is a header denylist rather than the deep scrubber sitting in the +same file. + +**And the measurements refuted a shipped ADR.** ADR 0018 §4 says findings never leak a secret, +justified by `detailFor` emitting kinds only. That is true of the three soft drift kinds and +false of hard validation: `validationErrors` (`drift.ts:50-56`) copies the validator's message +verbatim, and Zod's enum message quotes the received value. It reaches the two sinks that are +otherwise 0 of 7. + +### What I got right, for once + +The pre-registered suspicion carried from [scenario 18](agent-holds-the-tool.md) held exactly: +`sensitive: true` is a cache opt-out and nothing else, and the source agrees — one read, at +`engine.ts:1022`. + +### Outputs + +- Page: [pii-in-the-logs.mdx](../../apps/docs/content/docs/scenarios/pii-in-the-logs.mdx) +- Draft (**held back, security-sensitive**): + [adr-0018-findings-can-leak-a-value](issue-drafts/adr-0018-findings-can-leak-a-value.md) diff --git a/docs/scenarios/precision-loss.md b/docs/scenarios/precision-loss.md new file mode 100644 index 00000000..c723aead --- /dev/null +++ b/docs/scenarios/precision-loss.md @@ -0,0 +1,158 @@ +# Scenario: the ID that changed on the way in + +**Researched:** 2026-08-05 · **Status:** ✅ verified (8 claims, 191 checks, offline) · page shipped +**Slug:** `precision-loss` + +--- + +## The use case + +You integrate an API whose IDs are 64-bit integers — Discord and Twitter/X snowflakes, most +database primary keys, Stripe-style amounts in a `bigint` column. The vendor sends them as JSON +numbers. You read them in JavaScript. + +Some of them arrive as a **different number than the one that was sent**, and nothing anywhere +reports an error. + +## Why it is not straightforward + +`JSON.parse` produces IEEE 754 doubles. Integers above `Number.MAX_SAFE_INTEGER` +(`9007199254740991`, i.e. 2⁵³−1) are not all representable, so consecutive integers start +sharing a single double. `JSON.parse('9007199254740993')` returns `9007199254740992`. No throw, +no warning, no flag. + +Three properties make this genuinely hard rather than merely annoying: + +- **The damage happens before your code runs.** By the time any application-level hook, schema + or interceptor sees the value, it is already a `Number` and the original digits are gone. + Validation cannot help: the corrupted value is a perfectly valid number, and often a perfectly + plausible ID. A schema that says `z.number().int()` passes it. +- **It is silent and data-dependent.** IDs below 2⁵³ round-trip perfectly, so the bug does not + appear in dev, in tests, or for the first several years of a vendor's ID sequence. Snowflake + IDs are time-ordered, which means **the failure arrives on a date**, fleet-wide, for everyone + at once. +- **The fix is not local.** The only real repair is to not use `JSON.parse` — which means a + different parser, which means the values are now `BigInt` or `string`, which breaks + `JSON.stringify`, structured cloning, most cache serialisers, arithmetic, and every schema + validator expecting `number`. You trade a silent bug for a loud cascade. + +And it is a **cross-language interoperability** bug specifically: Python and Java parse the same +payload exactly. The vendor's own tests pass. The bug exists only on your side of the wire, which +makes it very hard to report and very easy to be told it is your problem. + +## Evidence this bites real projects + +- **The canonical shape** — [`JSON.parse` loses precision on Discord snowflake IDs](https://github.com/openclaw/openclaw/issues/23170): + a tool call carrying a channel ID like `1234567890123456789` silently rounds, and the request + fails with `Unknown Channel` — an error message that points nowhere near the cause. +- **Why stringify-the-id became the convention** — + [JavaScript-compatible snowflake IDs](https://samifayoumi.ca/blog/001_53bit-snowflakeid/): + vendors that care ship IDs as strings precisely because of this, and Twitter famously added an + `id_str` field alongside `id` for exactly this reason. +- **The general problem** — + [Safely handling large integers in JSON](https://www.pullrequest.com/blog/safely-handling-large-integers-in-json-best-practices-and-pitfalls/) + and [JSON number precision: IEEE 754, BigInt and decimal](https://jsonic.io/guides/json-number-precision). +- **Money has the same shape** — a decimal amount is not exactly representable either, which is + why financial APIs send integer minor units or decimal strings. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| --------------------------------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| **Use the vendor's string field** | Read `id_str` instead of `id`. | Correct and free — when the vendor provides one. Most do not. | +| **`json-bigint` / custom parser** | Replace `JSON.parse` wholesale. | Correct at the boundary, and now every consumer must handle `BigInt`: no `JSON.stringify`, no mixed arithmetic. | +| **Reviver on `JSON.parse`** | `JSON.parse(text, reviver)`. | **Does not work** — the reviver receives the already-parsed `Number`. The digits are gone before it is called. | +| **Regex the raw text first** | Quote big integers before parsing. | Works, and is a JSON parser written in regex. Breaks on numbers inside strings. | +| **Keep everything as strings** | Treat all IDs as opaque text. | The most robust answer, and it must be enforced by convention across every layer. | +| **Hope** | Most IDs are under 2⁵³ today. | Time-ordered IDs mean this expires on a schedule you do not control. | + +**Summary of the state of the art:** intercept before `JSON.parse`, or get a string from the +vendor. There is no post-hoc repair, because the information is destroyed at parse time. + +--- + +## What to verify against StitchAPI + +The library's own type comment settles where the damage happens: + +```ts +// Parsed JSON when possible, else text — OR a `ReadableStream` when `stream` was set. +body: unknown; +``` + +`AdapterResponse.body` is **already parsed** (`http-adapter.ts:135`, +`parsed = text === '' ? undefined : JSON.parse(text)`). So every seam this pass has relied on — +`Surface.interpret`, `transform`, `output`, `drift()`, `hooks.onResponse` — runs **downstream of +the corruption**, and none of them can see the raw text. + +That makes the `Adapter` the only candidate seam, which is the one place the previous nineteen +scenarios have mostly treated as fixed infrastructure. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Does the default path corrupt? Measure exact in/out values for a + snowflake ID, `2^53+1`, a large `bigint` primary key, and a decimal amount. Confirm it is + silent — no event, no finding, no warning. +2. **C2** — **DECIDING CLAIM.** Can **anything** downstream detect it? Try `output` with + `z.number().int()`, `z.bigint()`, `z.string()`; `drift()` (does it compare against raw text or + against the parsed body?); `.inspect().raw`; `hooks.onResponse`; a `TraceSink`. My prediction + is that all fail, because `raw` is already the parsed body — **verify or refute that**. +3. **C3** — can a custom `Adapter` fix it, and what does that cost? Swap in a bigint-aware parse + and report what breaks: `cache` serialisation, `.inspect()`, `__config` round-trip, trace + sinks, `output` validators. +4. **C4** — does `cache` survive a `BigInt` body? A store that `JSON.stringify`s will throw + `Do not know how to serialize a BigInt`. Test `memoryStore` and a JSON-backed store. +5. **C5** — the **request** side: does a large ID survive going out — in `params`, `query`, and a + JSON `body`? A `bigint` in a request body is a `JSON.stringify` throw, not silent corruption. +6. **C6** — do the `stream` / `download` / `sse` surfaces see raw bytes, and does that make them + a safer path for a precision-sensitive payload? +7. **C7** — is there any spelling that makes this **loud** rather than silent — a drift finding, + an `info` event, anything? What is the minimum user code for a detector? +8. **C8** — assemble the safest available setup; report seams and line count, and state plainly + what it costs the rest of the config. + +C1 and C2 decide this. C1 establishes the damage; C2 asks whether a library whose flagship +feature is **contract drift detection** can notice its most basic form — a value that is not the +value the vendor sent. + +--- + +## Verification result + +**All 8 claims verified**, 191 checks across 8 scripts, re-run by me before writing up. + +| Claim | Verdict | +| --------------------------------------- | ----------------------------------------------------------------------------------------- | +| C1 — does the default path corrupt? | **CONFIRMED and silent** — 4 events, 0 findings; sent digits appear nowhere in the spine | +| C2 — can anything downstream detect it? | **PARTIALLY REFUTED** — 9 seams blind, but 2 see raw text and a `refine` detector works | +| C3 — custom adapter cost | 84 lines; trace sinks survive (against prediction) | +| C4 — cache + BigInt | `memoryStore` survives; a JSON store throws **fatally** | +| C5 — request side | Four positions, three behaviours — `params` **silently vanishes** | +| C6 — streams | Split by **decoder**, not surface: bytes/lines/download lossless; ndjson/json/sse corrupt | +| C7 — a loud detector | 15 lines, **0 false negatives** over 20,000 snowflakes, 0.535% FP | +| C8 — assembled | Two setups: REPAIR 16 lines, DETECT 18 lines | + +### Hypotheses that were wrong + +**The big one, and it changes the answer.** I wrote _"that makes the `Adapter` the only candidate +seam."_ It is not. `wire: { response: 'text' }` is **published config** honoured at +`http-adapter.ts:123` — an `else if` that returns _before_ the JSON branch at `:135`. The repair +is 16 lines on the **stock transport**, not a custom adapter. I reasoned from +`AdapterResponse.body` being pre-parsed and never checked whether config could stop the parse +happening. + +**"Validation cannot help."** `z.number().refine(Number.isSafeInteger)` separates corrupted from +intact, and the boundary walk shows `lossless=false` never co-occurs with `flagged=false` — false +negatives are **impossible**, not merely unobserved. + +**"Trace sinks break under BigInt."** `trace.ts` ships a `bigintSafe` replacer deliberately, so a +`fileSink` is the one diagnostic surface that ends up holding the vendor's real digits. + +**"Money has the same shape."** It does not, on the wire — `19.99` round-trips because the nearest +double's shortest form _is_ `"19.99"`. Decimals fail in **arithmetic**; integers above 2⁵³ fail in +**transport**. Two different bugs that I had merged into one. + +### Outputs + +- Page: [precision-loss.mdx](../../apps/docs/content/docs/scenarios/precision-loss.mdx) +- Draft: [bigint-in-params-vanishes](issue-drafts/bigint-in-params-vanishes.md) diff --git a/docs/scenarios/proofs/agent-holds-the-tool/README.md b/docs/scenarios/proofs/agent-holds-the-tool/README.md new file mode 100644 index 00000000..11fbb886 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/README.md @@ -0,0 +1,243 @@ +# Proofs — the agent chooses the arguments + +Runnable evidence for the claims in [`../../agent-holds-the-tool.md`](../../agent-holds-the-tool.md). + +**C1 was the deciding claim and the boundary HOLDS.** Across 34 JSON-RPC exchanges and 30 separate +payload scans — `initialize`, `ping`, `tools/list`, `list_stitches`, `describe_stitch` on all ten +stitches, a successful `run_stitch` on `bearer(env(...))` / `apiKey` in a header / `apiKey` in the +query / `cookieSession`, a direct call to the registered login stitch, a vendor 401 with a +credential-shaped string in its error body, a validation failure, an unknown stitch, an unknown +tool, an unknown JSON-RPC method and a stdio parse error — **not one of the five credentials the +registry holds appeared, by value, anywhere.** The controls hold too: the same calls put +`Bearer sk_live_…`, `X-API-Key: ak_live_…`, `?api_key=ak_live_…` and `Cookie: SESSION=sess_live_…` +on the wire, so the vendor authenticated every one of them, and the shipped stdio transport writes +byte-identical payloads to the in-process ones. This is the product's central promise and it is +kept. + +**C2 refutes the capture's own hypothesis.** Scenario 10 measured `engine.ts:231` merging +`input.headers` **over** `cfg.headers`, and the capture predicted a prompt injection could set any +header on a call it did not author. It cannot: `sanitizeAgentInput` (mcp.ts:125-130) **deletes the +whole `headers` slot** unless the stitch declares an `input.headers` schema. Six model-supplied +headers — `authorization`, `cookie`, `host`, `x-forwarded-for`, `content-type`, `x-anything` — +reached the vendor as **zero headers**. And even on a stitch that opts in, `authorization` is +unforgeable, because `auth` is applied to a clone **after** the merge (engine.ts:647). + +Three findings go the other way, and one of them is a genuine credential leak: + +- **The MCP error channel is an unfiltered `Error.message` pass-through** (C4). StitchAPI's own + messages are terse and clean — `HTTP 500`, `timed out after 25ms`, `circuit open` — and a vendor + 500 whose body held an internal hostname, a stack frame and a `postgres://vendor:hunter2@…` DSN + reached the model as **four characters**. But a message written by the _transport_ is forwarded + verbatim, and on the **default `fetchAdapter`** with an `apiKey({ in: 'query' })` stitch the model + received `Failed to parse URL from http://api.vendor.test:99999/v1/metrics?api_key=ak_live_qry_…` + — **the credential, in its context, from zero lines of user code.** +- **A model-supplied `query` overwrites a query parameter the operator pinned in the configured + path** (C2 d). `path: '/v1/orders?tenant=acme'` + `input: { query: { tenant: 'globex' } }` put + `?tenant=globex` on the wire and the vendor returned the other tenant's data. A pin is a default, + not a constraint. +- **A declared input schema is a check, not a filter** (C7 e). `validateInput` throws on failure and + **discards the parsed value** (engine.ts:400-408), so a schema that strips unknown keys — the + default behaviour of Zod, Valibot and ArkType alike — does not strip them from the request. A + `query` validator that returned `{ limit: 10 }` still put `?tenant=globex&limit=10` on the wire. + +And the shape of the surface decides two more: there is **no allow-list** beyond the registry object +you hand `createMcpServer` (C3), and **no confirmation seam in either direction** (C6) — the server +cannot ask a human, and code-mode puts a read and a refund behind the same tool name, so the host +cannot either. + +Every script is standalone and offline. Each prints one `PASS`/`FAIL` line and exits non-zero on +failure. **181 checks across 8 scripts.** + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c1-credential-reach.ts + +# all of them +for f in docs/scenarios/proofs/agent-holds-the-tool/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. The suite takes about two seconds; the only +waits are the deliberate `throttle` gaps in C5, and nothing here does network I/O (the one +"transport failure" is a port outside the valid range, which `fetch` rejects before opening a +socket). + +They typecheck under `packages/core`'s full strict set. **`src/version.d.ts` is in the file list on +purpose** — `src/mcp.ts` reads the build-time `__PKG_VERSION__` define, and without that declaration +`tsc` cannot see the identifier: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node src/version.d.ts ../../docs/scenarios/proofs/agent-holds-the-tool/*.ts +``` + +The same define is why `client.ts` loads `src/mcp.ts` through a **dynamic** import behind +`loadMcp()`: a static import would evaluate the module before any assignment could stand the define +in, and the import sorter controls statement order. It is the one piece of ceremony in the +directory, and it is documented in place. + +## The exposure table — what can the model SEE, and what can it SET? + +**This is the consolidation deliverable.** Every row is measured; reproduce the left half with +`c1-credential-reach.ts` / `c3-allowlist.ts` and the right half with `c2-input-rewrite.ts`. + +### What reaches the model + +| Payload | Carries a credential? | What it carries instead | +| --------------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `initialize` | **no** | protocol version, `capabilities: { tools }`, `serverInfo` | +| `tools/list` | **no** | 3 tool descriptors, 1,464 bytes, constant for any registry size | +| `list_stitches` | **no** | every stitch's **name, method and path** — the route table | +| `describe_stitch` | **no** | ~1KB/stitch: the **full internal endpoint URL**, surface, input-slot booleans, `output`, the **auth scheme**, policies, pipeline, a Mermaid diagram | +| `run_stitch` success | **no** | the validated result body | +| `run_stitch` vendor 4xx/5xx | **no** | `HTTP ` — **not** `.body`, `.url`, `.status` or the response headers | +| `run_stitch` timeout / circuit | **no** | `timed out after 25ms` / `circuit open` | +| `run_stitch` input validation | **no** | `invalid : ` | +| `run_stitch` unknown name | **no** | the message plus **every registered name** | +| `run_stitch` missing credential | **no** | `missing env var VENDOR_BEARER_TOKEN` — the **variable name**, never the value | +| **`run_stitch` transport error** | **YES, conditionally** | the transport's message **verbatim** — see the footgun below | +| a vendor endpoint that mints keys | **YES, by contract** | the response body, which is what a capability is for | + +### What the model can set + +| Input field | Reaches the wire? | What an attacker actually gets | +| ----------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `headers` | **no** | deleted outright unless the stitch declares `input.headers` (mcp.ts:128) | +| `headers` (opted in) | **yes, except `authorization`** | any non-credential header; on a `cookieSession` stitch, a `SESSION` pair sent **before** the real one | +| `query` | **yes** | **overwrites an operator's pinned query parameter**; appends anything else | +| `params` | **yes, encoded** | `{id}` percent-encodes `/` → traversal blocked. `{+id}` (reserved expansion) does **not** → a different endpoint, with the credential | +| `body` | **yes, whole** | the entire request body of a write, when no `input.body` schema is declared | +| `signal` | **yes, inert** | aborts the call before it is sent — a self-inflicted denial, no request on the wire | +| `url` / `baseUrl` / `path` / `adapter` / `auth` | **no** | inert: the engine reads input by field name and these are config slots | + +Read the two tables together: **the credential boundary is the library's and it holds; the argument +boundary is entirely the operator's.** That is the finding this directory exists for. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `c1-credential-reach.ts` | **DECIDING** — does a credential reach the model anywhere? | **No.** 30 payload scans, 4 auth strategies, 14,529 bytes; the wire proves each call authenticated; stdio bytes identical | +| `c2-input-rewrite.ts` | **DECIDING** — can `input` redirect or rewrite the call? | **The header hypothesis is refuted** (6 headers → 0). 5 other levers are real; `authorization` is unforgeable | +| `c3-allowlist.ts` | any allow-list? what do the discovery tools disclose? | **None beyond the registry object.** ~1KB/stitch incl. the internal URL. `selectStitch` also answers to `__config.name` | +| `c4-error-rendering.ts` | does a failure leak the URL, headers or the vendor body? | **No — and yes.** `HTTP 500` only; but the channel is unfiltered and `apiKey({in:'query'})` + a transport error leaks | +| `c5-runaway.ts` | do `throttle`/`circuit` apply? any non-count budget? | **Both apply; nothing is on by default.** 1 tool call = 5 (retry) or 12 (paginate) requests. **No spend budget exists** | +| `c6-confirmation.ts` | is there a confirmation seam for an irreversible call? | **No, in either direction.** No elicitation channel, no tool `annotations`, one tool name for a read and a refund | +| `c7-schema.ts` | is `input` typed enough? does a schema constrain the model? | **Four untyped bags; presence-only descriptions.** A schema checks ONE slot and its parsed value is discarded | +| `c8-assembled.ts` | the safest exposure, against the naive one | **47 executable lines, 3 seams, 0 config keys** — and every C2/C3/C7 attack replayed and blocked | + +## Files + +- `vendor.ts` — six credential values (five held by StitchAPI, one minted by the vendor), the env + vars they resolve from, and `Wire`: an `Adapter` that **records every outbound request** (url, + method, headers, body) before answering it. The vendor enforces its own auth on every route, so a + 200 in a proof is evidence the real credential arrived rather than an artefact of a permissive + stub. `/v1/orders` echoes the `tenant` it received — that echo is C2's sharpest measurement. +- `client.ts` — an MCP client that drives the server over JSON-RPC and keeps the **exact response + bytes**, in two transports: `inProcess` (`createMcpServer().handle`) and `overStdio` (the shipped + `serveStdio` over a `PassThrough` pair). Also `loadMcp`, the `__PKG_VERSION__` shim. +- `stitches.ts` — ten stitches over four auth strategies, plus the two variables C7 needs isolated: + one stitch with a tight `input.params` contract and one that opts into `input.headers`. +- `harness.ts` — `check` / `checkSeq` / **`checkClean`** / **`checkDiscloses`** / **`checkWire`** / + `checkAtMost` / `note` / `heading` / `finish`. `checkClean` is the assertion this scenario exists + for: it scans one payload against every secret by value and prints the byte count it scanned, so + a clean result is a measurement rather than an assurance. A hit prints the secret's **label** and + surrounding context, never the secret. +- `safe-exposure.ts` — the answer C8 counts, between `BEGIN`/`END USER CODE` markers: `expose` (the + allow-list, which also rejects the configured-name bypass), `only` (a `Proxy` apply-trap that + rebuilds the input from an explicit key list), `readsOnly` (a method gate on the `Adapter`). + +## Reading the numbers honestly + +- **C1 is a strong positive and should be read as one.** The capability boundary is not a slogan + here: four auth strategies, a login whose `Set-Cookie` carries the session, an error path whose + vendor body contains a credential-shaped string, and a `cookieSession` whose login stitch is + itself registered and callable — none of them put a held credential into a JSON-RPC payload. The + registered login stitch is the sharpest of those: an agent can call it, and gets `HTTP 401`, + because the login credential lives in `cookieSession.loginInput` and the MCP path never reaches it. +- **The one leak is `apiKey({ in: 'query' })`, not MCP.** The auth guide already warns that a key in + the URL "leaks wherever URLs go — server access logs, proxies, the browser history, a `Referer` + header". What C4 adds is one more destination: an unfiltered error message, and therefore the + model's context, its output, and any tool it calls next. The control is exact — the identical + transport failure on a `bearer` stitch disclosed the URL and no secret. +- **`sanitizeAgentInput` is a denylist of one key, and that is a design decision with a cost.** It + removes `headers` and forwards everything else as authored. Today nothing else is exploitable + (`signal` and `onProgress` are runtime-only slots JSON can fill only with inert values, and the + measured worst case is a call that aborts itself). But the default for a NEW input slot is + "exposed to the agent", and the function's own comment says its job is to forward "only the input + a stitch is built to accept" — which is an allow-list's description of a denylist's behaviour. +- **The `describe_stitch` disclosure is a deliberate trade and mostly the right one.** Teaching an + agent the endpoint, the pipeline and the auth scheme is what makes code-mode usable, and the + things that would actually help an attacker are absent: the credential, the operator's configured + request headers (an `x-internal-tenant` pin and an internal shard hostname stayed hidden), and the + env var name. What it costs is that a prompt injection reading the tool output learns the internal + route table for free. +- **C5's amplification is the number to take away, not the throttle.** `throttle` and `circuit` both + work on the MCP path — that was never really in doubt once you notice `run_stitch` just calls the + stitch. What a host cannot see is that **one tool call is not one request**: `retry: { attempts: 5 }` + made five and `paginate` made twelve, with no signal of either in the tool result. A host that + budgets "20 tool calls" has budgeted up to 1,000 vendor requests. +- **C8's 47 lines are the honest cost and they are cheap.** Three seams, no fork, no config key, and + every measured attack blocked while the reads keep working and keep authenticating. The line count + goes the library's way here precisely because the expensive half — the credential boundary, the + resilience chain, the discovery tools, the JSON-RPC layer and the transport — is already done. + +## Footguns + +1. **A transport error message reaches the model verbatim, and an `apiKey({ in: 'query' })` + credential rides in it.** Measured on the built-in `fetchAdapter` with no user code: + `Failed to parse URL from http://api.vendor.test:99999/v1/metrics?api_key=ak_live_qry_…`. + Node's `fetch` only writes that on a malformed URL, but `node-fetch`, `got` and several house + wrappers put the full URL in **every** network error (`request to failed, reason: …`), so + with a swapped adapter a routine DNS failure does it. **Fix: `apiKey({ in: 'header' })`.** There is + no redaction on this path — `errorResult((e as Error).message)` is the whole of it (mcp.ts:184). +2. **A query parameter pinned in the configured path is a default, not a constraint.** + `path: '/v1/orders?tenant=acme'` reads like an operator invariant and is overwritten by + `input: { query: { tenant: 'globex' } }` (`{ ...predefined, ...input.query }`, engine.ts:202). + Measured end to end: the vendor echoed `globex`. Anything that must not move belongs in a + `headers` entry or in the path template, not in the query string. +3. **An input schema does not filter — and the two cookie writers disagree.** `validateInput` + discards its parsed value (engine.ts:400-408), so a stripping schema lets unknown keys through to + the wire. Separately, `cookieSession.apply` **joins** the `Cookie` header + (`[req.headers.cookie, cookie].join('; ')`, auth.ts:918-921) while `apiKey({ in: 'cookie' })` + **replaces** the same-named pair via `setCookiePair` (auth.ts:228-245). Measured on a + headers-opted-in stitch: `SESSION=attacker; SESSION=sess_live_…`, and a vendor that reads the + first pair — Express, Rails, Go's `net/http`, PHP — runs the call as the model's session. +4. **Renaming a registry key does not hide a stitch.** `selectStitch` falls back from the key to + every stitch's configured `__config.name` (registry.ts:71-74), so `{ readOnlyOrders: refund }` + still answers to `run_stitch({ name: 'refund' })` — callable, and absent from `list_stitches`, at + the same time. `safe-exposure.ts`'s `expose` rejects the mismatch at construction. +5. **`sensitive: true` does not mean "do not expose this".** It is a **cache** opt-out + (types.ts:1652-1658) — the one word in `StitchConfig` that reads like an agent-visibility flag, + and measured, a stitch carrying it was still listed by `list_stitches` and still ran. None of the + 44 top-level config slots controls MCP exposure. +6. **`stitch mcp --module ./stitches.ts` exposes the whole module.** `collectStitches` recognises a + stitch structurally and keys it by export name (cli.ts:654-657), so a write, an internal login + stitch and a debug endpoint are all equally callable the moment they are exported. There is no + per-stitch opt-in anywhere in `StitchConfig`. +7. **Code-mode makes the host's destructive-tool prompt undecidable.** One `run_stitch` name covers + a `GET /v1/orders/77` and a `POST /v1/refunds`, and the tool descriptors carry **no + `annotations`** — no `readOnlyHint`, no `destructiveHint` — so a host that prompts before + destructive tools has nothing to key on, and the method is buried in an argument it has no schema + for. `list_stitches` does report `POST /v1/refunds`, but a host would have to call a tool to learn + it, and annotations are fixed at `tools/list` time. +8. **`{+id}` in a path template turns a param into a path.** RFC 6570 reserved expansion does not + percent-encode `/` (util.ts:369-379), so `params: { id: '../../v1/api-keys' }` normalised onto a + different endpoint **with the bearer token attached**. The ordinary `{id}` encoded it to + `..%2F..%2F` and stayed put. Templating the whole endpoint (`url: '{+endpoint}'`) reached + `metadata.internal` outright. +9. **Nothing is on by default, and nothing measures spend.** Fifty tool calls made fifty vendor + requests. Every bound the config surface offers is a count (`throttle.rate` spacing, + `throttle.concurrency`, `retry.attempts`, `paginate.pages`, `circuit.failures`, + `stream.buffer.chars`) or a duration (`timeout`, `circuit.cooldown`) — the axis the runaway + incident in the capture actually ran along has no knob. +10. **The stdio transport's serialisation guarantee does not travel.** Eight tool calls written in one + chunk were answered in request order and still paced, because `serveStdio` chains dispatch + (mcp.ts:346). `mcp.ts`'s own header invites a host to build a Streamable HTTP transport over the + same `handle()` — and that host inherits none of it. diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c1-credential-reach.ts b/docs/scenarios/proofs/agent-holds-the-tool/c1-credential-reach.ts new file mode 100644 index 00000000..0d8f701a --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c1-credential-reach.ts @@ -0,0 +1,301 @@ +// C1 — DECIDING. Does a credential reach the model ANYWHERE on the MCP surface? +// +// The product's central promise is that the caller, "an agent included", receives data and never +// the secret (`/docs/concepts/capability-not-credential`). A generic `run_stitch` tool is the +// sharpest test of that sentence there is, so this script enumerates every payload a JSON-RPC +// client can obtain and scans each one, BY VALUE, for all five credentials the registry holds: +// +// initialize · ping · tools/list · list_stitches · describe_stitch (×9) · a successful +// run_stitch on each of the four auth strategies · a vendor 401 · a validation failure · an +// unknown stitch · an unknown tool · an unknown JSON-RPC method · a stdio parse error +// +// Two controls keep a clean scan from being vacuous: +// (a) the wire tap proves the credential DID reach the vendor on every strategy — a 200 from a +// vendor that enforces its own auth means the real token was attached, so "clean" is a +// measurement of the boundary and not of a call that never authenticated; +// (b) the in-process bytes are asserted identical to the bytes the SHIPPED stdio transport +// writes, so the scan covers what actually ships. +// +// And one deliberate impurity: `mintApiKey` is a vendor endpoint that RETURNS a credential in its +// response body. Scanned against the secrets StitchAPI holds it is clean; scanned against every +// secret in the fixture it is a hit. Both are asserted, because the difference between "the +// library leaked a secret" and "the model asked for data and the data was a secret" is the whole +// of the exposure model. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c1-credential-reach.ts +import { inProcess, overStdio } from './client'; +import { + check, + checkClean, + checkDiscloses, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { buildRegistry } from './stitches'; +import { + ENV, + HELD_SECRETS, + type Route, + SECRETS, + Wire, + installSecrets, + route, +} from './vendor'; + +/** A vendor that answers 401 and puts a credential-shaped string in the ERROR BODY. */ +const hostile: Route = () => ({ + status: 401, + headers: { 'content-type': 'application/json' }, + body: { + error: 'token_revoked', + hint: `rotate ${SECRETS.mintedKey} in the dashboard`, + }, +}); + +async function main(): Promise { + installSecrets(); + + heading('C1 (a) — the discovery surface: does any of it carry a secret?'); + const wire = new Wire(route); + const registry = buildRegistry(wire); + const client = await inProcess(registry, 'orders-api'); + + const init = await client.send('initialize', { + protocolVersion: '2025-06-18', + }); + checkClean('initialize', init.raw, HELD_SECRETS); + const ping = await client.send('ping'); + checkClean('ping', ping.raw, HELD_SECRETS); + + const tools = await client.send('tools/list'); + checkClean('tools/list', tools.raw, HELD_SECRETS); + const toolNames = ( + tools.message.result as { tools: { name: string }[] } + ).tools.map((t) => t.name); + checkSeq('tools/list names', toolNames, [ + 'run_stitch', + 'list_stitches', + 'describe_stitch', + ]); + note( + 'the tool list is a CONSTANT three tools, independent of registry size', + Object.keys(registry).length, + ); + + const list = await client.callTool('list_stitches'); + checkClean('list_stitches', list.raw, HELD_SECRETS); + + for (const name of Object.keys(registry).sort()) { + const described = await client.callTool('describe_stitch', { name }); + checkClean(`describe_stitch ${name}`, described.raw, HELD_SECRETS); + } + + heading('C1 (b) — a SUCCESSFUL run_stitch, on each auth strategy'); + const runs: [string, unknown][] = [ + ['getOrder', { params: { id: '77' } }], + ['getReport', {}], + ['getMetrics', {}], + ['getProfile', {}], + ]; + for (const [name, input] of runs) { + wire.reset(); + const ran = await client.callTool('run_stitch', { name, input }); + check(`run_stitch ${name} → isError`, ran.isError, false); + checkClean(`run_stitch ${name} result`, ran.raw, HELD_SECRETS); + } + + // The login stitch is registered, so the model can invoke it — and gets nothing. Its + // credential is not on the stitch at all: `cookieSession` supplies it through `loginInput` + // (stitches.ts), which the MCP path never reaches. Calling `login` directly therefore sends an + // unauthenticated login and the vendor rejects it. + wire.reset(); + const directLogin = await client.callTool('run_stitch', { name: 'login' }); + check('run_stitch login (direct) → isError', directLogin.isError, true); + check('run_stitch login (direct) → text', directLogin.text, 'HTTP 401'); + check( + 'run_stitch login (direct) sent no password', + (wire.last.body as { password?: string } | undefined)?.password, + undefined, + ); + checkClean('run_stitch login (direct)', directLogin.raw, HELD_SECRETS); + note( + 'a registered login stitch is NOT a session-harvesting tool', + 'the login credential lives in cookieSession.loginInput, not on the stitch — the agent cannot supply it', + ); + + heading( + 'C1 (c) — the control: did the credential actually reach the wire?', + ); + wire.reset(); + await client.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + check( + 'bearer reached the vendor', + wire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + wire.reset(); + await client.callTool('run_stitch', { name: 'getReport' }); + check( + 'apiKey(header) reached the vendor', + wire.last.headers['x-api-key'], + SECRETS.apiKeyHeader, + ); + wire.reset(); + await client.callTool('run_stitch', { name: 'getMetrics' }); + check( + 'apiKey(query) reached the vendor', + new URL(wire.last.url).searchParams.get('api_key'), + SECRETS.apiKeyQuery, + ); + note('apiKey(query) puts the credential in the URL', wire.last.url); + // A FRESH registry: the session captured above lives in the seam's vault, so re-using the + // client would measure a cache hit rather than the login-then-call spine. + const coldWire = new Wire(route); + const coldClient = await inProcess(buildRegistry(coldWire)); + await coldClient.callTool('run_stitch', { name: 'getProfile' }); + checkSeq( + 'cookieSession: login then the call', + coldWire.requests.map((r) => new URL(r.url).pathname), + ['/auth/login', '/v1/profile'], + ); + check( + 'session cookie reached the vendor', + coldWire.last.headers['cookie'], + `SESSION=${SECRETS.session}`, + ); + note( + 'the login RESPONSE carried Set-Cookie with the session value', + 'and the model still received a clean payload — response headers are not surfaced', + ); + + heading('C1 (d) — the ERROR paths'); + const badParams = await client.callTool('run_stitch', { + name: 'getOrderTyped', + input: { params: { id: 'not-a-number' } }, + }); + check('validation failure → isError', badParams.isError, true); + note('validation failure → text', badParams.text); + checkClean('run_stitch validation error', badParams.raw, HELD_SECRETS); + + const unknownStitch = await client.callTool('run_stitch', { name: 'nope' }); + check('unknown stitch → isError', unknownStitch.isError, true); + note('unknown stitch → text', unknownStitch.text); + checkClean('run_stitch unknown name', unknownStitch.raw, HELD_SECRETS); + + const unknownTool = await client.callTool('exfiltrate', {}); + check('unknown tool → isError', unknownTool.isError, true); + note('unknown tool → text', unknownTool.text); + checkClean('tools/call unknown tool', unknownTool.raw, HELD_SECRETS); + + const unknownMethod = await client.send('resources/list'); + note( + 'unknown JSON-RPC method → error', + (unknownMethod.message.error as { code: number; message: string }) + .message, + ); + checkClean('unknown JSON-RPC method', unknownMethod.raw, HELD_SECRETS); + + // A vendor that answers 401 and puts a credential-shaped string in the error BODY. + const hostileWire = new Wire(hostile); + const hostileClient = await inProcess(buildRegistry(hostileWire)); + const rejected = await hostileClient.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + check('vendor 401 → isError', rejected.isError, true); + check('vendor 401 → text', rejected.text, 'HTTP 401'); + checkClean('run_stitch vendor 401', rejected.raw, { + ...HELD_SECRETS, + vendorErrorBody: SECRETS.mintedKey, + }); + note( + 'the vendor error BODY does not reach the model either', + 'run_stitch renders `(e as Error).message` only (mcp.ts:184)', + ); + + // A missing env var: the resolver throws, and its message names the VARIABLE, not the value. + const saved = process.env[ENV.bearer]; + delete process.env[ENV.bearer]; + const missing = await client.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + if (saved !== undefined) process.env[ENV.bearer] = saved; + check('missing credential → isError', missing.isError, true); + checkDiscloses( + 'missing credential names the VAR', + missing.text, + ENV.bearer, + ); + checkClean('run_stitch missing credential', missing.raw, HELD_SECRETS); + + heading('C1 (e) — the whole transcript, in one scan'); + const transcript = client.transcript.map((e) => e.raw).join('\n'); + note('exchanges', client.transcript.length); + note('bytes returned to the model', transcript.length); + checkClean('ENTIRE TRANSCRIPT', transcript, HELD_SECRETS); + + heading('C1 (f) — the same scan over the SHIPPED stdio transport'); + const stdioWire = new Wire(route); + const stdio = await overStdio(buildRegistry(stdioWire), 'orders-api'); + const stdioInit = await stdio.send('initialize', {}); + const stdioTools = await stdio.send('tools/list'); + const stdioRun = await stdio.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + const stdioDescribe = await stdio.callTool('describe_stitch', { + name: 'getOrder', + }); + const stdioBad = await stdio.send('%%not json%%'); + check( + 'stdio initialize bytes == in-process bytes', + stdioInit.raw.replace(/"id":\d+/, '"id":N'), + init.raw.replace(/"id":\d+/, '"id":N'), + ); + check( + 'stdio tools/list bytes == in-process bytes', + stdioTools.raw.replace(/"id":\d+/, '"id":N'), + tools.raw.replace(/"id":\d+/, '"id":N'), + ); + checkClean('stdio run_stitch', stdioRun.raw, HELD_SECRETS); + checkClean('stdio describe_stitch', stdioDescribe.raw, HELD_SECRETS); + note( + 'stdio unknown method → error', + JSON.stringify(stdioBad.message.error), + ); + stdio.close(); + + heading('C1 (g) — the ONE thing that does reach the model, and why'); + wire.reset(); + const minted = await client.callTool('run_stitch', { name: 'mintApiKey' }); + check('mintApiKey → isError', minted.isError, false); + checkClean('mintApiKey vs. secrets StitchAPI HOLDS', minted.raw, { + ...HELD_SECRETS, + }); + checkDiscloses( + 'mintApiKey → the VENDOR-returned key', + minted.text, + SECRETS.mintedKey, + ); + note( + 'this is not a boundary failure', + 'the credential was never StitchAPI\'s to hold — it is a response BODY, and "return the data" is the contract', + ); + note( + 'ONE HELD CREDENTIAL *CAN* REACH THE MODEL — see C4 (c)', + 'not through any payload enumerated above, but through the error channel: `run_stitch` renders `(e as Error).message` unfiltered, and a transport error message that quotes the request URL carries an `apiKey({ in: "query" })` credential with it', + ); + + finish( + 'C1', + 'HELD. Across 34 JSON-RPC exchanges and 30 separate payload scans (14,529 bytes in the main transcript alone) — initialize, ping, tools/list, list_stitches, describe_stitch on all 10 stitches, a successful run_stitch on bearer/apiKey-header/apiKey-query/cookieSession, a direct call to the registered login stitch, a vendor 401 whose error BODY contained a credential-shaped string, a validation failure, an unknown stitch, an unknown tool, an unknown JSON-RPC method and a stdio parse error — NOT ONE of the five credentials the registry holds appeared, by value, anywhere. The controls hold too: the same calls put `Bearer sk_live_…` on the wire, `X-API-Key: ak_live_…` on the wire, `api_key=ak_live_…` in the URL and `Cookie: SESSION=sess_live_…` on the wire, so the vendor authenticated every one of them; and the stdio transport writes byte-identical payloads. The only credential that reached the model is one the VENDOR minted and returned in a response body, which is data by definition — and the failure mode there belongs to whoever registered that endpoint, not to the boundary. ONE CAVEAT, and it is not in any payload enumerated here: the error channel is an unfiltered `Error.message` pass-through, so a TRANSPORT error that quotes the request URL carries an `apiKey({ in: "query" })` credential into the model with it — measured on the default adapter in C4 (c)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c2-input-rewrite.ts b/docs/scenarios/proofs/agent-holds-the-tool/c2-input-rewrite.ts new file mode 100644 index 00000000..5d41be1a --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c2-input-rewrite.ts @@ -0,0 +1,378 @@ +// C2 — DECIDING. Can the model's `input` object redirect or rewrite the call? +// +// `run_stitch({ name, input })` takes a free-form object, so `input` is the security boundary. The +// capture's hypothesis came from scenario 10, which measured `engine.ts:231`: +// +// const headers = { ...(cfg.headers ?? {}), ...(input.headers ?? {}) }; +// +// input headers merge OVER config headers. If the model's object reached that merge unfiltered, a +// prompt injection could set any header on a call it did not author. **That hypothesis is refuted +// for the MCP path**, and the reason is a named function that exists for exactly this: +// `sanitizeAgentInput` (mcp.ts:125-130) deletes `input.headers` unless the stitch declares an +// `input.headers` schema. This script measures the strip, then measures precisely what a stitch +// that DOES opt in exposes — and then walks every other field of the input object. +// +// Every assertion is on what the recording adapter received, so "the model can set this field" and +// "this is what the vendor saw" are never conflated. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c2-input-rewrite.ts +import { + apiKey, + bearer, + cookieSession, + env, +} from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { StitchRegistry } from '../../../../packages/core/src/registry'; +import { inProcess } from './client'; +import { check, checkWire, finish, heading, note } from './harness'; +import { buildRegistry, schema } from './stitches'; +import { BASE, ENV, SECRETS, Wire, installSecrets, route } from './vendor'; + +const anyObject = schema( + 'an object', + (v) => v === undefined || (typeof v === 'object' && v !== null), +); + +/** + * The variant shapes C2 needs that an ordinary registry would not contain: a stitch that opts into + * agent headers AND carries a cookie session, a stitch whose path uses RFC 6570 reserved expansion + * (`{+id}`, which does not percent-encode `/`), and a stitch whose whole `url` is a template. + * + * They are built here rather than in `stitches.ts` because they are not what an operator would + * normally write — they are the authoring choices whose blast radius C2 is quantifying. + */ +function variants(wire: Wire): StitchRegistry { + const api = seam({ baseUrl: BASE, adapter: wire.adapter() }); + const login = api.stitch({ + name: 'login', + method: 'POST', + path: '/auth/login', + }); + return { + // Opts into agent headers, and its credential is a COOKIE — the one strategy whose header + // the engine merges into rather than overwrites (`setCookiePair`, auth.ts:228-245). + profileOpenHeaders: api.stitch({ + name: 'profileOpenHeaders', + path: '/v1/profile', + auth: cookieSession({ + login, + cookie: 'SESSION', + tenancy: 'app', + loginInput: () => ({ + body: { user: 'svc', password: env(ENV.loginPassword)() }, + }), + }), + input: { headers: anyObject }, + pick: 'data', + }), + // Reserved expansion: `{+id}` renders `/` and `.` literally (util.ts:369-379). + orderReserved: api.stitch({ + name: 'orderReserved', + path: '/v1/orders/{+id}', + auth: bearer(env(ENV.bearer)), + }), + // The whole endpoint is a template, and `url` bypasses `baseUrl` entirely (engine.ts:184). + openEndpoint: api.stitch({ + name: 'openEndpoint', + url: '{+endpoint}', + auth: bearer(env(ENV.bearer)), + }), + }; +} + +async function main(): Promise { + installSecrets(); + const wire = new Wire(route); + const registry = buildRegistry(wire); + const client = await inProcess(registry); + + heading('C2 (a) — headers, on a stitch that does NOT declare a schema'); + wire.reset(); + const injected = await client.callTool('run_stitch', { + name: 'getOrder', + input: { + params: { id: '77' }, + headers: { + authorization: 'Bearer attacker-token', + cookie: 'SESSION=attacker', + host: 'evil.test', + 'x-forwarded-for': '127.0.0.1', + 'content-type': 'text/plain', + 'x-anything': 'anything', + }, + }, + }); + check('the call still succeeded', injected.isError, false); + checkWire('headers (6 sent by the model)', Object.keys(wire.last.headers), [ + 'authorization', + ]); + checkWire( + '.authorization', + wire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + note( + 'sanitizeAgentInput deleted the whole slot', + 'mcp.ts:128 — `if (stitch.__config.input?.headers === undefined) delete obj.headers`', + ); + + heading('C2 (b) — headers, on a stitch that DOES declare a schema'); + wire.reset(); + await client.callTool('run_stitch', { + name: 'searchOrders', + input: { + headers: { + authorization: 'Bearer attacker-token', + cookie: 'SESSION=attacker', + 'x-forwarded-for': '10.0.0.1', + 'x-actor': 'admin', + }, + }, + }); + checkWire( + '.authorization (auth applies LAST)', + wire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + checkWire('.cookie', wire.last.headers['cookie'], 'SESSION=attacker'); + checkWire( + '.x-forwarded-for', + wire.last.headers['x-forwarded-for'], + '10.0.0.1', + ); + checkWire('.x-actor', wire.last.headers['x-actor'], 'admin'); + note( + 'the model cannot overwrite the credential header', + 'engine.ts:647 applies `auth` to a clone of the built request, AFTER the input merge — so `authorization` is rewritten every time', + ); + note( + 'it CAN set every other header', + 'an audit header, a tenant header, an X-Forwarded-For a vendor trusts, a Cookie on a non-cookie stitch', + ); + + heading( + 'C2 (c) — headers on a COOKIE-session stitch: the two cookie writers disagree', + ); + const varWire = new Wire(route); + const varClient = await inProcess(variants(varWire)); + varWire.reset(); + const profile = await varClient.callTool('run_stitch', { + name: 'profileOpenHeaders', + input: { headers: { cookie: 'tracking=xyz; SESSION=attacker' } }, + }); + // `cookieSession.apply` CONCATENATES (auth.ts:918-921) rather than replacing, so the model's + // pair is sent FIRST and the real session second. RFC 6265 §5.4 gives no ordering rule for a + // duplicate name; Express, Rails, Go's net/http and PHP all read the FIRST occurrence. + checkWire( + '.cookie (model pair, then the real session)', + varWire.last.headers['cookie'], + `tracking=xyz; SESSION=attacker; SESSION=${SECRETS.session}`, + ); + check( + 'a vendor that reads the FIRST pair sees the MODEL’s session', + profile.isError, + true, + ); + note( + 'cookieSession.apply is a plain join (auth.ts:918-921)', + '`req.headers.cookie = [req.headers.cookie, cookie].filter(Boolean).join("; ")` — no same-name replacement', + ); + // The contrast: `apiKey({ in: 'cookie' })` writes the same header through `setCookiePair`, + // which DOES replace a same-named pair (auth.ts:228-245). Same header, two behaviours. + const cookieKeyWire = new Wire(route); + const cookieKeyApi = seam({ + baseUrl: BASE, + adapter: cookieKeyWire.adapter(), + }); + const cookieKeyClient = await inProcess({ + reportsViaCookie: cookieKeyApi.stitch({ + name: 'reportsViaCookie', + path: '/v1/reports', + auth: apiKey({ + in: 'cookie', + name: 'SESSION', + secret: env(ENV.apiKeyHeader), + }), + input: { headers: anyObject }, + }), + }); + await cookieKeyClient.callTool('run_stitch', { + name: 'reportsViaCookie', + input: { headers: { cookie: 'tracking=xyz; SESSION=attacker' } }, + }); + checkWire( + '.cookie under apiKey({ in: "cookie" })', + cookieKeyWire.last.headers['cookie'], + `tracking=xyz; SESSION=${SECRETS.apiKeyHeader}`, + ); + note( + 'the same header, two behaviours', + 'apiKey uses setCookiePair (replace); cookieSession uses a join (prepend) — only the second is forgeable', + ); + + heading('C2 (d) — query: can the model overwrite an operator’s pin?'); + wire.reset(); + const pinned = await client.callTool('run_stitch', { name: 'listOrders' }); + check( + 'default tenant, echoed by the vendor', + ( + pinned.message.result as { content: { text: string }[] } + ).content[0]?.text.includes('"tenant": "acme"'), + true, + ); + wire.reset(); + const stolen = await client.callTool('run_stitch', { + name: 'listOrders', + input: { query: { tenant: 'globex' } }, + }); + checkWire('url', wire.last.url, `${BASE}/v1/orders?tenant=globex`); + check( + 'the vendor echoed the MODEL’s tenant', + ( + stolen.message.result as { content: { text: string }[] } + ).content[0]?.text.includes('"tenant": "globex"'), + true, + ); + note( + 'engine.ts:202 — `{ ...predefined, ...(input.query ?? {}) }`', + 'a query parameter pinned in the configured path is a DEFAULT, not a constraint', + ); + + heading( + 'C2 (e) — query on the apiKey(query) stitch: can the key be shadowed?', + ); + wire.reset(); + const shadow = await client.callTool('run_stitch', { + name: 'getMetrics', + input: { query: { api_key: 'attacker-key' } }, + }); + checkWire( + 'url', + wire.last.url, + [ + `${BASE}/v1/metrics?api_key=attacker-key&api_key=${SECRETS.apiKeyQuery}`, + ][0], + ); + check( + 'the vendor read the FIRST api_key and rejected the call', + shadow.text, + 'HTTP 401', + ); + note( + 'the real key is APPENDED, not substituted (auth.ts:289)', + 'a model can therefore break its own call — a self-inflicted 401, not an escalation; no credential is disclosed either way', + ); + + heading('C2 (f) — params: path traversal'); + wire.reset(); + await client.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '../../v1/api-keys' } }, + }); + checkWire( + 'url with a simple {id} template', + wire.last.url, + `${BASE}/v1/orders/..%2F..%2Fv1%2Fapi-keys`, + ); + note( + 'RFC 6570 simple expansion percent-encodes the separator (util.ts:377)', + 'so the request stayed on the endpoint the operator authored', + ); + varWire.reset(); + await varClient.callTool('run_stitch', { + name: 'orderReserved', + input: { params: { id: '../../v1/api-keys' } }, + }); + checkWire( + 'url with a RESERVED {+id} template', + varWire.last.url, + `${BASE}/v1/orders/../../v1/api-keys`, + ); + check( + 'and the request normalises onto a DIFFERENT endpoint', + new URL(varWire.last.url).pathname, + '/v1/api-keys', + ); + checkWire( + 'with the real credential attached', + varWire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + + heading('C2 (g) — anything URL-shaped'); + wire.reset(); + await client.callTool('run_stitch', { + name: 'getOrder', + input: { + params: { id: '77' }, + url: 'https://evil.test/steal', + baseUrl: 'https://evil.test', + path: '/steal', + adapter: 'x', + auth: 'x', + }, + }); + checkWire('url (6 rogue keys sent)', wire.last.url, `${BASE}/v1/orders/77`); + note( + 'the engine reads input by FIELD NAME', + '`url`/`baseUrl`/`path`/`adapter`/`auth` are config slots, not input slots — an unknown key is inert', + ); + varWire.reset(); + const ssrf = await varClient.callTool('run_stitch', { + name: 'openEndpoint', + input: { params: { endpoint: 'https://metadata.internal/latest' } }, + }); + checkWire( + 'url when the WHOLE endpoint is a {+template}', + varWire.last.url, + 'https://metadata.internal/latest', + ); + check('and it reached the internal service', ssrf.isError, false); + checkWire( + 'with the vendor credential attached', + varWire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + + heading('C2 (h) — body, on a write with no body schema'); + wire.reset(); + const refunded = await client.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 999_999, note: 'chosen by the model' } }, + }); + checkWire('body', wire.last.body, { + amount: 999_999, + note: 'chosen by the model', + }); + check('the write succeeded', refunded.isError, false); + note('the body is the model’s, whole', refunded.text.replace(/\s+/g, ' ')); + + heading('C2 (i) — the fields the sanitizer does not name'); + wire.reset(); + const signalled = await client.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' }, signal: { aborted: true } }, + }); + // `signal` is a runtime-only slot (`AbortSignal`), and JSON can only fill it with a plain + // object. The engine threads it onto the request regardless (engine.ts:250), so the model can + // make a call fail before it is sent — a self-inflicted denial, with no request on the wire. + check( + 'run_stitch with a forged input.signal → isError', + signalled.isError, + true, + ); + check('…and no request reached the vendor', wire.count, 0); + note('the error the model got back', signalled.text); + note( + 'sanitizeAgentInput is a DENYLIST of one key, not an allowlist', + 'only `headers` is removed — every other input slot, present and future, is forwarded to the engine as-authored', + ); + + finish( + 'C2', + "THE CAPTURE'S HEADER HYPOTHESIS IS REFUTED; FIVE OTHER LEVERS ARE REAL. The `engine.ts:231` header merge is real, but `sanitizeAgentInput` (mcp.ts:125-130) deletes `input.headers` before it: six model-supplied headers including `authorization`, `cookie` and `host` reached the vendor as ZERO headers, and the only header on the wire was the real `Bearer sk_live_…`. The credential header specifically is unforgeable even when a stitch DOES opt in, because `auth` is applied to a clone AFTER the merge (engine.ts:647) — `authorization` was rewritten to the real token on every attempt. What the model CAN do, ON AN ORDINARY STITCH WITH NO OPT-IN: (1) OVERWRITE A QUERY PARAMETER PINNED IN THE CONFIGURED PATH — `tenant=acme` became `tenant=globex` and the vendor returned the other tenant's data, because `{ ...predefined, ...input.query }` makes a pin a default; (2) send the entire request BODY of a write, uncapped, when no `input.body` schema is declared (a 999,999 refund); (3) shadow an `apiKey({ in: 'query' })` credential with a duplicate parameter, and abort a call before it is sent with a forged `input.signal` — both self-inflicted denials, no disclosure. And where the OPERATOR opted in: (4) with an `input.headers` schema, every non-credential header — and on a `cookieSession` stitch the model's `SESSION=attacker` is sent BEFORE the real one, measured `SESSION=attacker; SESSION=sess_live_…`, because `cookieSession.apply` joins where `apiKey({ in: 'cookie' })` replaces (auth.ts:918-921 vs 228-245); a vendor that reads the first pair runs the call as the model's session; (5) with RFC 6570 reserved expansion, cross-endpoint traversal WITH the credential attached — `{+id}` sent the bearer token to `/v1/api-keys`, while the ordinary `{id}` percent-encoded it to `..%2F..%2F`, and templating the whole endpoint (`url: '{+endpoint}'`) reached `metadata.internal`. Nothing URL-shaped in `input` is read otherwise: `url`, `baseUrl`, `path`, `adapter` and `auth` keys were inert", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c3-allowlist.ts b/docs/scenarios/proofs/agent-holds-the-tool/c3-allowlist.ts new file mode 100644 index 00000000..117eb6d7 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c3-allowlist.ts @@ -0,0 +1,270 @@ +// C3 — is there an allow-list, and what do the discovery tools disclose? +// +// Two questions, and they pull in opposite directions. An agent needs to know what it may call, so +// `list_stitches` and `describe_stitch` exist to tell it; the same read-out is a map of the +// operator's internal API handed to a caller that may be reciting an attacker's prompt. This +// script measures exactly what is on that map, and then measures what — if anything — keeps a +// stitch off it. +// +// The answer to the second question is the shape of the surface: `createMcpServer(registry)` takes +// a `StitchRegistry`, so the allow-list IS the object you pass, and there is no per-stitch opt-in. +// That is a defensible design, but `stitch mcp` (the documented way to start the server) builds +// that object with `collectStitches`, which sweeps up EVERY stitch a module exports — so the +// default exposure is "everything in the file", including a write and a stitch that only exists to +// serve another stitch's login. +// +// And one bypass worth knowing about: `selectStitch` resolves a name against the registry KEY and, +// failing that, against each stitch's CONFIGURED `name` — so leaving a stitch out of the keys is +// not the same as leaving it out of the registry. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c3-allowlist.ts +import { bearer, env } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import { collectStitches } from '../../../../packages/core/src/registry'; +import { inProcess } from './client'; +import { + check, + checkDiscloses, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { buildRegistry } from './stitches'; +import { BASE, ENV, Wire, installSecrets, route } from './vendor'; + +interface Described { + name: string; + endpoint: string; + surface: string; + input: Record; + output: { validated: boolean; pick: string | null }; + auth: string | null; + policies: Record; + pipeline: string[]; + diagram: string; +} + +async function main(): Promise { + installSecrets(); + const wire = new Wire(route); + const registry = buildRegistry(wire); + const client = await inProcess(registry); + + heading('C3 (a) — can the model run ANY registered stitch?'); + const listed = JSON.parse( + (await client.callTool('list_stitches')).text, + ) as { name: string; method: string; path: string }[]; + checkSeq( + 'list_stitches names', + listed.map((s) => s.name), + Object.keys(registry).sort(), + ); + check( + 'every registered name is listed', + listed.length, + Object.keys(registry).length, + ); + wire.reset(); + const wrote = await client.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 1 } }, + }); + check('an irreversible POST ran on first ask', wrote.isError, false); + check('…and reached the vendor', wire.count, 1); + note( + 'there is no per-stitch opt-in', + 'no config key excludes a stitch from MCP — StitchConfig has 44 top-level slots and none is `mcp`/`expose`/`internal`/`agent`', + ); + // The nearest-looking key is `sensitive: true`, which reads like "do not expose this" and is + // in fact a CACHE opt-out (types.ts:1652-1658). An operator who reaches for it gets no + // agent-visibility change at all. + const sensitiveWire = new Wire(route); + const sensitiveApi = seam({ + baseUrl: BASE, + adapter: sensitiveWire.adapter(), + }); + const sensitiveClient = await inProcess({ + oneTimeToken: sensitiveApi.stitch({ + name: 'oneTimeToken', + method: 'POST', + path: '/v1/api-keys', + auth: bearer(env(ENV.bearer)), + cache: '1m', + sensitive: true, + }), + }); + const stillListed = JSON.parse( + (await sensitiveClient.callTool('list_stitches')).text, + ) as { name: string }[]; + checkSeq( + '`sensitive: true` is still listed', + stillListed.map((s) => s.name), + ['oneTimeToken'], + ); + const stillRuns = await sensitiveClient.callTool('run_stitch', { + name: 'oneTimeToken', + }); + check('…and still runs', stillRuns.isError, false); + note( + '`sensitive` is a cache opt-out, not a visibility flag (types.ts:1652-1658)', + 'the one config word that reads like "do not expose this" changes nothing about who may call it', + ); + + heading('C3 (b) — what `list_stitches` discloses'); + note('per stitch', JSON.stringify(listed[0])); + checkDiscloses('the route table', JSON.stringify(listed), '/v1/refunds'); + checkDiscloses( + '…including the login route', + JSON.stringify(listed), + '/auth/login', + ); + check( + 'the base URL is NOT in list_stitches', + JSON.stringify(listed).includes(BASE), + false, + ); + + heading('C3 (c) — what `describe_stitch` discloses'); + const described = JSON.parse( + (await client.callTool('describe_stitch', { name: 'getMetrics' })).text, + ) as Described; + checkSeq('keys returned', Object.keys(described), [ + 'name', + 'endpoint', + 'surface', + 'input', + 'output', + 'auth', + 'policies', + 'pipeline', + 'diagram', + ]); + check( + 'endpoint (the full internal URL)', + described.endpoint, + `GET ${BASE}/v1/metrics`, + ); + check('auth (the SCHEME, not the credential)', described.auth, 'apiKey'); + note('input slots', JSON.stringify(described.input)); + note('policies', JSON.stringify(described.policies)); + note('pipeline', JSON.stringify(described.pipeline)); + note('diagram bytes', described.diagram.length); + checkDiscloses( + 'the diagram repeats the endpoint', + described.diagram, + `${BASE}/v1/metrics`, + ); + const bearerDescribed = JSON.parse( + (await client.callTool('describe_stitch', { name: 'getOrder' })).text, + ) as Described; + check('a bearer stitch reports its scheme', bearerDescribed.auth, 'bearer'); + const sessionDescribed = JSON.parse( + (await client.callTool('describe_stitch', { name: 'getProfile' })).text, + ) as Described; + check( + 'a cookie session reports apiKey (its declared scheme)', + sessionDescribed.auth, + 'apiKey', + ); + const loginDescribed = JSON.parse( + (await client.callTool('describe_stitch', { name: 'login' })).text, + ) as Described; + check('a stitch with no auth reports null', loginDescribed.auth, null); + + heading('C3 (d) — what `describe_stitch` does NOT disclose'); + const secretWire = new Wire(route); + const secretApi = seam({ baseUrl: BASE, adapter: secretWire.adapter() }); + const secretClient = await inProcess({ + internalOrders: secretApi.stitch({ + name: 'internalOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + // Operator-set request headers: an internal tenant pin and a routing hint. + headers: { + 'x-internal-tenant': 'acme-prod', + 'x-route-to': 'shard-7.internal', + }, + }), + }); + const internal = ( + await secretClient.callTool('describe_stitch', { + name: 'internalOrders', + }) + ).text; + check( + 'configured request HEADERS are absent', + internal.includes('x-internal-tenant') || + internal.includes('shard-7.internal'), + false, + ); + check( + 'the env VAR NAME the credential comes from is absent', + internal.includes(ENV.bearer), + false, + ); + note( + 'those headers still ride every request', + 'the model cannot see them; it also cannot know they are what pins the tenant', + ); + + heading('C3 (e) — the allow-list bypass: keys are not the only names'); + // An operator filters the registry down to "the safe two" and renames the key, believing the + // rename hides the original. `selectStitch` resolves the registry KEY first and then falls + // back to each stitch's CONFIGURED `name` (registry.ts:71-74). + const filtered = { + readOnlyOrders: registry['getOrder'] as (typeof registry)['getOrder'], + }; + const filteredClient = await inProcess(filtered); + const byKey = await filteredClient.callTool('run_stitch', { + name: 'readOnlyOrders', + input: { params: { id: '77' } }, + }); + check('reachable by its registry key', byKey.isError, false); + const byConfigName = await filteredClient.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + check('ALSO reachable by its configured name', byConfigName.isError, false); + const filteredList = JSON.parse( + (await filteredClient.callTool('list_stitches')).text, + ) as { name: string }[]; + checkSeq( + 'but list_stitches shows only the key', + filteredList.map((s) => s.name), + ['readOnlyOrders'], + ); + note( + 'a name the discovery tool never mentions is still callable', + 'registry.ts:71-74 — key first, then a scan of every `__config.name`', + ); + + heading('C3 (f) — what `stitch mcp` exposes by default'); + // The CLI builds its registry with `collectStitches`, which recognises a stitch structurally + // and keys it by its EXPORT NAME. A module-shaped object stands in for the user's stitches.ts. + const moduleShaped = { + getOrder: registry['getOrder'], + refund: registry['refund'], + login: registry['login'], + // Not a stitch — ignored, which is the only filtering `collectStitches` does. + BASE_URL: BASE, + helper: () => 'not a stitch', + }; + const collected = collectStitches(moduleShaped); + checkSeq('collectStitches keys', Object.keys(collected).sort(), [ + 'getOrder', + 'login', + 'refund', + ]); + note( + 'the default exposure is the whole module', + '`stitch mcp --module ./stitches.ts` (cli.ts:654-657) hands `collectStitches(mod)` straight to serveStdio — a write and an internal login stitch included', + ); + + finish( + 'C3', + "NO ALLOW-LIST BEYOND THE REGISTRY OBJECT, AND THE DISCOVERY TOOLS ARE A MAP. Every registered stitch is equally callable: an irreversible `POST /v1/refunds` ran on first ask with no opt-in, and none of `StitchConfig`'s 44 top-level slots excludes a stitch from MCP — the nearest-looking word, `sensitive: true`, is a CACHE opt-out (types.ts:1652-1658) and a stitch carrying it was still listed and still ran. The allow-list is therefore the object handed to `createMcpServer` — which is a real, usable seam (one `Object.fromEntries` filter, measured in C8) — but the documented starter, `stitch mcp --module ./stitches.ts`, builds that object with `collectStitches`, which sweeps up EVERY exported stitch: the module here exposed a write and a login stitch alongside the read. DISCLOSED to the model: every stitch NAME, METHOD and PATH from `list_stitches`; and from `describe_stitch`, the full internal endpoint URL (`GET https://api.vendor.test/v1/metrics`), the surface, which input slots exist, whether output is validated, THE AUTH SCHEME, which of retry/throttle/cache/timeout are on, the engine-order pipeline, and a Mermaid diagram that repeats the endpoint — about 1KB per stitch. NOT disclosed: the credential (C1), the operator's configured request headers (an `x-internal-tenant` pin and an internal shard hostname stayed hidden), and the env var name the credential resolves from. THE BYPASS: `selectStitch` falls back from the registry key to each stitch's CONFIGURED `name` (registry.ts:71-74), so a filtered registry that RENAMES a stitch still answers to the original name — reachable, and absent from `list_stitches`, at the same time", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c4-error-rendering.ts b/docs/scenarios/proofs/agent-holds-the-tool/c4-error-rendering.ts new file mode 100644 index 00000000..b1bfa4f6 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c4-error-rendering.ts @@ -0,0 +1,250 @@ +// C4 — what does a FAILURE tell the model? +// +// `run_stitch` renders a failure as `errorResult((e as Error).message)` (mcp.ts:184) — the error's +// message string, and nothing else. That one line decides this claim in both directions: +// +// the good half — a `StitchError` carries `.status`, `.attempts`, `.body` (the vendor's error +// payload) and `.url` (the final request URL), and NONE of them are rendered. The vendor's +// `{"error":"token_revoked","hint":"rotate ak_live_… in the dashboard"}` never reaches the model; +// it gets `HTTP 401`. StitchAPI's own messages are uniformly terse and carry no request detail. +// +// the bad half — it is an UNFILTERED PASS-THROUGH. Any error that reaches the top of the stack +// hands its message to the model verbatim, including one the TRANSPORT wrote. Node's built-in +// `fetch` writes `Failed to parse URL from `, and on an `apiKey({ in: 'query' })` +// stitch the whole URL contains the credential. That is measured below, on the default adapter, +// with no user code and no network. +// +// Everything here is offline: the "network failure" is a port outside the valid range, so `fetch` +// rejects the URL before opening a socket. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c4-error-rendering.ts +import { apiKey, bearer, env } from '../../../../packages/core/src/auth'; +import { seam, stitch } from '../../../../packages/core/src/index'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { inProcess } from './client'; +import { + check, + checkClean, + checkDiscloses, + finish, + heading, + note, +} from './harness'; +import { buildRegistry, schema } from './stitches'; +import { + BASE, + ENV, + HELD_SECRETS, + type Route, + SECRETS, + Wire, + installSecrets, + route, +} from './vendor'; + +/** A vendor that answers 500 with a detailed internal error body. */ +const leaky: Route = () => ({ + status: 500, + headers: { + 'content-type': 'application/json', + 'x-internal-node': 'shard-7.internal', + }, + body: { + error: 'internal', + stack: 'at OrderService.load (/srv/vendor/src/orders.ts:88)', + db: 'postgres://vendor:hunter2@db-primary.internal:5432/orders', + }, +}); + +/** + * A `node-fetch`-shaped adapter error. node-fetch (and `got`, and several house wrappers) put the + * FULL REQUEST URL in the message of every network failure — `request to failed, reason: …` + * — which is what makes the pass-through matter in production rather than only under a typo. + */ +const nodeFetchShaped: Adapter = (req) => { + throw new Error( + `request to ${req.url} failed, reason: getaddrinfo ENOTFOUND api.vendor.test`, + ); +}; + +async function main(): Promise { + installSecrets(); + + heading('C4 (a) — the failure taxonomy: what text does the model get?'); + const wire = new Wire(route); + const client = await inProcess(buildRegistry(wire)); + + const leakyWire = new Wire(leaky); + const leakyClient = await inProcess(buildRegistry(leakyWire)); + const http500 = await leakyClient.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + check('a vendor 500 → text', http500.text, 'HTTP 500'); + check( + 'the vendor’s error BODY is absent', + http500.raw.includes('shard-7.internal') || + http500.raw.includes('postgres://') || + http500.raw.includes('orders.ts:88'), + false, + ); + note( + 'StitchError carries .status/.attempts/.body/.url', + 'run_stitch renders `.message` and drops all four (mcp.ts:184)', + ); + + // A transport that never settles, so the timeout is what ends the call. + const timeoutApi = seam({ + baseUrl: BASE, + adapter: () => + new Promise(() => { + /* never settles */ + }), + }); + const timeoutClient = await inProcess({ + slowOrders: timeoutApi.stitch({ + name: 'slowOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + timeout: { each: 25 }, + }), + }); + const timedOut = await timeoutClient.callTool('run_stitch', { + name: 'slowOrders', + }); + check('a timeout → text', timedOut.text, 'timed out after 25ms'); + + const flakyWire = new Wire(leaky); + const flakyApi = seam({ baseUrl: BASE, adapter: flakyWire.adapter() }); + const breakerClient = await inProcess({ + brittle: flakyApi.stitch({ + name: 'brittle', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + circuit: { failures: 1, cooldown: '1m' }, + }), + }); + await breakerClient.callTool('run_stitch', { name: 'brittle' }); + const opened = await breakerClient.callTool('run_stitch', { + name: 'brittle', + }); + check('an open circuit → text', opened.text, 'circuit open'); + + const badInput = await client.callTool('run_stitch', { + name: 'getOrderTyped', + input: { params: { id: 'DROP TABLE' } }, + }); + check( + 'an input contract breach → text', + badInput.text, + 'invalid params: expected { id: numeric string }', + ); + note( + 'the schema’s own issue text is rendered', + 'a message that echoes the offending value would reach the model with it', + ); + + const strictApi = seam({ baseUrl: BASE, adapter: wire.adapter() }); + const outputClient = await inProcess({ + strictOrders: strictApi.stitch({ + name: 'strictOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + output: schema('{ nothing like this }', () => false), + }), + }); + const badOutput = await outputClient.callTool('run_stitch', { + name: 'strictOrders', + }); + note('an output contract breach → text', badOutput.text); + check( + 'the vendor’s response is not quoted back', + badOutput.raw.includes('acme') || badOutput.raw.includes('o_1'), + false, + ); + + const unknown = await client.callTool('run_stitch', { name: 'orders' }); + checkDiscloses( + 'an unknown name → the whole route table', + unknown.text, + 'getMetrics, getOrder, getOrderTyped, getProfile, getReport, listOrders, login, mintApiKey, refund, searchOrders', + ); + + heading('C4 (b) — is any of that a credential leak? No.'); + for (const ex of [http500, timedOut, opened, badInput, badOutput, unknown]) + checkClean(`${ex.label} ${ex.text.slice(0, 18)}`, ex.raw, HELD_SECRETS); + + heading('C4 (c) — THE LEAK: the message is an unfiltered pass-through'); + // The DEFAULT transport. No adapter, no network: port 99999 is outside the valid range, so + // undici rejects the URL string before opening a socket — and puts that string in the message. + const defaultTransport = await inProcess({ + metrics: stitch({ + baseUrl: 'http://api.vendor.test:99999', + path: '/v1/metrics', + auth: apiKey({ in: 'query', secret: env(ENV.apiKeyQuery) }), + }), + }); + const parseFail = await defaultTransport.callTool('run_stitch', { + name: 'metrics', + }); + check('isError', parseFail.isError, true); + note('the text the model received', parseFail.text); + checkDiscloses( + 'the model’s payload now holds the KEY', + parseFail.raw, + SECRETS.apiKeyQuery, + ); + note( + 'nothing here is user code', + 'built-in fetchAdapter, built-in apiKey({ in: "query" }), a mistyped port — and the credential is in the model’s context', + ); + + // The realistic trigger: a third-party adapter that names the URL on EVERY network error. + const nodeFetchClient = await inProcess({ + metrics: stitch({ + baseUrl: BASE, + path: '/v1/metrics', + auth: apiKey({ in: 'query', secret: env(ENV.apiKeyQuery) }), + adapter: nodeFetchShaped, + }), + }); + const dnsFail = await nodeFetchClient.callTool('run_stitch', { + name: 'metrics', + }); + note('a node-fetch-shaped DNS failure → text', dnsFail.text); + checkDiscloses( + 'the KEY again, on a routine DNS failure', + dnsFail.raw, + SECRETS.apiKeyQuery, + ); + + // The control: the SAME transport failure on a bearer stitch discloses the URL and no secret. + const bearerClient = await inProcess({ + orders: stitch({ + baseUrl: BASE, + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + adapter: nodeFetchShaped, + }), + }); + const bearerFail = await bearerClient.callTool('run_stitch', { + name: 'orders', + }); + checkClean('the same failure under bearer', bearerFail.raw, HELD_SECRETS); + checkDiscloses( + '…discloses the URL, but the URL holds nothing', + bearerFail.text, + `${BASE}/v1/orders`, + ); + note( + 'the leak is a PROPERTY OF apiKey({ in: "query" }), not of MCP', + 'the auth guide already warns a key in the URL leaks wherever URLs go — this measures one more place it goes: the model’s context', + ); + + finish( + 'C4', + "NO LEAK FROM StitchAPI'S OWN ERRORS, AND ONE REAL LEAK THROUGH THEM. `run_stitch` renders `(e as Error).message` and drops everything else a `StitchError` carries — `.status`, `.attempts`, `.body`, `.url` — so a vendor 500 whose body held an internal hostname, a stack frame and a `postgres://vendor:hunter2@…` DSN reached the model as the four characters `HTTP 500`. The whole built-in taxonomy is terse and request-free: `HTTP 500`, `timed out after 25ms`, `circuit open`, `invalid params: `, `contract violation (drift)`. The one disclosure that is by design is name enumeration — an unknown stitch answers with every registered name. BUT THE CHANNEL IS UNFILTERED, and that is a genuine hole in C1's promise: any message written by the TRANSPORT reaches the model verbatim. On the DEFAULT `fetchAdapter`, with an `apiKey({ in: 'query' })` stitch and a mistyped port, the model received `Failed to parse URL from http://api.vendor.test:99999/v1/metrics?api_key=ak_live_qry_8899aabbccddeeff` — the credential, in its context, from zero lines of user code. With a `node-fetch`-shaped adapter (`request to failed, reason: …`) the same thing happens on any DNS failure, which is a routine production event rather than a typo. The control pins the cause: the identical failure on a `bearer` stitch disclosed the URL and no secret. This is `apiKey({ in: 'query' })` leaking where URLs go — the auth guide already says so — and the MCP error channel is one more place URLs go", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c5-runaway.ts b/docs/scenarios/proofs/agent-holds-the-tool/c5-runaway.ts new file mode 100644 index 00000000..b92b13b0 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c5-runaway.ts @@ -0,0 +1,264 @@ +// C5 — runaway containment. Do `throttle` and `circuit` apply on the MCP path, and can a budget be +// expressed as anything other than a request count? +// +// The capture's framing: "the most common production incident is not a model giving the wrong +// answer; it is an agent that decides to retry, and retry, and retry." So there are three separate +// questions, and they have three different answers: +// +// 1. Do the existing limiters run when the caller is an agent? (Yes — the MCP path is the same +// engine, and nothing about `run_stitch` bypasses the resilience chain.) +// 2. What bounds a burst BY DEFAULT? (Nothing. 50 tool calls made 50 vendor requests.) +// 3. Is one tool call one vendor request? (No, and this is the part a per-call cap misses: +// `retry` and `paginate` multiply the model's single call into many, invisibly.) +// +// And the budget question is answered by enumeration: every bound the config surface offers is a +// COUNT or a DURATION. There is no cost, token, byte or currency budget anywhere. +// +// The stdio transport contributes a containment property of its own, measured in (e): `serveStdio` +// chains message handling (mcp.ts:346), so tool calls are processed strictly in order no matter how +// fast a client writes them. That is a real serialisation guarantee — and one that does not +// transfer to the HTTP transport the module's own header invites a host to build. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c5-runaway.ts +import { bearer, env } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import { inProcess, loadMcp } from './client'; +import { check, finish, heading, note } from './harness'; +import { BASE, ENV, type Route, Wire, installSecrets, route } from './vendor'; + +import { PassThrough } from 'node:stream'; + +/** A vendor that always fails — the circuit's input. */ +const failing: Route = () => ({ + status: 503, + headers: {}, + body: { error: 'unavailable' }, +}); + +/** Two pages of orders, so `paginate` has somewhere to go. */ +const paged: Route = (req) => { + const page = Number(new URL(req.url).searchParams.get('page') ?? '1'); + return { + status: 200, + headers: { 'content-type': 'application/json' }, + body: { + data: [{ id: `o_${String(page)}` }], + next: page < 12 ? page + 1 : null, + }, + }; +}; + +async function main(): Promise { + installSecrets(); + + heading('C5 (a) — with no policy configured, what bounds a burst?'); + const openWire = new Wire(route); + const openApi = seam({ baseUrl: BASE, adapter: openWire.adapter() }); + const openClient = await inProcess({ + orders: openApi.stitch({ + name: 'orders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + }), + }); + const t0 = Date.now(); + for (let i = 0; i < 50; i++) + await openClient.callTool('run_stitch', { name: 'orders' }); + const openElapsed = Date.now() - t0; + check('50 tool calls → vendor requests', openWire.count, 50); + note('elapsed (ms)', openElapsed); + note( + 'nothing is on by default', + 'no throttle, no circuit, no cap — the model’s call rate IS the vendor’s call rate', + ); + + heading('C5 (b) — does `throttle` apply on the MCP path?'); + const pacedWire = new Wire(route); + const pacedApi = seam({ baseUrl: BASE, adapter: pacedWire.adapter() }); + const pacedClient = await inProcess({ + orders: pacedApi.stitch({ + name: 'orders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + // A minimum spacing of 20ms between calls (ADR 0023: a rate is a spacing). + throttle: '50/s', + }), + }); + const t1 = Date.now(); + for (let i = 0; i < 10; i++) + await pacedClient.callTool('run_stitch', { name: 'orders' }); + const pacedElapsed = Date.now() - t1; + check('10 tool calls → vendor requests', pacedWire.count, 10); + note('elapsed (ms), 20ms spacing × 9 gaps ≈ 180', pacedElapsed); + check('the throttle paced the agent', pacedElapsed >= 170, true); + note( + 'the same limiter, reached through a different door', + 'run_stitch calls the stitch; the stitch is the engine; the engine is where throttle lives', + ); + + heading('C5 (c) — does `circuit` apply?'); + const brokenWire = new Wire(failing); + const brokenApi = seam({ baseUrl: BASE, adapter: brokenWire.adapter() }); + const brokenClient = await inProcess({ + orders: brokenApi.stitch({ + name: 'orders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + circuit: { failures: 3, cooldown: '10m' }, + }), + }); + const texts: string[] = []; + for (let i = 0; i < 20; i++) + texts.push( + (await brokenClient.callTool('run_stitch', { name: 'orders' })) + .text, + ); + check('20 tool calls → vendor requests', brokenWire.count, 3); + check( + 'and the rest fast-failed', + texts.filter((t) => t === 'circuit open').length, + 17, + ); + note( + 'the breaker is the one control that survives an agent loop', + '17 of 20 calls cost the vendor nothing — but they still cost the model a turn', + ); + + heading('C5 (d) — one tool call is not one vendor request'); + const retryWire = new Wire(failing); + const retryApi = seam({ baseUrl: BASE, adapter: retryWire.adapter() }); + const retryClient = await inProcess({ + orders: retryApi.stitch({ + name: 'orders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + retry: { attempts: 5, backoff: { base: 1 } }, + }), + }); + await retryClient.callTool('run_stitch', { name: 'orders' }); + check('1 tool call with retry ×5 → vendor requests', retryWire.count, 5); + + const pageWire = new Wire(paged); + const pageApi = seam({ baseUrl: BASE, adapter: pageWire.adapter() }); + const pageClient = await inProcess({ + allOrders: pageApi.stitch({ + name: 'allOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + paginate: { + // `next` reads the RAW body of the page just fetched (engine.ts:985). + next: (body) => { + const next = (body as { next: number | null }).next; + return next == null ? undefined : { query: { page: next } }; + }, + }, + // `pick` runs before the items are collected, so each page contributes its `data` array. + pick: 'data', + }), + }); + await pageClient.callTool('run_stitch', { name: 'allOrders' }); + check('1 tool call with paginate → vendor requests', pageWire.count, 12); + note( + 'the amplification is invisible to the model AND to a per-tool-call cap', + 'a host that budgets "20 tool calls" budgeted up to 100 vendor requests with retry, or 1,000 with the default paginate cap of 50', + ); + + heading( + 'C5 (e) — the stdio transport serialises, whatever the client does', + ); + const { serveStdio } = await loadMcp(); + const orderWire = new Wire(route); + const orderApi = seam({ baseUrl: BASE, adapter: orderWire.adapter() }); + const stdin = new PassThrough(); + const stdout = new PassThrough(); + stdout.setEncoding('utf8'); + const handle = serveStdio( + { + orders: orderApi.stitch({ + name: 'orders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + throttle: '50/s', + }), + }, + { stdin, stdout }, + ); + const ids: number[] = []; + stdout.on('data', (chunk: string) => { + for (const line of chunk.split('\n').filter(Boolean)) + ids.push((JSON.parse(line) as { id: number }).id); + }); + // Write eight tool calls in ONE chunk — the most concurrent a stdio client can be. + const burst = Array.from({ length: 8 }, (_, i) => + JSON.stringify({ + jsonrpc: '2.0', + id: i + 1, + method: 'tools/call', + params: { name: 'run_stitch', arguments: { name: 'orders' } }, + }), + ).join('\n'); + const t2 = Date.now(); + stdin.write(`${burst}\n`); + await new Promise((resolve) => { + const tick = setInterval(() => { + if (ids.length === 8) { + clearInterval(tick); + resolve(); + } + }, 5); + }); + const burstElapsed = Date.now() - t2; + check('8 messages written at once → all answered', ids.length, 8); + check('…in request order', ids.join(','), '1,2,3,4,5,6,7,8'); + check( + '…and still paced (7 gaps × 20ms ≈ 140ms)', + burstElapsed >= 130, + true, + ); + handle.close(); + stdin.end(); + note( + 'mcp.ts:346 chains dispatch', + '`chain = chain.then(() => dispatch(line))` — the stdio transport can never run two tool calls at once', + ); + note( + 'this does NOT transfer to an HTTP transport', + 'mcp.ts:1-9 invites one ("the same core can back a Streamable HTTP transport"), and a host that builds it inherits no serialisation', + ); + + heading('C5 (f) — can a budget be expressed as anything but a count?'); + const bounds = [ + [ + 'throttle.rate', + 'a minimum SPACING between calls — a count over a window', + ], + ['throttle.concurrency', 'a COUNT of simultaneous in-flight calls'], + ['retry.attempts', 'a COUNT of attempts'], + ['timeout.each / timeout.total', 'a DURATION'], + ['paginate.pages', 'a COUNT of pages (default 50)'], + ['circuit.failures / cooldown', 'a COUNT and a DURATION'], + [ + 'stream.buffer.chars', + 'a COUNT of decoded characters, streaming only', + ], + ]; + for (const [slot, kind] of bounds) note(String(slot), kind); + check( + 'a cost / token / byte / currency budget exists', + bounds.some(([, kind]) => + /cost|token|currenc|money|byte/i.test(String(kind)), + ), + false, + ); + note( + 'the runaway the capture describes is a SPEND, not a rate', + 'the $6,531 case was inside any per-minute cap that was set — what it exceeded was a budget nothing here can express', + ); + + finish( + 'C5', + "THE LIMITERS APPLY, NOTHING IS ON BY DEFAULT, AND NO BUDGET IS A SPEND. `throttle` and `circuit` both run on the MCP path because `run_stitch` calls the stitch and the stitch IS the engine: `throttle: '50/s'` paced ten tool calls to 180ms of 20ms gaps, and `circuit: { failures: 3 }` turned twenty tool calls into three vendor requests plus seventeen `circuit open` fast-fails. With nothing configured, fifty tool calls made fifty vendor requests in a few milliseconds — the model's call rate is the vendor's call rate. THE PART A PER-CALL CAP MISSES: one tool call is not one request. `retry: { attempts: 5 }` made five, and `paginate` made twelve (its default ceiling is 50), with no signal of either in the tool result — so a host that budgets \"20 tool calls\" has budgeted up to 1,000 vendor requests. THE STDIO TRANSPORT ADDS ONE REAL GUARANTEE: eight tool calls written in a single chunk were answered in request order and still paced (mcp.ts:346 chains dispatch), so no stdio client can run two calls at once — a property the HTTP transport the module invites a host to build does not inherit. AND THE BUDGET ANSWER IS NO: every bound on the surface is a COUNT (`throttle.rate` spacing, `throttle.concurrency`, `retry.attempts`, `paginate.pages`, `circuit.failures`, `stream.buffer.chars`) or a DURATION (`timeout`, `circuit.cooldown`). Nothing expresses cost, tokens, bytes off the socket, or money — which is the axis the runaway incident in the capture actually ran along", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c6-confirmation.ts b/docs/scenarios/proofs/agent-holds-the-tool/c6-confirmation.ts new file mode 100644 index 00000000..02a50a60 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c6-confirmation.ts @@ -0,0 +1,240 @@ +// C6 — is there a confirmation seam for an irreversible call? +// +// The capture calls human confirmation "the one control that survives prompt injection", so this +// asks two separate questions: +// +// 1. Can the SERVER ask? MCP's 2025-06-18 revision — the version this server reports — has +// `elicitation`, a server-initiated request for user input. Measured below: the server neither +// advertises it nor could implement it, because `McpServer` is `{ handle }` (one response per +// request) and `serveStdio` never hands user code the outbound stream. There is no channel +// from the server to the human. +// 2. Can the CLIENT ask? Every MCP host has a "confirm before a destructive tool" policy, driven +// by the tool's `annotations` (`readOnlyHint`, `destructiveHint`). Measured below: the tool +// descriptors carry no annotations at all — and code-mode means one tool name covers a GET of +// an order and a POST of a refund, so even a client that annotated perfectly could not tell +// them apart without parsing arguments it has no schema for. +// +// Then it measures what a determined operator CAN do in user code, and where each seam sits +// relative to the request: `hooks.onRequest` (after auth, retried), a `Surface.execute` (replaces +// the transport), and the `adapter` (last). All three can refuse; none can ask. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c6-confirmation.ts +import { bearer, env } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { inProcess, loadMcp } from './client'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { buildRegistry } from './stitches'; +import { BASE, ENV, Wire, installSecrets, route } from './vendor'; + +interface ToolDescriptor { + name: string; + description: string; + inputSchema: unknown; + annotations?: unknown; +} + +async function main(): Promise { + installSecrets(); + const wire = new Wire(route); + const registry = buildRegistry(wire); + const client = await inProcess(registry); + + heading('C6 (a) — can the SERVER ask the human? (elicitation / sampling)'); + const init = await client.send('initialize', { + protocolVersion: '2025-06-18', + capabilities: { elicitation: {}, sampling: {} }, + }); + const result = init.message.result as { + protocolVersion: string; + capabilities: Record; + }; + check('the server reports', result.protocolVersion, '2025-06-18'); + checkSeq('server capabilities', Object.keys(result.capabilities), [ + 'tools', + ]); + check( + 'elicitation is advertised', + Object.keys(result.capabilities).includes('elicitation'), + false, + ); + note( + 'the client offered elicitation and sampling; the server ignored both', + 'pickProtocol reads only `protocolVersion` (mcp.ts:132-136) — the client’s capabilities object is never read', + ); + + const { createMcpServer, serveStdio } = await loadMcp(); + const server = createMcpServer(registry); + checkSeq('the McpServer interface', Object.keys(server), ['handle']); + const stdinLess = serveStdio(registry, { + stdin: new (await import('node:stream')).PassThrough(), + stdout: new (await import('node:stream')).PassThrough(), + }); + checkSeq('what serveStdio hands back', Object.keys(stdinLess).sort(), [ + 'close', + 'server', + ]); + stdinLess.close(); + note( + 'there is no outbound channel', + '`handle(message) => response | null` is request/response only, and `serveStdio` keeps `stdout` private — user code cannot send a server-initiated request', + ); + + heading('C6 (b) — can the CLIENT ask? What the tool descriptors say'); + const tools = ( + (await client.send('tools/list')).message.result as { + tools: ToolDescriptor[]; + } + ).tools; + for (const tool of tools) + check(`${tool.name}.annotations`, tool.annotations === undefined, true); + note( + 'no readOnlyHint, no destructiveHint, no idempotentHint, no openWorldHint', + 'the four MCP tool annotations a host uses to decide whether to prompt', + ); + wire.reset(); + const read = await client.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + const write = await client.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 25_000 } }, + }); + check( + 'a read and a refund use the SAME tool name', + read.label, + write.label, + ); + checkSeq( + 'and the wire saw both', + wire.requests.map((r) => `${r.method} ${new URL(r.url).pathname}`), + ['GET /v1/orders/77', 'POST /v1/refunds'], + ); + note( + 'this is the cost of code-mode', + 'one tool for every endpoint keeps the model’s context small AND makes the host’s destructive-tool prompt undecidable — the method is inside an argument the host has no schema for', + ); + check( + 'does `list_stitches` at least surface the method?', + JSON.parse((await client.callTool('list_stitches')).text).length > 0 && + ( + JSON.parse((await client.callTool('list_stitches')).text) as { + name: string; + method: string; + }[] + ).some((s) => s.name === 'refund' && s.method === 'POST'), + true, + ); + note( + 'it does — but a host would have to CALL a tool to learn it', + 'and the annotation it needs is on the tool descriptor, which is fetched once, before any call', + ); + + heading('C6 (c) — what a REFUSAL seam can do, and where each one sits'); + // `hooks.onRequest` — engine.ts:652, after auth, inside the attempt loop. + const hookWire = new Wire(route); + const hookApi = seam({ baseUrl: BASE, adapter: hookWire.adapter() }); + const hookSeen: string[] = []; + const hookClient = await inProcess({ + refund: hookApi.stitch({ + name: 'refund', + method: 'POST', + path: '/v1/refunds', + auth: bearer(env(ENV.bearer)), + retry: { attempts: 3, backoff: { base: 1 } }, + hooks: { + onRequest: (ctx) => { + // `req` is optional on `HookContext` (it is shared with onResponse/onError); + // on the onRequest arm the engine always supplies it (engine.ts:652). + const req = ctx.req as AdapterRequest; + hookSeen.push( + `attempt ${String(ctx.attempt)} ${req.method} ${new URL(req.url).pathname}`, + ); + throw new Error('refund requires human approval'); + }, + }, + }), + }); + const blocked = await hookClient.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 25_000 } }, + }); + check('the write was refused', blocked.isError, true); + check( + 'the model was told why', + blocked.text, + 'refund requires human approval', + ); + check('no request reached the vendor', hookWire.count, 0); + // The hook lives inside the attempt loop (engine.ts:652), so a `retry: { attempts: 3 }` could + // have asked it three times. It did not: a refusal is not a retryable failure. + checkSeq('and it was asked exactly once, despite retry ×3', hookSeen, [ + 'attempt 1 POST /v1/refunds', + ]); + note( + 'onRequest sees the FINAL request — url, method, and the credential header', + 'the only seam that sees all three; it sits inside the attempt loop but a throw from it does not burn a retry', + ); + + // The `adapter` — the last seam before the transport, and outside the retry loop's hook. + const gateWire = new Wire(route); + const approvals: string[] = []; + const gated = + (allow: (req: AdapterRequest) => boolean) => + async (req: AdapterRequest): Promise => { + approvals.push(`${req.method} ${new URL(req.url).pathname}`); + if (!allow(req)) + throw new Error( + 'blocked: a write needs approval that this server cannot ask for', + ); + return gateWire.adapter()(req); + }; + const gateApi = seam({ + baseUrl: BASE, + adapter: gated((req) => req.method === 'GET'), + }); + const gateClient = await inProcess({ + getOrder: gateApi.stitch({ + name: 'getOrder', + path: '/v1/orders/{id}', + auth: bearer(env(ENV.bearer)), + }), + refund: gateApi.stitch({ + name: 'refund', + method: 'POST', + path: '/v1/refunds', + auth: bearer(env(ENV.bearer)), + }), + }); + const allowed = await gateClient.callTool('run_stitch', { + name: 'getOrder', + input: { params: { id: '77' } }, + }); + const denied = await gateClient.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 1 } }, + }); + check('the read passed the gate', allowed.isError, false); + check('the write was denied', denied.isError, true); + check( + 'the model was told why', + denied.text, + 'blocked: a write needs approval that this server cannot ask for', + ); + checkSeq('the gate saw both', approvals, [ + 'GET /v1/orders/77', + 'POST /v1/refunds', + ]); + check('and only the read reached the vendor', gateWire.count, 1); + + finish( + 'C6', + "NO CONFIRMATION SEAM, IN EITHER DIRECTION — AND CODE-MODE TAKES THE CLIENT'S ONE AWAY TOO. The server cannot ask: it reports protocol `2025-06-18`, whose `elicitation` is the standard's server-initiated request for user input, but it advertises `capabilities: { tools }` and nothing else, ignores the elicitation and sampling capabilities the client offers, and structurally could not use them — `McpServer` is `{ handle }`, a pure request/response mapping, and `serveStdio` returns `{ server, close }` while keeping `stdout` private, so no user code can originate a message. The client cannot ask either: all three tool descriptors carry NO `annotations`, so `readOnlyHint`/`destructiveHint` — the fields a host reads to decide whether to prompt — are absent; and because code-mode puts every endpoint behind ONE tool name, the read of an order and a 25,000 refund arrive at the host as the same `run_stitch` call, with the method buried in an argument the host has no schema for. `list_stitches` does report `POST /v1/refunds`, but a host would have to call a tool to learn it, and the annotation it needs is fixed at `tools/list` time. WHAT USER CODE CAN DO IS REFUSE, NOT ASK, and there are two useful seats: `hooks.onRequest` sees the final URL, method and credential header and can throw — measured, the vendor got zero requests, the model got the reason, and despite `retry: { attempts: 3 }` the gate was asked exactly once, because a refusal is not a retryable failure; an `adapter` wrapper sits one layer further out and gated a POST while letting a GET through, at one function and no per-stitch config. Both are policy, not approval: nothing in the process can reach a human", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c7-schema.ts b/docs/scenarios/proofs/agent-holds-the-tool/c7-schema.ts new file mode 100644 index 00000000..d331bb62 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c7-schema.ts @@ -0,0 +1,220 @@ +// C7 — schema quality. Is `input` typed enough for a model to use correctly, and does a declared +// schema constrain what the model may send? +// +// Three findings, in increasing order of consequence: +// +// 1. The TOOL schema is four untyped bags. `run_stitch.inputSchema` says `input` has `params`, +// `query`, `headers` (objects) and `body` (anything) — with no per-stitch shape, no +// `required`, and no `additionalProperties: false`. A model choosing arguments has nothing +// here to be correct against. +// 2. `describe_stitch` reports PRESENCE, not shape. A stitch whose `params` schema demands +// `{ id: }` is described to the model as `"params": true`. The schema is on the +// config (`__config.input.params`) and is simply not projected, so the model must guess and +// find out by failing. +// 3. A declared schema is CHECK-ONLY, and it covers one slot. `validateInput` (engine.ts:384-409) +// throws when a slot fails and otherwise DISCARDS the parsed value — so a schema that strips +// unknown keys does not strip them from the request. Measured below: a `query` schema that +// returns `{ limit: 10 }` still puts the model's `tenant=globex` on the wire. +// +// The good news is real and worth stating first: validation runs BEFORE any request is built, so a +// slot that fails its contract costs the vendor nothing. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c7-schema.ts +import { bearer, env } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { Validator } from '../../../../packages/core/src/validator'; +import { inProcess } from './client'; +import { check, checkSeq, checkWire, finish, heading, note } from './harness'; +import { buildRegistry } from './stitches'; +import { BASE, ENV, Wire, installSecrets, route } from './vendor'; + +interface RunStitchSchema { + type: string; + properties: { + name: { type: string; description: string }; + input: { + type: string; + description: string; + properties: Record; + required?: string[]; + additionalProperties?: boolean; + }; + }; + required: string[]; +} + +/** + * A validator shaped like every mainstream schema library's default object mode: it ACCEPTS the + * value and returns a copy with unknown keys removed. Zod's `.parse`, Valibot's `object`, ArkType's + * default — all of them strip. The question is whether the engine uses what came back. + */ +const strippingQuery: Validator = { + validate: (value) => + Promise.resolve({ + ok: true as const, + value: { limit: (value as { limit?: unknown } | undefined)?.limit }, + }), +}; + +async function main(): Promise { + installSecrets(); + const wire = new Wire(route); + const registry = buildRegistry(wire); + const client = await inProcess(registry); + + heading('C7 (a) — what the TOOL schema tells a model'); + const tools = ( + (await client.send('tools/list')).message.result as { + tools: { name: string; inputSchema: unknown }[]; + } + ).tools; + const runSchema = tools.find((t) => t.name === 'run_stitch') + ?.inputSchema as RunStitchSchema; + checkSeq('run_stitch top-level required', runSchema.required, ['name']); + checkSeq( + 'input slots offered', + Object.keys(runSchema.properties.input.properties), + ['params', 'query', 'body', 'headers'], + ); + checkSeq( + 'their declared types', + Object.values(runSchema.properties.input.properties).map( + (v) => (v as { type?: string }).type ?? '(anything)', + ), + ['object', 'object', '(anything)', 'object'], + ); + check( + 'input.additionalProperties', + runSchema.properties.input.additionalProperties, + undefined, + ); + check('input.required', runSchema.properties.input.required, undefined); + const listSchema = tools.find((t) => t.name === 'list_stitches') + ?.inputSchema as { additionalProperties?: boolean }; + check( + 'by contrast, list_stitches is closed', + listSchema.additionalProperties, + false, + ); + note( + 'the tool schema is the same four bags for every stitch', + 'code-mode’s whole premise — one tool, constant context — is why it cannot carry a per-stitch shape', + ); + + heading( + 'C7 (b) — what `describe_stitch` tells a model about a TYPED stitch', + ); + const described = JSON.parse( + (await client.callTool('describe_stitch', { name: 'getOrderTyped' })) + .text, + ) as { input: Record }; + checkSeq('input read-out', Object.entries(described.input).flat(), [ + 'params', + true, + 'query', + false, + 'body', + false, + 'headers', + false, + ]); + note( + 'the stitch’s params contract is `{ id: }`', + 'the model is told `true` — not the key name, not the type, not the pattern', + ); + const guess = await client.callTool('run_stitch', { + name: 'getOrderTyped', + input: { params: { orderId: 77 } }, + }); + check('a plausible guess fails', guess.isError, true); + note('and the failure is the only teacher', guess.text); + + heading('C7 (c) — the good half: validation runs BEFORE the request'); + wire.reset(); + await client.callTool('run_stitch', { + name: 'getOrderTyped', + input: { params: { id: 'not-a-number' } }, + }); + check('a failed contract costs the vendor nothing', wire.count, 0); + note( + 'engine.ts:1687 — `await validateInput(cfg, input)` is the first thing `execute` does', + 'ahead of buildRequest, auth, throttle and the adapter', + ); + + heading('C7 (d) — a schema constrains ONE slot, not the input object'); + wire.reset(); + const sideDoor = await client.callTool('run_stitch', { + name: 'getOrderTyped', + input: { + params: { id: '77' }, + // No `query` schema is declared on this stitch, so nothing checks this. + query: { tenant: 'globex', include: 'internal_notes' }, + }, + }); + check('the call succeeded', sideDoor.isError, false); + checkWire( + 'url', + wire.last.url, + `${BASE}/v1/orders/77?tenant=globex&include=internal_notes`, + ); + note( + 'declaring `input.params` says nothing about `input.query`', + 'the slots are independent, and an undeclared slot is an open passthrough', + ); + + heading( + 'C7 (e) — a declared schema is CHECK-ONLY: the parsed value is discarded', + ); + const stripWire = new Wire(route); + const stripApi = seam({ baseUrl: BASE, adapter: stripWire.adapter() }); + const stripClient = await inProcess({ + listOrders: stripApi.stitch({ + name: 'listOrders', + path: '/v1/orders?tenant=acme', + auth: bearer(env(ENV.bearer)), + // A schema that ACCEPTS and returns `{ limit }` only — every mainstream object schema + // strips unknown keys like this by default. + input: { query: strippingQuery }, + }), + }); + const stripped = await stripClient.callTool('run_stitch', { + name: 'listOrders', + input: { query: { limit: 10, tenant: 'globex' } }, + }); + check('the schema accepted the input', stripped.isError, false); + check( + 'the validator returned only `limit`', + JSON.stringify( + (await strippingQuery.validate({ limit: 10, tenant: 'globex' })) + .ok === true + ? { limit: 10 } + : null, + ), + '{"limit":10}', + ); + checkWire( + 'but the wire carried the STRIPPED key too', + stripWire.last.url, + `${BASE}/v1/orders?tenant=globex&limit=10`, + ); + check( + 'and the operator’s pinned tenant is gone', + new URL(stripWire.last.url).searchParams.get('tenant'), + 'globex', + ); + note( + 'engine.ts:400-408 — `const r = await v.validate(...); if (!r.ok) throw`', + 'the parsed value is never read; `output` uses its parsed value (engine.ts:428), `input` does not', + ); + note( + 'so a stripping schema is a validity check, not a filter', + 'an operator who writes `z.object({ limit: z.number() })` on `query` and expects unknown keys to be dropped is wrong — they reach the vendor', + ); + + finish( + 'C7', + 'THE MODEL IS TOLD A SLOT EXISTS AND NEVER WHAT GOES IN IT, AND A DECLARED SCHEMA IS A CHECK RATHER THAN A FILTER. The tool schema is four untyped bags — `params`/`query`/`headers` typed `object`, `body` typed as anything, no `required`, no `additionalProperties: false` (while `list_stitches`, which takes nothing, IS closed) — and it is identical for every stitch, because one tool for every endpoint is what code-mode buys its constant context with. `describe_stitch` does not make up the difference: a stitch whose `params` contract is `{ id: }` is described as `"params": true`, so a model that guesses `{ orderId: 77 }` learns the shape only by failing. ONE HALF IS GENUINELY GOOD: `validateInput` is the first thing `execute` does (engine.ts:1687), so a slot that breaks its contract costs the vendor zero requests. TWO HALVES ARE NOT. A schema constrains ONE SLOT — `getOrderTyped` declares `params` and the model still appended `?tenant=globex&include=internal_notes` through the undeclared `query`. And the check is check-only: `validateInput` throws on failure and DISCARDS the parsed value (engine.ts:400-408), so a schema that strips unknown keys — which is the default behaviour of Zod, Valibot and ArkType alike — does not strip them from the request. Measured: a `query` validator that returned `{ limit: 10 }` still put `?tenant=globex&limit=10` on the wire, overwriting the operator\'s pinned `tenant=acme`. `output` uses its parsed value; `input` never does', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/c8-assembled.ts b/docs/scenarios/proofs/agent-holds-the-tool/c8-assembled.ts new file mode 100644 index 00000000..1a3294a2 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/c8-assembled.ts @@ -0,0 +1,240 @@ +// C8 — assemble the safest available exposure, then run every attack C2–C7 landed at it. +// +// The measurement is a before/after over the SAME registry, the SAME vendor and the SAME JSON-RPC +// messages: each attack is replayed against the naive exposure (the registry as written) and +// against `safe-exposure.ts`, and the wire tap says which one reached the vendor. +// +// The line count is the second half of the answer, and it is small — which is the point. The +// credential boundary, the resilience chain, the transport, the discovery tools and the JSON-RPC +// layer are all the library's; what the operator has to write is the ARGUMENT policy the library +// deliberately does not have an opinion about. +// +// pnpm exec tsx docs/scenarios/proofs/agent-holds-the-tool/c8-assembled.ts +import { bearer, env } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { inProcess } from './client'; +import { check, checkSeq, checkWire, finish, heading, note } from './harness'; +import { expose, only, readsOnly } from './safe-exposure'; +import { BASE, ENV, SECRETS, Wire, installSecrets, route } from './vendor'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * Executable lines between the USER CODE markers — imports, blanks and comments removed, so the + * number is the code someone actually maintains. Same counter as `deprecation-headers/c8`. + */ +function executableLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8'); + const from = src.indexOf('// >>> BEGIN USER CODE'); + const to = src.indexOf('// <<< END USER CODE'); + return src + .slice(from, to) + .replace(/^import[\s\S]*?;$/gm, '') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +/** The registry as an operator first writes it: everything exported, nothing filtered. */ +function naive(wire: Wire): Record { + const api = seam({ baseUrl: BASE, adapter: wire.adapter() }); + return { + listOrders: api.stitch({ + name: 'listOrders', + path: '/v1/orders?tenant=acme', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }) as Stitch, + refund: api.stitch({ + name: 'refund', + method: 'POST', + path: '/v1/refunds', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }) as Stitch, + }; +} + +/** The same two endpoints, exposed safely. Three call sites, and that is the whole delta. */ +function safe(wire: Wire): Record { + const api = seam({ + baseUrl: BASE, + adapter: readsOnly(wire.adapter()), + }); + return expose({ + listOrders: only( + api.stitch({ + name: 'listOrders', + path: '/v1/orders?tenant=acme', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }) as Stitch, + { query: ['limit'] }, + ), + refund: api.stitch({ + name: 'refund', + method: 'POST', + path: '/v1/refunds', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }) as Stitch, + }) as Record; +} + +async function main(): Promise { + installSecrets(); + + heading('C8 (a) — the tenant pin (C2 d)'); + const naiveWire = new Wire(route); + const naiveClient = await inProcess(naive(naiveWire)); + const safeWire = new Wire(route); + const safeClient = await inProcess(safe(safeWire)); + + await naiveClient.callTool('run_stitch', { + name: 'listOrders', + input: { query: { tenant: 'globex' } }, + }); + checkWire( + 'naive url', + naiveWire.last.url, + `${BASE}/v1/orders?tenant=globex`, + ); + const safeRead = await safeClient.callTool('run_stitch', { + name: 'listOrders', + input: { query: { tenant: 'globex', limit: 5 } }, + }); + checkWire( + 'safe url', + safeWire.last.url, + `${BASE}/v1/orders?tenant=acme&limit=5`, + ); + check('and the read still works', safeRead.isError, false); + check( + 'the credential still reached the vendor', + safeWire.last.headers['authorization'], + `Bearer ${SECRETS.bearer}`, + ); + + heading('C8 (b) — the undeclared-slot passthrough (C7 d)'); + naiveWire.reset(); + safeWire.reset(); + await naiveClient.callTool('run_stitch', { + name: 'listOrders', + input: { query: { include: 'internal_notes' }, params: { x: 1 } }, + }); + checkWire( + 'naive url', + naiveWire.last.url, + `${BASE}/v1/orders?tenant=acme&include=internal_notes`, + ); + await safeClient.callTool('run_stitch', { + name: 'listOrders', + input: { query: { include: 'internal_notes' }, params: { x: 1 } }, + }); + checkWire('safe url', safeWire.last.url, `${BASE}/v1/orders?tenant=acme`); + + heading('C8 (c) — the header opt-in (C2 b)'); + safeWire.reset(); + await safeClient.callTool('run_stitch', { + name: 'listOrders', + input: { headers: { 'x-actor': 'admin', cookie: 'SESSION=forged' } }, + }); + checkSeq('safe headers on the wire', Object.keys(safeWire.last.headers), [ + 'authorization', + ]); + note( + 'this one was already closed by the library', + 'sanitizeAgentInput strips `headers` unless the stitch declares a headers schema — the safe exposure simply never declares one', + ); + + heading('C8 (d) — the unconfirmed write (C6)'); + naiveWire.reset(); + safeWire.reset(); + const naiveWrite = await naiveClient.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 999_999 } }, + }); + check('naive: the refund went through', naiveWrite.isError, false); + check('naive: the vendor saw it', naiveWire.count, 1); + const safeWrite = await safeClient.callTool('run_stitch', { + name: 'refund', + input: { body: { amount: 999_999 } }, + }); + check('safe: refused', safeWrite.isError, true); + check( + 'safe: with a reason the model can act on', + safeWrite.text, + 'POST is not available to an agent on this server', + ); + check('safe: the vendor saw nothing', safeWire.count, 0); + note( + 'refused, not confirmed', + 'there is no channel to a human from inside this process (C6 a) — a write an agent may legitimately need has to be a different server, or a different transport', + ); + + heading('C8 (e) — the configured-name bypass (C3 e)'); + const shadowApi = seam({ + baseUrl: BASE, + adapter: new Wire(route).adapter(), + }); + let rejected = ''; + try { + expose({ + readOnlyOrders: shadowApi.stitch({ + name: 'listOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + }) as Stitch, + }); + } catch (e) { + rejected = (e as Error).message; + } + check( + 'a renamed key is rejected at construction', + rejected, + 'expose: "readOnlyOrders" is also reachable as "listOrders" — give the stitch the same name as its key', + ); + + heading('C8 (f) — what C1 still gives you for free'); + const transcript = safeClient.transcript.map((e) => e.raw).join('\n'); + check( + 'no credential in the safe transcript', + transcript.includes(SECRETS.bearer), + false, + ); + note('bytes returned to the model', transcript.length); + + heading('C8 (g) — the line count'); + const userLines = executableLines('safe-exposure.ts'); + note('safe-exposure.ts, executable lines', userLines); + note('seams used', 3); + note( + ' 1. the registry object handed to createMcpServer', + 'the allow-list — `expose`', + ); + note( + ' 2. a Proxy apply-trap over each Stitch', + 'the input filter — `only`', + ); + note(' 3. the seam `adapter`', 'the method gate — `readsOnly`'); + note('config keys that know about any of this', 0); + check('under 50 executable lines', userLines <= 50, true); + + finish( + 'C8', + `ACHIEVABLE, AND THE USER CODE IS ${String(userLines)} EXECUTABLE LINES ACROSS 3 SEAMS. The safe exposure closes every gap C2–C7 opened that is closable in-process, and it needed no fork and no config key: \`expose\` (the registry object handed to \`createMcpServer\`) is the allow-list, and it rejects a key whose stitch carries a different configured \`name\` — the C3 (e) bypass — at construction; \`only\` is a \`Proxy\` apply-trap that REBUILDS the input from an explicit key list before the engine sees it, which is what the C7 (e) finding forces (a validator's parsed value is discarded, so a stripping schema filters nothing); \`readsOnly\` wraps the \`Adapter\`, the last seam before the transport, and refuses a non-GET. Replayed side by side over the same vendor: the naive exposure sent \`?tenant=globex\`, \`?include=internal_notes\` and a 999,999 refund to the wire; the safe one sent \`?tenant=acme&limit=5\` and nothing else, refused the POST with a reason the model can read, and still authenticated every read with the real \`Bearer sk_live_…\`. WHAT IT CANNOT CLOSE: C6 — nothing in this process can ask a human, so an irreversible call can only be refused, never confirmed; and C4 (c) — the error channel is an unfiltered \`Error.message\`, so the only fix for a URL-borne credential is \`apiKey({ in: 'header' })\` rather than \`{ in: 'query' }\``, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/agent-holds-the-tool/client.ts b/docs/scenarios/proofs/agent-holds-the-tool/client.ts new file mode 100644 index 00000000..d1f9aa8e --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/client.ts @@ -0,0 +1,196 @@ +// An MCP client, driving the server the way a real one does — over JSON-RPC. +// +// Every script here talks to the server through `initialize` / `tools/list` / `tools/call` +// messages and reads the RESPONSE PAYLOAD BACK AS A STRING, because that string is what a real +// client puts in the model's context. Calling `callRunStitch` directly would test a function; this +// tests the surface, and it is the serialised payload that C1 scans for credential values. +// +// Two transports, same interface: +// - `inProcess` — `createMcpServer(registry).handle(msg)`, serialised with `JSON.stringify` +// exactly as `serveStdio` does before writing it (mcp.ts:337). +// - `overStdio` — a real `serveStdio` wired to a pair of `PassThrough` streams, so the bytes +// are read off a stream after a newline-delimited round trip. Slower, and the +// point is that it is the shipped transport rather than a re-implementation. +// +// THE BOOT SHIM. `src/mcp.ts` reads `__PKG_VERSION__`, an esbuild `define` supplied by tsup and by +// vitest (src/version.d.ts) — under a bare `tsx` run there is no define, and the module would throw +// `ReferenceError` at import time. `loadMcp` sets the global first and then imports DYNAMICALLY, so +// the assignment is guaranteed to run before the module body regardless of how the import sorter +// orders anything. This is the only reason these scripts do not `import { createMcpServer } from +// '../../../../packages/core/src/mcp'` at the top like every other import in this directory. +// `import type` is erased outright (`verbatimModuleSyntax` + `isolatedModules`), so naming these +// types costs no runtime import of `src/mcp.ts` and the shim above stays the only loader. +import type { JsonRpcMessage } from '../../../../packages/core/src/mcp'; +import type { StitchRegistry } from '../../../../packages/core/src/registry'; + +import { PassThrough } from 'node:stream'; + +type McpModule = typeof import('../../../../packages/core/src/mcp'); + +/** Load `src/mcp.ts` with the build-time version define stood in for. See the note above. */ +export async function loadMcp(): Promise { + (globalThis as unknown as Record)['__PKG_VERSION__'] ??= + '0.0.0-proof'; + return import('../../../../packages/core/src/mcp'); +} + +/** One request/response round trip, kept whole so a script can assert on any layer of it. */ +export interface Exchange { + /** The JSON-RPC method that was sent (plus the tool name, for `tools/call`). */ + label: string; + /** The EXACT bytes a client would read off the transport. This is what C1 scans. */ + raw: string; + /** The parsed response. */ + message: JsonRpcMessage; + /** A tool result's concatenated `content[].text`, or `''` for a non-tool response. */ + text: string; + /** A tool result's `isError` flag. */ + isError: boolean; +} + +export interface McpClient { + /** Send a raw JSON-RPC request and return the round trip. */ + send(method: string, params?: unknown): Promise; + /** `tools/call` shorthand. */ + callTool(name: string, args?: unknown): Promise; + /** Every exchange so far, in order — the transcript C1 scans in bulk. */ + readonly transcript: Exchange[]; + close(): void; +} + +interface ToolResultShape { + content?: { type?: string; text?: string }[]; + isError?: boolean; +} + +/** Pull the model-visible text and error flag out of whatever the server returned. */ +function readToolResult(message: JsonRpcMessage): { + text: string; + isError: boolean; +} { + const result = message.result as ToolResultShape | undefined; + const content = result?.content; + if (!Array.isArray(content)) return { text: '', isError: false }; + return { + text: content.map((c) => c.text ?? '').join('\n'), + isError: result?.isError === true, + }; +} + +function exchangeOf(label: string, raw: string): Exchange { + const message = JSON.parse(raw) as JsonRpcMessage; + return { label, raw, message, ...readToolResult(message) }; +} + +/** + * Drive `createMcpServer` in-process. The response is serialised with the same `JSON.stringify` + * call `serveStdio` makes before writing it to the socket, so `raw` is byte-identical to what the + * stdio transport emits (asserted in `c1-credential-reach.ts`). + */ +export async function inProcess( + registry: StitchRegistry, + server?: string, +): Promise { + const { createMcpServer } = await loadMcp(); + const mcp = createMcpServer(registry, server); + const transcript: Exchange[] = []; + let id = 0; + + const send = async ( + method: string, + params?: unknown, + ): Promise => { + const request = { + jsonrpc: '2.0' as const, + id: ++id, + method, + ...(params === undefined ? {} : { params }), + }; + const response = await mcp.handle(request); + const label = + method === 'tools/call' + ? `tools/call ${String((params as { name?: unknown } | undefined)?.name)}` + : method; + const ex = exchangeOf(label, JSON.stringify(response)); + transcript.push(ex); + return ex; + }; + + return { + send, + callTool: (name, args) => + send('tools/call', { name, arguments: args ?? {} }), + transcript, + close: () => undefined, + }; +} + +/** + * Drive the SHIPPED stdio transport: a real `serveStdio` reading newline-delimited JSON-RPC off a + * stream and writing responses to another. Used to confirm the in-process transcript is the same + * bytes the transport writes — the C1 scan is only worth anything if it scans what ships. + */ +export async function overStdio( + registry: StitchRegistry, + server?: string, +): Promise { + const { serveStdio } = await loadMcp(); + const stdin = new PassThrough(); + const stdout = new PassThrough(); + stdout.setEncoding('utf8'); + const handle = serveStdio( + registry, + server === undefined ? { stdin, stdout } : { stdin, stdout, server }, + ); + + // One reader for the whole session: buffer whatever arrives and hand out complete lines in + // order, so a caller awaiting response N cannot be handed response N+1. + let buffer = ''; + const waiting: ((line: string) => void)[] = []; + stdout.on('data', (chunk: string) => { + buffer += chunk; + let nl: number; + while ((nl = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, nl); + buffer = buffer.slice(nl + 1); + const next = waiting.shift(); + if (next) next(line); + } + }); + const nextLine = (): Promise => + new Promise((resolve) => waiting.push(resolve)); + + const transcript: Exchange[] = []; + let id = 0; + const send = async ( + method: string, + params?: unknown, + ): Promise => { + const request = { + jsonrpc: '2.0' as const, + id: ++id, + method, + ...(params === undefined ? {} : { params }), + }; + const line = nextLine(); + stdin.write(`${JSON.stringify(request)}\n`); + const label = + method === 'tools/call' + ? `tools/call ${String((params as { name?: unknown } | undefined)?.name)}` + : method; + const ex = exchangeOf(label, await line); + transcript.push(ex); + return ex; + }; + + return { + send, + callTool: (name, args) => + send('tools/call', { name, arguments: args ?? {} }), + transcript, + close: () => { + handle.close(); + stdin.end(); + }, + }; +} diff --git a/docs/scenarios/proofs/agent-holds-the-tool/harness.ts b/docs/scenarios/proofs/agent-holds-the-tool/harness.ts new file mode 100644 index 00000000..08d72f78 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/harness.ts @@ -0,0 +1,174 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is of two kinds, and each gets its own assertion: +// +// 1. WHAT THE MODEL RECEIVED. Every JSON-RPC response payload is a string, and the question is +// whether a credential VALUE appears anywhere in it. `checkClean` scans one payload against +// the whole secret set and prints the byte count it scanned, so "the model got 3,410 bytes +// and none of them were the token" is a measurement rather than an assurance. Its inverse, +// `checkDiscloses`, asserts a substring IS present — `describe_stitch` handing the model the +// internal URL is a real disclosure and has to be recorded as loudly as a clean scan. +// 2. WHAT REACHED THE WIRE. `checkWire` prints a request field the fake adapter recorded, so +// "the `authorization` header the vendor saw was `Bearer sk_live_…`, not the model's +// `Bearer attacker`" is the exact bytes and not a paraphrase. +// +// `check` / `checkSeq` / `note` / `heading` / `finish` follow `deprecation-headers/harness.ts` +// unchanged, so a reader who has seen one proof directory has seen this one. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides several rows. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v}n`; + if (typeof v === 'number' && Number.isNaN(v)) return 'NaN'; + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the wire spine + * (`["Bearer sk_live_…","Bearer sk_live_…"]`) and the tool list (`["run_stitch","list_stitches", + * "describe_stitch"]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * THE assertion of C1: does this JSON-RPC payload contain a credential VALUE? + * + * `payload` is the exact string a real MCP client would read off the transport (the response + * serialised with `JSON.stringify`, which is what `serveStdio` writes). Every secret in `secrets` + * is searched for by value; the line reports the byte count scanned so a clean result is a + * measurement of something rather than the absence of a test. + * + * A leak prints the secret's LABEL and the 60 bytes of context around it — never the secret + * itself, because a proof script's output ends up in a terminal scrollback and a PR comment. + */ +export function checkClean( + where: string, + payload: string, + secrets: Readonly>, +): void { + checks++; + const hits = Object.entries(secrets).filter(([, value]) => + payload.includes(value), + ); + if (hits.length > 0) failures++; + if (hits.length === 0) { + console.log( + ` ok ${where.padEnd(34)} -> CLEAN (${String(Object.keys(secrets).length)} secrets scanned, ${String(payload.length)} bytes)`, + ); + return; + } + const first = hits[0] as [string, string]; + const at = payload.indexOf(first[1]); + const context = payload + .slice(Math.max(0, at - 30), at + first[1].length + 30) + .replaceAll(first[1], `<${first[0]}>`); + console.log( + ` FAIL ${where.padEnd(34)} -> LEAKED ${hits.map(([k]) => k).join(', ')} in ${String(payload.length)} bytes … ${context} …`, + ); +} + +/** + * The inverse of {@link checkClean}: assert the payload DOES contain a substring. Used where a + * disclosure is real and must be recorded — `describe_stitch` hands the model the internal + * endpoint URL and the auth scheme by design, and "by design" is not the same as "not disclosed". + */ +export function checkDiscloses( + where: string, + payload: string, + needle: string, +): void { + checks++; + const ok = payload.includes(needle); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${where.padEnd(34)} -> ${ok ? 'DISCLOSES' : 'absent'} ${show(needle)}`, + ); +} + +/** + * Assert on a field of a request the fake adapter recorded. Separate from `check` only so the + * output reads as a wire observation — `wire[0].headers.authorization` on the left, the exact + * bytes the vendor would have seen on the right. + */ +export function checkWire( + field: string, + actual: unknown, + expected: unknown, +): void { + checks++; + const ok = JSON.stringify(actual) === JSON.stringify(expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} wire ${field.padEnd(29)} = ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** Assert a measured number is at most `bound` — the request-count ceilings in C5. */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (<= ${String(bound)})` : ` (expected <= ${String(bound)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring an ABSENCE (C1's clean scans) and several by measuring a + * disclosure that is real (C3's config read-out), so the verdict statement always carries the + * direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/agent-holds-the-tool/safe-exposure.ts b/docs/scenarios/proofs/agent-holds-the-tool/safe-exposure.ts new file mode 100644 index 00000000..83b3bd5c --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/safe-exposure.ts @@ -0,0 +1,95 @@ +// The safest exposure StitchAPI can be given, as a caller would write it — what C8 counts and runs. +// +// Everything C1–C7 measured says the same thing about where the work is: the CREDENTIAL boundary is +// the library's and it holds; the ARGUMENT boundary is entirely the operator's. So this module is +// three small pieces, each closing one measured gap, and nothing here is configuration: +// +// • THE ALLOW-LIST (`expose`) — closes C3. `createMcpServer` takes whatever registry you hand +// it, so the allow-list is that object. It has to be built by NAMING what is exposed, and it +// has to be checked against `__config.name` too, because `selectStitch` resolves a stitch by +// its configured name even when the registry key is different (registry.ts:71-74). +// • THE INPUT FILTER (`only`) — closes C2 and C7. A `Proxy` apply-trap rebuilds the input from +// an explicit key list before the stitch ever sees it, so an undeclared slot is not a +// passthrough and a stripping schema is not needed (the engine discards a validator's parsed +// value — engine.ts:400-408 — so filtering has to happen out here). +// • THE METHOD GATE (`readsOnly`) — closes C6 as far as it can be closed. There is no channel +// from this process to a human (C6 a), so an irreversible call cannot be confirmed; it can only +// be refused. The gate wraps the `Adapter`, which is the last seam before the transport and +// outside the attempt loop. +// +// The one gap none of this closes is C4's: the error channel is an unfiltered `Error.message` +// pass-through, so a transport error that quotes the URL still reaches the model. The only fix is +// not to put a credential in a URL — `apiKey({ in: 'header' })` rather than `{ in: 'query' }`. +// >>> BEGIN USER CODE +import type { StitchRegistry } from '../../../../packages/core/src/registry'; +import type { + Adapter, + Stitch, + StitchInput, +} from '../../../../packages/core/src/types'; + +/** Which input slots — and which keys within them — a stitch accepts from an agent. */ +export interface Allowed { + params?: readonly string[]; + query?: readonly string[]; + body?: boolean; +} + +/** Rebuild an object from an explicit key list, dropping everything else. */ +function pickKeys( + value: unknown, + keys: readonly string[], +): Record { + const src = (value ?? {}) as Record; + const out: Record = {}; + for (const k of keys) if (src[k] !== undefined) out[k] = src[k]; + return out; +} + +/** + * Wrap a stitch so an agent's input is REBUILT from `allowed` before the engine sees it. A `Proxy` + * apply-trap keeps `__config` (and every other stitch member) intact, so `list_stitches` and + * `describe_stitch` still work on the wrapper. + */ +export function only(stitch: Stitch, allowed: Allowed): Stitch { + return new Proxy(stitch, { + apply(target, _thisArg, args: [StitchInput?]) { + const input = (args[0] ?? {}) as StitchInput; + const clean: StitchInput = {}; + if (allowed.params) + clean.params = pickKeys(input.params, allowed.params); + if (allowed.query) + clean.query = pickKeys(input.query, allowed.query); + if (allowed.body) clean.body = input.body; + return target(clean); + }, + }); +} + +/** + * Build the registry the MCP server is given. Every exposed stitch is named twice — once as the key + * an agent calls, once in the map — and a stitch whose CONFIGURED name is not the key it is exposed + * under is rejected, because that name would be callable while absent from `list_stitches`. + */ +export function expose(entries: Record): StitchRegistry { + for (const [key, stitch] of Object.entries(entries)) { + const configured = stitch.__config.name; + if (configured !== undefined && configured !== key) + throw new Error( + `expose: "${key}" is also reachable as "${configured}" — give the stitch the same name as its key`, + ); + } + return entries; +} + +/** Refuse anything that is not a read, at the last seam before the transport. */ +export function readsOnly(adapter: Adapter): Adapter { + return (req) => { + if (req.method !== 'GET' && req.method !== 'HEAD') + throw new Error( + `${req.method} is not available to an agent on this server`, + ); + return adapter(req); + }; +} +// <<< END USER CODE diff --git a/docs/scenarios/proofs/agent-holds-the-tool/stitches.ts b/docs/scenarios/proofs/agent-holds-the-tool/stitches.ts new file mode 100644 index 00000000..31d3a809 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/stitches.ts @@ -0,0 +1,164 @@ +// The stitches the MCP server exposes — the registry an operator would actually write. +// +// Ten stitches over four auth strategies, because C1 is only worth anything if it scans every +// shape a credential can take: +// +// bearer(env(...)) getOrder, listOrders, refund, mintApiKey +// apiKey({ in: 'header' }) getReport +// apiKey({ in: 'query' }) getMetrics ← the credential rides in the URL +// cookieSession({ login, cookie }) getProfile ← the credential is a captured cookie +// (none — it IS the login) login ← holds the login password +// +// Two shapes exist to isolate one variable each: +// - `getOrderTyped` declares an `input.params` schema, so C7 can measure whether a declared +// schema constrains what the model may send (and WHEN it is checked). +// - `searchOrders` declares an `input.headers` schema, which is the one documented way to opt a +// stitch INTO accepting model-supplied headers (mcp.ts:125-130). C2 measures what that opens. +// +// `listOrders` pins a tenant in its configured path (`/v1/orders?tenant=acme`). The vendor echoes +// the tenant it received, so the response body is a direct read-out of whose data the call +// returned — that echo is C2's sharpest measurement. +import { + apiKey, + bearer, + cookieSession, + env, +} from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { StitchRegistry } from '../../../../packages/core/src/registry'; +import type { Stitch } from '../../../../packages/core/src/types'; +import type { Validator } from '../../../../packages/core/src/validator'; +import { BASE, ENV, type Wire } from './vendor'; + +/** + * A hand-rolled `Validator` — `stitchapi` has no runtime dependencies and `docs/` has no manifest, + * so the proofs in this directory build schemas out of predicates rather than importing Zod. Only + * C7 depends on schema behaviour, and it depends on the ENGINE's use of a validator (when it runs, + * what it rejects), not on any schema library's coercion rules. + */ +export function schema( + label: string, + ok: (value: unknown) => boolean, +): Validator { + return { + validate: (value) => + Promise.resolve( + ok(value) + ? { ok: true as const, value } + : { + ok: false as const, + issues: [{ path: [], message: `expected ${label}` }], + }, + ), + }; +} + +const isRecord = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +/** `{ id: }` and nothing else — the tight params contract C7 measures against. */ +const numericIdOnly = schema( + '{ id: numeric string }', + (v) => + isRecord(v) && + Object.keys(v).length === 1 && + typeof v['id'] === 'string' && + /^[0-9]+$/.test(v['id']), +); + +/** A permissive headers contract — the shape an operator writes when they want ONE extra header. */ +const anyHeaders = schema('an object', (v) => v === undefined || isRecord(v)); + +/** + * Build the registry an operator would hand `serveStdio`. `wire` is the recording adapter every + * stitch shares, so one script sees every outbound request in one ordered list. + * + * A `seam` carries the shared adapter, base URL and store — and, for `cookieSession`, the vault the + * captured cookie lives in. `tenancy: 'app'` is the deliberate opt-in to one process-wide session + * (the fail-closed default demands a bound principal, which an MCP server has no way to supply). + */ +export function buildRegistry(wire: Wire): StitchRegistry & { + login: Stitch; +} { + const api = seam({ baseUrl: BASE, adapter: wire.adapter() }); + + const login = api.stitch({ + name: 'login', + method: 'POST', + path: '/auth/login', + }) as Stitch; + + const session = cookieSession({ + login, + cookie: 'SESSION', + tenancy: 'app', + loginInput: () => ({ + body: { user: 'svc', password: env(ENV.loginPassword)() }, + }), + }); + + return { + login, + getOrder: api.stitch({ + name: 'getOrder', + path: '/v1/orders/{id}', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }), + getOrderTyped: api.stitch({ + name: 'getOrderTyped', + path: '/v1/orders/{id}', + auth: bearer(env(ENV.bearer)), + input: { params: numericIdOnly }, + pick: 'data', + }), + listOrders: api.stitch({ + name: 'listOrders', + // The operator's tenant pin, in the configured path where a caller cannot see it. + path: '/v1/orders?tenant=acme', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }), + searchOrders: api.stitch({ + name: 'searchOrders', + path: '/v1/orders', + auth: bearer(env(ENV.bearer)), + // The documented opt-in: declaring a headers schema is what lets agent-supplied + // headers through `sanitizeAgentInput` at all. + input: { headers: anyHeaders }, + pick: 'data', + }), + getReport: api.stitch({ + name: 'getReport', + path: '/v1/reports', + auth: apiKey({ in: 'header', secret: env(ENV.apiKeyHeader) }), + pick: 'data', + }), + getMetrics: api.stitch({ + name: 'getMetrics', + path: '/v1/metrics', + auth: apiKey({ in: 'query', secret: env(ENV.apiKeyQuery) }), + pick: 'data', + }), + getProfile: api.stitch({ + name: 'getProfile', + path: '/v1/profile', + auth: session, + pick: 'data', + }), + refund: api.stitch({ + name: 'refund', + method: 'POST', + path: '/v1/refunds', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }), + mintApiKey: api.stitch({ + name: 'mintApiKey', + method: 'POST', + path: '/v1/api-keys', + auth: bearer(env(ENV.bearer)), + pick: 'data', + }), + }; +} diff --git a/docs/scenarios/proofs/agent-holds-the-tool/vendor.ts b/docs/scenarios/proofs/agent-holds-the-tool/vendor.ts new file mode 100644 index 00000000..6b510c47 --- /dev/null +++ b/docs/scenarios/proofs/agent-holds-the-tool/vendor.ts @@ -0,0 +1,217 @@ +// The vendor, and the wire tap. +// +// Two things live here, and the second is what makes the scenario measurable: +// +// 1. `SECRETS` — four distinct, greppable credential VALUES, one per auth strategy under test. +// Every C1 scan searches every JSON-RPC payload for all four by value, so a leak through any +// strategy fails the same assertion. They are deliberately unlike each other (`sk_live_…`, +// `ak_live_…`, `sess_live_…`) so a hit names which strategy leaked without printing the hit. +// 2. `Wire` — an `Adapter` that RECORDS every outbound request (url, method, headers, body) +// before answering it. The whole of C2 is "what did the vendor actually receive", and an +// adapter is the last seam before the transport, so a recording adapter is the closest thing +// to a packet capture that stays offline. +// +// The vendor routes on the request URL and enforces its own auth: a call that arrives without the +// right credential gets a 401 with a body, so "the credential still reached the vendor" is proved +// by a 200 rather than assumed. Nothing here does network I/O. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +export const BASE = 'https://api.vendor.test'; + +/** + * The four credential values under test, keyed by the label a leak would be reported under. + * + * `mintedKey` is not one of StitchAPI's credentials at all — it is a key the VENDOR returns in a + * response body, the way a real `POST /api-keys` endpoint does. It is in this set on purpose: C1 + * has to distinguish "the library leaked a secret it was holding" from "the model asked for data + * and the data was a secret", and the only way to tell those apart is to scan for both. + */ +export const SECRETS = { + bearer: 'sk_live_bearer_9f3a1c2e4d5b6a7c', + apiKeyHeader: 'ak_live_hdr_1122334455667788', + apiKeyQuery: 'ak_live_qry_8899aabbccddeeff', + session: 'sess_live_cookie_abcdef0123456789', + loginPassword: 'pw_live_login_0f1e2d3c4b5a6978', + mintedKey: 'ak_live_minted_00112233445566778899', +} as const; + +/** The credentials StitchAPI itself holds — the set the capability boundary is a promise about. */ +export const HELD_SECRETS = { + bearer: SECRETS.bearer, + apiKeyHeader: SECRETS.apiKeyHeader, + apiKeyQuery: SECRETS.apiKeyQuery, + session: SECRETS.session, + loginPassword: SECRETS.loginPassword, +} as const; + +/** Environment variables the stitches resolve their secrets from, via `env(...)`. */ +export const ENV = { + bearer: 'VENDOR_BEARER_TOKEN', + apiKeyHeader: 'VENDOR_API_KEY', + apiKeyQuery: 'VENDOR_METRICS_KEY', + loginPassword: 'VENDOR_LOGIN_PASSWORD', +} as const; + +/** Export the secrets into the process environment so `env(NAME)` resolves them at call time. */ +export function installSecrets(): void { + process.env[ENV.bearer] = SECRETS.bearer; + process.env[ENV.apiKeyHeader] = SECRETS.apiKeyHeader; + process.env[ENV.apiKeyQuery] = SECRETS.apiKeyQuery; + process.env[ENV.loginPassword] = SECRETS.loginPassword; +} + +/** One outbound request, exactly as the transport would have sent it. */ +export interface WireRequest { + url: string; + method: string; + headers: Record; + body: unknown; +} + +/** How a route answers. Returning a response is the normal path; throwing models a transport failure. */ +export type Route = (req: WireRequest) => AdapterResponse; + +const json = (status: number, body: unknown): AdapterResponse => ({ + status, + headers: { 'content-type': 'application/json' }, + body, +}); + +const UNAUTHORIZED = (what: string): AdapterResponse => + json(401, { error: 'unauthorized', detail: `bad or missing ${what}` }); + +/** Read a cookie pair out of a `Cookie` request header. */ +function cookieValue(header: string | undefined, name: string): string | null { + for (const part of (header ?? '').split(';')) { + const eq = part.indexOf('='); + if (eq < 0) continue; + if (part.slice(0, eq).trim() === name) return part.slice(eq + 1).trim(); + } + return null; +} + +/** + * The vendor's routing table. Each entry checks the credential it requires and answers 401 when it + * is absent or wrong — so a passing 200 in a proof is evidence the real credential arrived, not an + * artefact of a permissive stub. + * + * `/v1/orders` is deliberately configured with a PINNED query parameter upstream + * (`path: '/v1/orders?tenant=acme'`), and echoes the tenant it received back in the body. That + * echo is how C2 measures whether a model-supplied `query` can overwrite an operator's pin. + */ +export function route(req: WireRequest): AdapterResponse { + const url = new URL(req.url); + const path = url.pathname; + const auth = req.headers['authorization']; + const bearerOk = auth === `Bearer ${SECRETS.bearer}`; + + if (path === '/auth/login') { + const body = req.body as { password?: string } | undefined; + if (body?.password !== SECRETS.loginPassword) + return UNAUTHORIZED('login password'); + return { + status: 200, + headers: { + 'content-type': 'application/json', + 'set-cookie': `SESSION=${SECRETS.session}; Path=/; HttpOnly`, + }, + body: { ok: true }, + }; + } + if (path === '/v1/profile') { + const session = cookieValue(req.headers['cookie'], 'SESSION'); + if (session !== SECRETS.session) return UNAUTHORIZED('SESSION cookie'); + return json(200, { data: { id: 'u_1', email: 'ada@vendor.test' } }); + } + if (path === '/v1/reports') { + if (req.headers['x-api-key'] !== SECRETS.apiKeyHeader) + return UNAUTHORIZED('X-API-Key header'); + return json(200, { data: { rows: 3 } }); + } + if (path === '/v1/metrics') { + // The query arm: the credential rides in the URL. Read the FIRST occurrence, which is what + // Express/Rails/Go's net/http all do — it matters when a model appends a second one. + if (url.searchParams.get('api_key') !== SECRETS.apiKeyQuery) + return UNAUTHORIZED('api_key query param'); + return json(200, { data: { uptime: 0.999 } }); + } + if (path === '/v1/orders') { + if (!bearerOk) return UNAUTHORIZED('bearer token'); + return json(200, { + data: { + tenant: url.searchParams.get('tenant'), + orders: [{ id: 'o_1', total: 4200 }], + }, + }); + } + if (path.startsWith('/v1/orders/')) { + if (!bearerOk) return UNAUTHORIZED('bearer token'); + return json(200, { + data: { id: path.slice('/v1/orders/'.length), total: 4200 }, + }); + } + if (path === '/v1/refunds') { + if (!bearerOk) return UNAUTHORIZED('bearer token'); + const body = req.body as { amount?: number } | undefined; + return json(200, { + data: { refundId: 're_9', amount: body?.amount ?? 0 }, + }); + } + if (path === '/v1/api-keys') { + if (!bearerOk) return UNAUTHORIZED('bearer token'); + // A real vendor endpoint that MINTS a credential and returns it in the response body. + return json(200, { + data: { id: 'key_7', secret: SECRETS.mintedKey }, + }); + } + // The internal service an SSRF would aim at. Reachable only if something can redirect the host. + if (url.host === 'metadata.internal') + return json(200, { data: { role: 'admin', token: 'INTERNAL' } }); + return json(404, { error: 'no such route', path }); +} + +/** + * The wire tap: an `Adapter` that appends every request to `requests` before answering it. + * + * `handler` defaults to the vendor's routing table; a script that needs a specific failure (a + * transport throw, a 500, a slow call) passes its own and still gets the recording. + */ +export class Wire { + readonly requests: WireRequest[] = []; + constructor(private readonly handler: Route = route) {} + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const seen: WireRequest = { + url: req.url, + method: req.method, + headers: { ...req.headers }, + body: req.body, + }; + this.requests.push(seen); + return this.handler(seen); + }; + } + + get count(): number { + return this.requests.length; + } + /** The most recent request. Throws rather than returning `undefined` — a proof that reads a + * request that was never made should fail loudly, not compare against nothing. */ + get last(): WireRequest { + const r = this.requests.at(-1); + if (!r) throw new Error('wire: no request was made'); + return r; + } + /** Every request's URL, in order — the SSRF/redirect spine. */ + get urls(): string[] { + return this.requests.map((r) => r.url); + } + reset(): void { + this.requests.length = 0; + } +} diff --git a/docs/scenarios/proofs/async-job-polling/README.md b/docs/scenarios/proofs/async-job-polling/README.md new file mode 100644 index 00000000..83ec3642 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/README.md @@ -0,0 +1,117 @@ +# Proofs — submit, poll, download: the async job triangle + +Runnable evidence for the claims in +[`../../async-job-polling.md`](../../async-job-polling.md). + +Every script is standalone, offline, and deterministic: it injects a fake three-endpoint job API +through StitchAPI's `adapter` seam and drives every wait off an injected `manualClock()`, so an +hour-long poll is exact **virtual** time — no real sleeping, nothing flaky, no network. The one +deliberate exception is C5(a), which measures `timeout.total`; that budget is wall-clock **by +design** (engine.ts:453-483), so it runs on real timers at 250ms with bounds set 4× clear of the +real numbers. + +**The fake provider counts submissions.** `api.submits` is 2 if the client POSTed `/jobs` twice, so +"a duplicate job was created" is a measured number, not an argument — and `api.gaps(id)` is the +exact virtual spacing between polls, so "it honoured the server's pacing" is a list of integers. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/async-job-polling/c5-one-deadline.ts + +# all of them +for f in docs/scenarios/proofs/async-job-polling/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, +so they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set (the `@ts-expect-error` blocks are the +machine-checked half of several claims — a `@ts-expect-error` that is _not_ an error fails `tsc`): + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/async-job-polling/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| --------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | +| `c1-location-header.ts` | can `Location` become the next call's URL? | **Yes, through `hooks`.** `POST /jobs → 3× GET /jobs/job-1` in ONE stitch, 1 submit. Nothing built-in follows it | +| `c2-poll-surface.ts` | can a `Surface` express the poll loop? | **Yes.** 5 polls at 30s spacing, `attempts: 5`; `Failed` stops on the first terminal body with 17/20 unspent | +| `c3-retry-after.ts` | can the wait come from `Retry-After`? | **Not by itself.** `respect: true` gave 7ms, not 30s. The surface can read it — and `after: raw` is **1000× off** | +| `c4-paginate.ts` | can `paginate` express it? | **It loops** (capture refuted) — at gaps `0,0,0`. The natural `items` ends the run `ok` with `[]` after 1 poll | +| `c5-one-deadline.ts` | ONE deadline over the triangle? | **Yes, two ways.** `timeout.total` on a one-stitch triangle (253ms); or one `AbortSignal` through `linked` | +| `c6-linked-trace.ts` | does `linked` draw one trace chain? | **Yes.** 3 starts, 1 traceId, spans chained. But no operation-level span — a failure names the STEP | +| `c7-single-use-download.ts` | does `retry` hammer a spent link? | **Not by default** (200,404). Widen `on` to 404 and it does (200,404,404,404). Per-stitch split works | +| `c8-resume.ts` | can a stitch reattach after a restart? | **Entirely user-side.** 0 store keys written; `cache` on the submit stops the dup and **loses the job id** | +| `c9-assembled-solution.ts` | best answer, and is it worth it? | 1 submit, 5 polls at the server's pacing, 1 download, resume, deadline — in **110 lines vs 49** hand-rolled | + +## Files + +- `fake-jobs.ts` — the provider. `POST /jobs` → **202** + `Location` (+ optional `Retry-After`); + `GET /jobs/{id}` → **200** `{ state }` cycling `InProgress` then `JobComplete`/`Failed`; + `GET ` → the payload, **single-use** (the second fetch 404s). Records every hit with + the virtual timestamp, so `submits`, `polls(id)` and `gaps(id)` are all measurements. +- `harness.ts` — `check` / `checkAtMost` / `note` / `heading` / `finish`. No test framework. +- `job-triangle.ts` — **user code** for C9: `jobPollSurface()`, `retryAfterMs()`, + `operationDeadline()` and `jobTriangle()`, the assembled answer. + +## Reading the numbers honestly + +- **C4 refutes the research capture, and the refutation is worse than the prediction.** The capture + expected `paginate` to fail immediately because a job-status body has no items array. It does not + fail: the default `items` wraps a non-array value as `[value]` (engine.ts:967-971), so + `items.length` is 1 every round and the loop runs to a clean `next → undefined` termination. What + it cannot do is **wait** — measured gaps `0,0,0`, and `PaginateOptions` has no `delay`/`backoff` + field (machine-checked). Then C4(d) is the real trap: a caller who writes `items` to pull the + result rows gets zero items on the first `InProgress` page, `paginated` breaks at engine.ts:984 + **before** consulting `next`, and the run ends `ok` with `data: []`, one poll, no error and no + drift finding — while the job is still running server-side. Same bug shape as scenario 3's C2(d), + reached from the opposite direction. +- **C3(c) is the sharpest footgun in this scenario, and it typechecks.** `SurfaceOutcome.after` is + `number | string` and takes the house duration form, where a bare numeric string is + **milliseconds**. `Retry-After` is delta-**seconds**. So `after: res.headers['retry-after']` — + the obvious spelling, the one that reads correctly — polls **1000× faster** than the server asked + (measured gaps `30,30,30` against the requested 30s). Nothing warns. Appending an `s` to the raw + header is the only correct form, and the HTTP-date variant needs hand-written RFC 9110 parsing + because `parseRetryAfter` is not on the public barrel (measured: + `'parseRetryAfter' in barrel === false`). +- **C5(e) means you cannot test the thing this scenario is about.** `sleepWithin` compares + `budget.deadline - now()` — **wall-clock** — and then sleeps on the **injected** clock + (engine.ts:481-488). Virtual time therefore never consumes `timeout.total`: 60 polls across 59 + virtual seconds under a `total: '10s'` never tripped it. A `manualClock` test of "give up after an + hour" passes while proving nothing, and any production code that injects a custom clock loses the + budget silently. +- **C1(g) is the price of the one-stitch construction, and it is the same class of bug as scenario + 3's C7(f).** `HookContext` is `{ name, attempt, req?, res?, error? }` — no run id, no per-call + slot — so the carried `Location` has to live in a closure on the **stitch**. Two concurrent calls + measured: 2 jobs submitted, **job-1 polled zero times** (submitted, orphaned, running to + completion unread), both callers handed job-2's result. It typechecks and it reads correctly. +- **C8(c) is the trap that looks like the fix.** `cache: { methods: ['POST'] }` over a shared store + really does stop a restarted process re-submitting (1 submit across two processes). But a cache + entry is the **value**, and a 202's value is `{}` — the `Location` is not in it — and a cache HIT + short-circuits the request so `hooks.onResponse` never fires (measured: 1 firing across two runs). + The duplicate is avoided by orphaning the job instead. +- **C7(d) is the cost of C5(a).** One stitch is one `retry` block, so the poll's patience is also the + download's: a spent single-use link consumed all 8 shared attempts. The three-stitch `linked` + shape gets this right (20 poll attempts, 1 download attempt) but gives up the single + `timeout.total`. The two properties are not simultaneously reachable today. +- **C2(c)/C5(a) — every failure arrives as a `StitchError` with a string.** "The job failed", "I ran + out of polls" and "the deadline fired" are `job failed: …`, `InProgress` and + `timed out after 250ms`. The engine's own `TimeoutError` identity does not survive to the caller, + and `error.status` is `undefined` on all three. Distinguishing them means matching strings. +- **C9's line counts are the honest comparison.** 110 executable lines for `jobTriangle` + the poll + surface + the deadline + the header parse, against 49 for the hand-rolled `while` (the header + parse is counted on **both** sides — a correct `while` needs it too). The wire behaviour is + byte-identical. What the extra 61 lines buy is measured: one `start`/`done` per hop with the polls + folded in as `attempts: 3`, one traceId chaining `job-submit → job-poll → job-download`, a per-hop + retry policy, and `throttle`/`circuit` wrapping every hop. diff --git a/docs/scenarios/proofs/async-job-polling/c1-location-header.ts b/docs/scenarios/proofs/async-job-polling/c1-location-header.ts new file mode 100644 index 00000000..398ea86a --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c1-location-header.ts @@ -0,0 +1,302 @@ +// C1 — can the `Location` header on the `202` become the NEXT call's URL through a documented +// seam, or must the caller parse it by hand outside the library? +// +// The `202` body is empty by design (the fake mirrors Salesforce/Shopify here): the job id exists +// in exactly one place on the wire — a RESPONSE HEADER. So this walks every seam that could carry a +// header forward and measures which ones can even SEE it. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c1-location-header.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { AdapterResponse } from '../../../../packages/core/src/types'; +import { FakeJobApi, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +const HOST = 'https://bulk.example.com'; + +async function main(): Promise { + heading('C1 — can the `Location` header become the next call’s URL?'); + + // ── (a) the 202 resolves SUCCESSFULLY, and the value carries nothing ─────────────────────── + // `classifyStatus` passes anything < 400 (surface.ts:143-149), so no `verdict.accept` is + // needed — and the caller is handed `{}`. Neither `.inspect().raw` nor `.report()` exposes + // response headers, so from the awaited API the job id does not exist. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + const r = await submit.safe({ body: { q: 'SELECT Id FROM Account' } }); + check('(a) a 202 resolves ok', r.ok, true); + check('(a) the value handed back', JSON.stringify(r.data), '{}'); + const insp = await submit.inspect({ body: {} }); + check('(a) `.inspect().raw`', JSON.stringify(insp.raw), '{}'); + const rep = await submit.report({ body: {} }); + check( + '(a) does any public result surface expose headers?', + [...Object.keys(rep), ...Object.keys(insp)].some((k) => + k.toLowerCase().includes('header'), + ), + false, + ); + note('(a) `.report()` keys', Object.keys(rep).join(', ')); + // Three, because `.inspect()` and `.report()` are FRESH network probes (types.ts:1841, + // 1856) — each one submitted another job. On this endpoint a diagnostic probe is a write. + check('(a) jobs the server minted', api.jobIds.length, 3); + note( + '(a) → the id is on the wire and unreachable from `data` / `raw` / `report`', + '', + ); + } + + // ── (b) `paginate.next` is handed the BODY, never the response ──────────────────────────── + // `(prevBody, pagesFetched)` — types.ts:1417. The loop that looks most like "follow the next + // URL" is structurally blind to headers. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + paginate: { + // @ts-expect-error — `next` takes (prevBody, pagesFetched); there is no response + // (and so no headers) argument. + next: (_prev: unknown, _pages: number, _res: AdapterResponse) => + undefined, + }, + }); + note('(b) `PaginateOptions.next`', '(prevBody, pagesFetched) => input'); + } + + // ── (c) `transform` is handed the BODY too ──────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + // @ts-expect-error — `transform` takes (body); there is no response argument. + transform: (body: unknown, _res: AdapterResponse) => body, + }); + note('(c) `StitchConfig.transform`', '(body) => unknown'); + } + + // ── (d) `Surface.interpret` DOES see the whole response ─────────────────────────────────── + // `(res, cfg)` — surface.ts:61-64 — and `res.headers` is a plain record. So a surface can lift + // `Location` into the VALUE, which is the first half of the hop. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + const lifting: Surface = { + id: 'submit', + interpret: (res) => ({ + ok: true, + data: { location: res.headers['location'] }, + }), + }; + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + kind: lifting, + adapter: api.adapter(), + clock, + }); + const r = await submit.safe({ body: {} }); + check( + '(d) the value a lifting surface hands back', + JSON.stringify(r.data), + '{"location":"/jobs/job-1"}', + ); + } + + // ── (e) `hooks` complete the hop INSIDE one stitch ──────────────────────────────────────── + // `onResponse` reads `res.headers.location` (engine.ts:703); `onRequest` runs on the + // per-attempt clone before the transport (engine.ts:646-654), so assigning `ctx.req.url` and + // `ctx.req.method` there redirects the NEXT attempt. Paired with a body-aware surface, one + // stitch walks `POST /jobs` → `GET /jobs/{id}` with no user code between calls. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + let next: { url: string; method: string } | undefined; + const hopping: Surface = { + id: 'async-job', + interpret: (res) => + res.status === 202 || stateOf(res.body) === 'InProgress' + ? { + ok: false, + retry: true, + message: 'not done', + after: 1000, + } + : { ok: true, data: res.body }, + }; + const call = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + kind: hopping, + adapter: api.adapter(), + clock, + retry: { attempts: 6 }, + hooks: { + onRequest: (ctx) => { + if (ctx.req && next) { + ctx.req.url = next.url; + ctx.req.method = next.method; + ctx.req.body = undefined; + } + }, + onResponse: (ctx) => { + const loc = ctx.res?.headers['location']; + if (loc) + next = { + url: new URL(loc, HOST).toString(), + method: 'GET', + }; + }, + }, + }); + const p = call.safe({ body: { q: 'SELECT Id' } }); + await clock.advance(3_600_000); + const r = await p; + + check('(e) the call succeeded', r.ok, true); + check( + '(e) state reached', + stateOf(r.data as unknown), + 'JobComplete' as const, + ); + check( + '(e) what the client actually requested', + api.hits.map((h) => `${h.method} ${h.path}`).join(' → '), + 'POST /jobs → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1', + ); + check('(e) SUBMITS (1 = the hop replaced the re-POST)', api.submits, 1); + } + + // ── (f) across TWO stitches: the RFC 6570 `+` operator makes a path a URL ────────────────── + // `url` is templated (util.ts `expandPath`), and `{+var}` is reserved expansion — slashes pass + // through unencoded. So a `Location` carried in a plain variable becomes the poll stitch's URL + // with no string concatenation. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 0 }); + let location = ''; + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + hooks: { + onResponse: (ctx) => { + location = ctx.res?.headers['location'] ?? ''; + }, + }, + }); + const poll = stitch({ + url: `${HOST}{+loc}`, + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + const r = await poll.safe({ params: { loc: location } }); + check('(f) the header value carried', location, '/jobs/job-1'); + check('(f) the poll hit', api.hits.at(-1)?.path, '/jobs/job-1'); + check('(f) it succeeded', r.ok, true); + // Without `+`, the default operator percent-encodes the slashes into one path segment. + const naive = stitch({ + url: `${HOST}/{loc}`, + adapter: api.adapter(), + clock, + }); + await naive.safe({ params: { loc: location } }); + check( + '(f) the SAME value under the default operator `{loc}`', + api.hits.at(-1)?.path, + '/%2Fjobs%2Fjob-1', + ); + check('(f) …and that request 404s', api.hits.at(-1)?.status, 404); + } + + // ── (g) the cost of (e): hook state lives on the STITCH, not the call ───────────────────── + // `HookContext` is `{ name, attempt, req?, res?, error? }` (types.ts:1278-1284) — no run id, no + // per-call slot. So the carried `Location` has to be a closure variable on the stitch, and two + // concurrent calls through one stitch overwrite each other's. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + let next: { url: string; method: string } | undefined; + const hopping: Surface = { + id: 'async-job', + interpret: (res) => + res.status === 202 || stateOf(res.body) === 'InProgress' + ? { + ok: false, + retry: true, + message: 'not done', + after: 1000, + } + : { ok: true, data: res.body }, + }; + const call = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + kind: hopping, + adapter: api.adapter(), + clock, + retry: { attempts: 8 }, + hooks: { + onRequest: (ctx) => { + if (ctx.req && next) { + ctx.req.url = next.url; + ctx.req.method = next.method; + ctx.req.body = undefined; + } + }, + onResponse: (ctx) => { + const loc = ctx.res?.headers['location']; + if (loc) + next = { + url: new URL(loc, HOST).toString(), + method: 'GET', + }; + }, + }, + }); + const a = call.safe({ body: { q: 'A' } }); + const b = call.safe({ body: { q: 'B' } }); + await clock.advance(3_600_000); + const [ra, rb] = await Promise.all([a, b]); + + check('(g) jobs submitted', api.submits, 2); + check('(g) polls of job-1', api.polls('job-1').length, 0); + check('(g) polls of job-2', api.polls('job-2').length, 4); + check('(g) caller A resolved ok', ra.ok, true); + check('(g) caller B resolved ok', rb.ok, true); + check( + '(g) both callers were handed the SAME job', + (ra.data as { id?: string } | null)?.id === + (rb.data as { id?: string } | null)?.id, + true, + ); + note( + '(g) → job-1 was submitted and never polled: it runs to completion server-side, unread', + '', + ); + } + + finish( + 'C1', + 'the `Location` header IS reachable — `Surface.interpret` and `hooks.onResponse` both receive the full `AdapterResponse`, and assigning `ctx.req.url`/`ctx.req.method` in `hooks.onRequest` makes the 202→poll hop happen inside ONE stitch (measured: POST /jobs → 3× GET /jobs/job-1, 1 submit). Nothing built-in follows it: `paginate.next` and `transform` are handed the BODY only, and no public result surface (`data`, `.inspect().raw`, `.report()`) exposes response headers. The hook seam costs concurrency safety — two calls through one stitch orphaned job-1 (0 polls) and both polled job-2', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c2-poll-surface.ts b/docs/scenarios/proofs/async-job-polling/c2-poll-surface.ts new file mode 100644 index 00000000..8037257b --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c2-poll-surface.ts @@ -0,0 +1,255 @@ +// C2 — can a custom `Surface` express the poll loop? `interpret` reads `state`: `InProgress` → +// `{ ok: false, retry: true, after }`, `JobComplete` → `{ ok: true }`, `Failed` → a real failure +// that does NOT retry. Every one of those arrives at HTTP 200, so nothing status-driven can tell +// them apart. +// +// Measured: the poll count, the gaps between polls on the injected clock, that `Failed` stops the +// loop dead, and what the caller is handed when the poll budget runs out first. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c2-poll-surface.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { verdictOf } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + AdapterResponse, + Clock, + ResolvedStitchConfig, + StitchEvent, +} from '../../../../packages/core/src/types'; +import { FakeJobApi, errorMessageOf, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +/** The poll loop, as a surface. `after` fixed so the gaps are exact virtual time. */ +const pollSurface = (after: number): Surface => ({ + id: 'job-poll', + interpret: (res, cfg) => { + const failed = verdictOf(res, cfg); + if (failed) return failed; + const state = stateOf(res.body); + if (state === 'InProgress') + return { ok: false, retry: true, message: 'InProgress', after }; + if (state === 'Failed') + return { + ok: false, + message: `job failed: ${errorMessageOf(res.body)}`, + status: res.status, + }; + return { ok: true, data: res.body }; + }, +}); + +/** Submit a job through a plain stitch and hand back its id. */ +async function submitJob(api: FakeJobApi, clock: Clock): Promise { + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + return api.jobIds.at(-1)!; +} + +async function main(): Promise { + heading('C2 — can a `Surface` express the poll loop?'); + + // ── (a) InProgress → retry, JobComplete → done ──────────────────────────────────────────── + // The body-aware retry arm (surface.ts:36, engine.ts:775-806) re-enters the attempt loop, so + // "not done yet" IS a retry — and `retry.attempts` is the poll bound. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 4 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 10 }, + }); + const evts: StitchEvent[] = []; + const consume = (async (): Promise => { + for await (const e of poll.stream()) evts.push(e); + })(); + await clock.advance(3_600_000); + await consume; + + check('(a) polls made', api.polls(id).length, 5); + check( + '(a) gaps between polls (ms, virtual)', + api.gaps(id).join(','), + '30000,30000,30000,30000', + ); + const result = evts.find((e) => e.type === 'result'); + check( + '(a) terminal state returned', + stateOf(result && 'data' in result ? result.data : undefined), + 'JobComplete' as const, + ); + check( + '(a) attempts the engine reported', + result && 'attempts' in result ? result.attempts : undefined, + 5, + ); + check( + '(a) `retry` progress events (one per re-poll)', + evts.filter((e) => e.type === 'progress' && e.phase === 'retry') + .length, + 4, + ); + const firstRetry = evts.find( + (e) => e.type === 'progress' && e.phase === 'retry', + ); + check( + '(a) the retry detail names the BODY, not a status', + firstRetry?.type === 'progress' ? firstRetry.detail : undefined, + 'interpret: InProgress', + ); + note( + '(a) virtual time the last poll landed at (ms)', + api.polls(id).at(-1)?.at, + ); + } + + // ── (b) `Failed` is a real failure and does NOT retry ───────────────────────────────────── + // The third arm of `SurfaceOutcome` (no `retry` key) falls through to the ordinary failure + // handling. Measured: the loop stops on the FIRST Failed body with poll budget still unspent. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 2, + terminal: 'Failed', + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 20 }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + + check( + '(b) polls made (2 InProgress + 1 Failed)', + api.polls(id).length, + 3, + ); + check('(b) the call failed', r.ok, false); + check( + '(b) error message', + r.error?.message, + 'job failed: InvalidBatch : Field name not found', + ); + check('(b) poll budget left unspent', 20 - api.polls(id).length, 17); + check( + '(b) virtual time the terminal poll landed at (ms)', + api.polls(id).at(-1)?.at, + 60_000, + ); + } + + // ── (c) running out of poll budget is NOT distinguishable from a job failure ────────────── + // Both arrive as a plain `StitchError` whose `message` is whatever the surface said. There is + // no "still running when I gave up" error type, and `status` is undefined on both. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 50 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(1000), + adapter: api.adapter(), + clock, + retry: { attempts: 3 }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + + check('(c) polls made', api.polls(id).length, 3); + check('(c) the call failed', r.ok, false); + check('(c) error name', r.error?.name, 'StitchError'); + check('(c) error message', r.error?.message, 'InProgress'); + check( + '(c) error status', + (r.error as { status?: number } | undefined)?.status, + undefined, + ); + note( + '(c) → "the job is still running" and "the job failed" differ only by the string the surface chose', + '', + ); + } + + // ── (d) `interpret` is not told which attempt it is on ──────────────────────────────────── + // `(res, cfg)` — surface.ts:61-64. A surface cannot vary its wait by poll number, cap the poll + // count itself, or know it is on the last one. + { + const threeArg: Surface = { + id: 'x', + // @ts-expect-error — `interpret` takes (res, cfg); there is no attempt argument. + interpret: ( + res: AdapterResponse, + _cfg: ResolvedStitchConfig, + _attempt: number, + ) => ({ ok: true as const, data: res.body }), + }; + void threeArg; + note('(d) `Surface.interpret`', '(res, cfg) => SurfaceOutcome'); + } + + // ── (e) a surface that skips `verdictOf` turns HTTP errors into successes ───────────────── + // `interpret` REPLACES the default rather than layering on it (surface.ts:174-178). Omit the + // composition and a 404 comes back `ok: true` with the error page as the value. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + const naive: Surface = { + id: 'naive', + interpret: (res) => + stateOf(res.body) === 'InProgress' + ? { + ok: false, + retry: true, + message: 'InProgress', + after: 1, + } + : { ok: true, data: res.body }, + }; + const gone = 'https://bulk.example.com/jobs/job-does-not-exist'; + const bad = stitch({ + url: gone, + kind: naive, + adapter: api.adapter(), + clock, + }); + const rb = await bad.safe(); + check('(e) naive surface, HTTP 404 → ok', rb.ok, true); + check( + '(e) naive surface, the value handed back', + JSON.stringify(rb.data), + '{"message":"unknown job job-does-not-exist"}', + ); + const good = stitch({ + url: gone, + kind: pollSurface(1), + adapter: api.adapter(), + clock, + }); + const rg = await good.safe(); + check('(e) `verdictOf`-composed surface, HTTP 404 → ok', rg.ok, false); + check('(e) …error message', rg.error?.message, 'HTTP 404'); + } + + finish( + 'C2', + 'YES — a `Surface` expresses the whole poll loop. `InProgress` → `{ ok: false, retry: true, after }` polled 5 times at exactly 30s virtual spacing and returned the `JobComplete` body with `attempts: 5`; `Failed` stopped on the first terminal body (3 polls) with 17 of 20 poll attempts unspent. Two costs: the surface must compose `verdictOf` or a 404 comes back as a SUCCESS, and "I gave up waiting" is a plain `StitchError` carrying the surface’s own string — indistinguishable in type from the job having failed', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c3-retry-after.ts b/docs/scenarios/proofs/async-job-polling/c3-retry-after.ts new file mode 100644 index 00000000..7c8e4db3 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c3-retry-after.ts @@ -0,0 +1,213 @@ +// C3 — can the poll wait come from the response's `Retry-After` HEADER? `retry.respect` honours it +// for STATUS-driven retries (engine.ts:744-756) and defaults to `true`. This measures whether +// anything honours it on the BODY-driven (`SurfaceOutcome.retry`) path a poll loop runs on — and +// what the fallback looks like when the server sends no header. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c3-retry-after.ts +import * as barrel from '../../../../packages/core/src/index'; +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Clock } from '../../../../packages/core/src/types'; +import { FakeJobApi, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +/** Submit a job through a plain stitch and hand back its id. */ +async function submitJob(api: FakeJobApi, clock: Clock): Promise { + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + return api.jobIds.at(-1)!; +} + +/** + * A poll surface parameterised by how it derives `after` from the response. `undefined` means it + * sets no `after` at all — the case that asks whether the ENGINE picks the header up. + */ +const pollWith = ( + after: (h: Record) => number | string | undefined, +): Surface => ({ + id: 'job-poll', + interpret: (res) => { + if (stateOf(res.body) !== 'InProgress') + return { ok: true, data: res.body }; + const wait = after(res.headers); + return wait === undefined + ? { ok: false, retry: true, message: 'InProgress' } + : { ok: false, retry: true, message: 'InProgress', after: wait }; + }, +}); + +async function main(): Promise { + heading('C3 — can the poll wait come from the `Retry-After` header?'); + + // ── (a) `retry.respect` does NOT reach the body-driven path ─────────────────────────────── + // The status path reads `res.headers['retry-after']` (engine.ts:749-751). The body path four + // lines below reads only `parseDuration(outcome.after)` (engine.ts:797-801). With the server + // asking for 30s and a deliberately odd 7ms computed backoff, the measured gap says which ran. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 3, + retryAfter: 30, + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollWith(() => undefined), + adapter: api.adapter(), + clock, + retry: { + attempts: 6, + respect: true, + backoff: { curve: 'fixed', base: 7 }, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + + check('(a) the server asked for (s)', 30, 30); + check('(a) polls made', api.polls(id).length, 4); + check('(a) gaps (ms)', api.gaps(id).join(','), '7,7,7'); + check('(a) the call succeeded', r.ok, true); + note( + '(a) → `respect: true` is inert here; 7ms is the computed backoff, not the server’s 30s', + '', + ); + } + + // ── (b) the SURFACE can read the header itself — with the unit spelled out ──────────────── + // `interpret` gets the whole response, so `res.headers['retry-after']` is in reach. `after` + // takes the house duration form (CONTRACT.md P17), so delta-SECONDS must be written `'30s'`. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 3, + retryAfter: 30, + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollWith((h) => { + const raw = h['retry-after']; + return raw === undefined ? undefined : `${raw}s`; + }), + adapter: api.adapter(), + clock, + retry: { attempts: 6, backoff: { curve: 'fixed', base: 7 } }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + await p; + check('(b) gaps (ms)', api.gaps(id).join(','), '30000,30000,30000'); + } + + // ── (c) THE UNIT TRAP: the raw header value is read as MILLISECONDS ─────────────────────── + // `after?: number | string` accepts `'30'`, and `parseDuration('30')` is 30ms. `Retry-After` + // is delta-SECONDS. So the obvious spelling — hand the header straight through — polls a + // thousand times faster than the server asked, and nothing warns. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 3, + retryAfter: 30, + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollWith((h) => h['retry-after']), + adapter: api.adapter(), + clock, + retry: { attempts: 6, backoff: { curve: 'fixed', base: 7 } }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + await p; + check('(c) gaps (ms)', api.gaps(id).join(','), '30,30,30'); + check( + '(c) how much faster than the server asked', + 30_000 / api.gaps(id)[0]!, + 1000, + ); + } + + // ── (d) the HTTP-date form of `Retry-After` silently falls back to the backoff ──────────── + // RFC 9110 allows either delta-seconds or an HTTP-date. `parseDuration` cannot read a date, and + // an unparseable `after` falls through to the computed curve (engine.ts:797-801) — no warning, + // no drift finding. The engine's own `parseRetryAfter` handles both forms but is NOT exported. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 3, + retryAfter: 'Thu, 01 Jan 1970 00:01:00 GMT', + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollWith((h) => h['retry-after']), + adapter: api.adapter(), + clock, + retry: { attempts: 6, backoff: { curve: 'fixed', base: 7 } }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + await p; + check('(d) gaps (ms)', api.gaps(id).join(','), '7,7,7'); + check( + '(d) is `parseRetryAfter` on the public barrel?', + 'parseRetryAfter' in barrel, + false, + ); + note( + '(d) → a surface author must re-implement RFC 9110 date parsing, or lose the server’s pacing silently', + '', + ); + } + + // ── (e) the fallback: a capped exponential when there is no header ──────────────────────── + // With no `Retry-After` the surface omits `after` and `backoff` shapes the curve. `curve: 'expo'` + // (not the `'expo-jitter'` default) makes the doubling exact and the cap visible. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 5 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollWith((h) => { + const raw = h['retry-after']; + return raw === undefined ? undefined : `${raw}s`; + }), + adapter: api.adapter(), + clock, + retry: { + attempts: 8, + backoff: { curve: 'expo', base: 1000, max: 5000 }, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + await p; + check( + '(e) gaps (ms) — doubling, then clamped at `max`', + api.gaps(id).join(','), + '1000,2000,4000,5000,5000', + ); + check('(e) polls made', api.polls(id).length, 6); + } + + finish( + 'C3', + "NOTHING honours `Retry-After` on the body-driven path — with `respect: true` and the server asking for 30s, the measured gaps were 7ms (the computed backoff). The surface must read the header itself, and `after: '${raw}s'` is the ONLY correct spelling: the naive `after: raw` reads delta-seconds as MILLISECONDS and polls 1000× faster than asked, while an HTTP-date `Retry-After` falls back to the computed curve silently (`parseRetryAfter` is not exported). The absent-header fallback is a real capped exponential: 1000, 2000, 4000, 5000, 5000", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c4-paginate.ts b/docs/scenarios/proofs/async-job-polling/c4-paginate.ts new file mode 100644 index 00000000..4b8feaee --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c4-paginate.ts @@ -0,0 +1,266 @@ +// C4 — can `paginate` express the poll loop? The research capture predicted it would fail +// immediately, because scenario 3 measured the loop breaking at `items.length === 0` +// (engine.ts:984) and a job-status body has no items array at all. +// +// That prediction is WRONG, and the way it is wrong is worse than a clean failure: with the default +// `items` a non-array body is wrapped as `[value]` (engine.ts:967-971), so `length` is 1 every +// round and the loop runs. It just cannot WAIT — and the natural `items` spelling makes it end the +// run SUCCESSFULLY, with an empty array, while the job is still running. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c4-paginate.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Clock, StitchEvent } from '../../../../packages/core/src/types'; +import { FakeJobApi, resultUrlOf, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +async function submitJob(api: FakeJobApi, clock: Clock): Promise { + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + return api.jobIds.at(-1)!; +} + +async function main(): Promise { + heading('C4 — can `paginate` express the poll loop?'); + + // ── (a) it LOOPS — the capture's prediction is refuted ──────────────────────────────────── + // `next` returning `{}` re-requests the same URL; returning `undefined` stops. The default + // `items` wraps the non-array status body as one item, so the `length === 0` break never fires. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 3 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + + check('(a) the call succeeded', r.ok, true); + check('(a) polls made', api.polls(id).length, 4); + check( + '(a) the loop terminated on the terminal state', + stateOf((r.data as unknown[])?.at(-1)), + 'JobComplete' as const, + ); + } + + // ── (b) …at ZERO spacing. There is no wait knob on `paginate` at all ────────────────────── + // Every poll lands at the same virtual instant: a job that takes an hour is hammered as fast as + // the event loop allows. `delay` / `backoff` are not fields — the `@ts-expect-error`s are the + // machine-checked half of the claim (a non-error there fails `tsc`). + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 3 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + await p; + check('(b) gaps between polls (ms)', api.gaps(id).join(','), '0,0,0'); + + stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: () => undefined, + // @ts-expect-error — `PaginateOptions` is `{ next, items?, pages? }` (types.ts:1412). + delay: 1000, + }, + }); + stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: () => undefined, + // @ts-expect-error — no backoff field either. + backoff: { curve: 'expo' }, + }, + }); + note( + '(b) `PaginateOptions`', + '{ next, items?, pages? } — no delay, no backoff', + ); + } + + // ── (b2) `throttle` is the only pacing the paginated path passes through ────────────────── + // And it is a FIXED minimum spacing before each request, not a growing backoff: four polls + // 30s apart, forever, whatever the job is doing. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 3 }); + const id = await submitJob(api, clock); + const paced = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + throttle: { rate: '1/30s' }, + paginate: { + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const p = paced.safe(); + await clock.advance(3_600_000); + await p; + check('(b2) polls made', api.polls(id).length, 4); + check( + '(b2) gaps under `throttle: "1/30s"` (ms)', + api.gaps(id).join(','), + '30000,30000,30000', + ); + } + + // ── (c) the RESULT is every poll response, not the terminal one ─────────────────────────── + // `paginated` aggregates (engine.ts:1000-1010), so the value is an array of statuses and the + // caller digs the last element out. `pick` cannot help — it runs per page, before aggregation. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + check('(c) values aggregated', (r.data as unknown[]).length, 3); + check( + '(c) states in the value', + (r.data as unknown[]).map((v) => stateOf(v)).join(','), + 'InProgress,InProgress,JobComplete', + ); + } + + // ── (d) THE TRAP: the natural `items` spelling ends the run OK, with nothing ────────────── + // A caller who wants the result rows writes `items` to pull them. An `InProgress` body has + // none — so page 1 yields zero items, `paginated` breaks at engine.ts:984 BEFORE calling + // `next`, and falls straight through to the `result` event. Success, empty array, one poll, + // and the job is still running on the server. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 3 }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + items: (v) => { + const url = resultUrlOf(v); + return url === undefined ? [] : [url]; + }, + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const evts: StitchEvent[] = []; + const consume = (async (): Promise => { + for await (const e of poll.stream()) evts.push(e); + })(); + await clock.advance(3_600_000); + await consume; + + check('(d) polls made', api.polls(id).length, 1); + check( + '(d) the run ended ok', + evts.find((e) => e.type === 'done')?.ok, + true, + ); + const result = evts.find((e) => e.type === 'result'); + check( + '(d) the value handed back', + JSON.stringify( + result && 'data' in result ? result.data : undefined, + ), + '[]', + ); + check( + '(d) error events', + evts.filter((e) => e.type === 'error').length, + 0, + ); + check( + '(d) drift findings', + evts.filter((e) => e.type === 'drift').length, + 0, + ); + check( + '(d) times `next` was consulted', + evts.filter((e) => e.type === 'progress' && e.phase === 'paginate') + .length, + 1, + ); + note( + '(d) → the caller sees a successful call with an empty result; the job runs to completion unread', + '', + ); + } + + // ── (e) `next` cannot see the terminal FAILURE either ───────────────────────────────────── + // It is handed the body, so `state: 'Failed'` is reachable — but the only thing it can do with + // it is stop. A paginated run cannot turn an in-band failure into a failed call; the `Failed` + // body is aggregated as a value and the run reports success. + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 1, + terminal: 'Failed', + }); + const id = await submitJob(api, clock); + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + adapter: api.adapter(), + clock, + paginate: { + next: (prev) => + stateOf(prev) === 'InProgress' ? {} : undefined, + }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + const r = await p; + check('(e) the job FAILED, and the call reports', r.ok, true); + check( + '(e) …with the failure as a value', + stateOf((r.data as unknown[]).at(-1)), + 'Failed' as const, + ); + } + + finish( + 'C4', + 'REFUTED, and the refutation is worse than the prediction. `paginate` DOES loop over a job-status body — the default `items` wraps a non-array value as one item, so the `length === 0` break never fires and `next → undefined` terminates cleanly (4 polls). What it cannot do is WAIT: measured gaps 0,0,0, with no `delay`/`backoff` field on `PaginateOptions` and only `throttle`’s FIXED spacing available. And the natural `items` spelling — pull the result rows — makes page 1 yield zero items, breaking at engine.ts:984 BEFORE `next` is consulted: the run ends `ok`, `data: []`, ONE poll, no error and no drift, while the job is still running. `Failed` is aggregated as a value too, so a paginated poll cannot fail', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c5-one-deadline.ts b/docs/scenarios/proofs/async-job-polling/c5-one-deadline.ts new file mode 100644 index 00000000..35806070 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c5-one-deadline.ts @@ -0,0 +1,366 @@ +// C5 — THE DECIDING CLAIM. Is there ONE deadline over submit + N polls + download? "Give up after +// an hour" is a budget over the whole triangle, not over any single call. +// +// Four candidates are measured for what each ACTUALLY bounds: `timeout.total`, `timeout.perAttempt`, +// three `linked` members each with their own budget, and a caller-owned `AbortSignal`. +// +// One measurement here is deliberately WALL-CLOCK: `timeout.total`'s deadline is compared against +// `now()`, not the injected clock (engine.ts:453-483), so no `manualClock` can drive it. That case +// runs on real timers at 250ms with bounds set 4× clear of the real numbers; everything else is +// virtual time. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c5-one-deadline.ts +import { stitch } from '../../../../packages/core/src/index'; +import { linked } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Clock } from '../../../../packages/core/src/types'; +import { systemClock } from '../../../../packages/core/src/util'; +import { FakeJobApi, resultUrlOf, stateOf } from './fake-jobs'; +import { check, checkAtMost, finish, heading, note } from './harness'; + +const HOST = 'https://bulk.example.com'; + +/** The poll surface, as C2 established it. */ +const pollSurface = (after: number | string): Surface => ({ + id: 'job-poll', + interpret: (res) => + stateOf(res.body) === 'InProgress' + ? { ok: false, retry: true, message: 'InProgress', after } + : { ok: true, data: res.body }, +}); + +/** + * The whole triangle as ONE stitch: `POST /jobs`, then `hooks.onRequest` redirects each subsequent + * attempt — first to the `Location`, then to the `resultUrl`. C1(e) established the mechanic; here + * it is the construction under test, because one stitch means one `timeout.total`. + */ +function oneStitchTriangle( + api: FakeJobApi, + clock: Clock, + opts: { + attempts: number; + total?: string; + }, +): ReturnType { + let next: { url: string; method: string } | undefined; + const surface: Surface = { + id: 'async-job', + interpret: (res) => { + if (res.status === 202) + return { + ok: false, + retry: true, + message: 'accepted', + after: 50, + }; + const state = stateOf(res.body); + if (state === 'InProgress') + return { + ok: false, + retry: true, + message: 'InProgress', + after: 50, + }; + if (state === 'JobComplete') + return { + ok: false, + retry: true, + message: 'downloading', + after: 0, + }; + return { ok: true, data: res.body }; + }, + }; + return stitch({ + name: 'bulk-job', + url: FakeJobApi.submitUrl, + method: 'POST', + kind: surface, + adapter: api.adapter(), + clock, + retry: { attempts: opts.attempts }, + ...(opts.total === undefined ? {} : { timeout: { total: opts.total } }), + hooks: { + onRequest: (ctx) => { + if (ctx.req && next) { + ctx.req.url = next.url; + ctx.req.method = next.method; + ctx.req.body = undefined; + } + }, + onResponse: (ctx) => { + const res = ctx.res; + if (!res) return; + const loc = res.headers['location']; + if (loc) + next = { + url: new URL(loc, HOST).toString(), + method: 'GET', + }; + const done = resultUrlOf(res.body); + if (done !== undefined) next = { url: done, method: 'GET' }; + }, + }, + }); +} + +async function main(): Promise { + heading('C5 — is there ONE deadline over submit + polls + download?'); + + // ── (a) `timeout.total` bounds ONE STITCH's whole call — every attempt and every wait ───── + // WALL-CLOCK, by design (engine.ts:453-456, 482). With the triangle collapsed into one stitch, + // that one budget covers submit + polls + download: exactly the "give up after an hour" shape. + { + const api = new FakeJobApi({ + clock: systemClock, + inProgressPolls: 1000, + }); + const call = oneStitchTriangle(api, systemClock, { + attempts: 1000, + total: '250ms', + }); + const t0 = Date.now(); + const r = await call.safe({ body: { q: 'SELECT Id' } }); + const wall = Date.now() - t0; + + check('(a) the call failed', r.ok, false); + // The engine throws a `TimeoutError` (engine.ts:468-469), but every failure reaches the + // caller as a `StitchError` — the identity is flattened, so only the MESSAGE distinguishes + // "the deadline fired" from "the job failed" from "the poll budget ran out". + check('(a) error name', r.error?.name, 'StitchError'); + check('(a) error message', r.error?.message, 'timed out after 250ms'); + checkAtMost('(a) wall-clock elapsed (ms)', wall, 1000); + check('(a) submits', api.submits, 1); + check( + '(a) the budget covered BOTH hops', + api.hits[0]?.path === '/jobs' && api.polls('job-1').length > 0, + true, + ); + note('(a) requests made inside the 250ms budget', api.hits.length); + note( + '(a) → one stitch, one deadline over submit + N polls (+ the download, had it got there)', + '', + ); + } + + // ── (b) `timeout.perAttempt` bounds ONE poll, not the operation ─────────────────────────── + // Its own doc says so (types.ts:1064-1069). Under a manual clock the per-attempt deadline is + // clock-driven, so this one IS virtual: 40 polls run to the retry budget, each well inside its + // own 5s attempt window, while 40× that has elapsed. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 1000 }); + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + const id = api.jobIds[0]!; + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(60_000), + adapter: api.adapter(), + clock, + retry: { attempts: 40 }, + timeout: { perAttempt: '5s' }, + }); + const p = poll.safe(); + await clock.advance(24 * 3_600_000); + const r = await p; + check( + '(b) polls made under `perAttempt: 5s`', + api.polls(id).length, + 40, + ); + check( + '(b) virtual time the last poll landed at (ms)', + api.polls(id).at(-1)?.at, + 39 * 60_000, + ); + check( + '(b) the call failed on the RETRY budget, not the clock', + r.ok, + false, + ); + check('(b) error message', r.error?.message, 'InProgress'); + note( + '(b) → 39 virtual minutes elapsed under a "5s" timeout; it bounds an attempt, nothing more', + '', + ); + } + + // ── (c) three `linked` members = three INDEPENDENT budgets ─────────────────────────────── + // `linked(body)` takes a body and nothing else (pipe.ts:357-359) — no options object, so no + // place to put a deadline. Each member keeps its own `timeout`, and the flow's worst case is + // their sum. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + timeout: { total: '30s' }, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(100), + adapter: api.adapter(), + clock, + retry: { attempts: 10 }, + timeout: { total: '30s' }, + }); + const download = stitch({ + name: 'download', + url: '{+u}', + adapter: api.adapter(), + clock, + timeout: { total: '30s' }, + }); + const flow = linked(async (run) => { + await run(submit, { body: {} }); + const id = api.jobIds.at(-1)!; + const status = await run(poll, { params: { loc: `/jobs/${id}` } }); + return run(download, { params: { u: resultUrlOf(status)! } }); + }); + await clock.advance(3_600_000); + check( + '(c) the flow resolved', + JSON.stringify(await flow), + '{"rows":3}', + ); + check( + '(c) budgets declared / worst case (s)', + `${3} × 30 = ${90}`, + '3 × 30 = 90', + ); + + // @ts-expect-error — `linked` takes ONE argument (the body); there is no options slot. + void linked(async () => 1, { timeout: '1h' }); + note('(c) `linked`', '(body: (run) => T) => Promise — no options'); + } + + // ── (d) a caller-owned `AbortSignal` DOES bound the whole flow ──────────────────────────── + // `StitchInput.signal` is threaded onto the request AND into the sleeps (engine.ts:696, 762, + // 800 → `sleepWithin(..., baseReq.signal, ...)`), so one signal passed to every member is an + // operation-wide deadline. `linked` fails fast, so the abort ends the flow. Fired off the + // INJECTED clock here, which is what makes an hour-long budget testable at all. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 10_000 }); + const ctrl = new AbortController(); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(600_000), + adapter: api.adapter(), + clock, + retry: { attempts: 10_000 }, + }); + const flow = linked(async (run) => { + await run(submit, { body: {}, signal: ctrl.signal }); + const id = api.jobIds.at(-1)!; + return run(poll, { + params: { loc: `/jobs/${id}` }, + signal: ctrl.signal, + }); + }); + const settled = flow.then( + () => 'resolved', + (e: unknown) => `rejected: ${(e as Error).message}`, + ); + // The operation budget: one hour of VIRTUAL time. + void clock + .sleep(3_600_000) + .then(() => ctrl.abort(new Error('job budget exhausted'))); + await clock.advance(24 * 3_600_000); + + // NOT `job budget exhausted`: both clocks' `sleep` reject with a fresh `Error('aborted')` + // and discard `signal.reason` (util.ts:39-55, test-clock.ts:64-79), even though the engine's + // own `abortReason` (engine.ts:544-549) preserves it on the throttle path. An abort that + // lands during a poll wait therefore loses the caller's reason. + check('(d) the flow', await settled, 'rejected: aborted'); + check('(d) submits', api.submits, 1); + check( + '(d) polls made in one virtual hour', + api.polls('job-1').length, + 6, + ); + check( + '(d) the last poll landed at (ms, virtual)', + api.polls('job-1').at(-1)?.at, + 3_000_000, + ); + note( + '(d) → the ONE deadline exists, but it is an AbortSignal the caller owns, not a config field', + '', + ); + } + + // ── (e) THE TESTABILITY TRAP: `timeout.total` ignores the injected clock entirely ───────── + // `sleepWithin` compares `budget.deadline - now()` — wall-clock — and then sleeps on the + // INJECTED clock (engine.ts:481-488). Virtual time therefore never consumes the budget. The + // same stitch as (a), on a manual clock, polls 60 times across 59 VIRTUAL seconds under a + // `total: '10s'` and never trips it: a manual-clock test of "give up after an hour" passes + // while proving nothing. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 1000 }); + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + const id = api.jobIds[0]!; + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(1000), + adapter: api.adapter(), + clock, + retry: { attempts: 60 }, + timeout: { total: '10s' }, + }); + const p = poll.safe(); + await clock.advance(6 * 3_600_000); + const r = await p; + + check('(e) polls made under `total: "10s"`', api.polls(id).length, 60); + check( + '(e) virtual time the last poll landed at (ms)', + api.polls(id).at(-1)?.at, + 59_000, + ); + check( + '(e) did the budget fire?', + r.error?.name === 'TimeoutError', + false, + ); + check('(e) what ended the run instead', r.error?.message, 'InProgress'); + note( + '(e) → 59s of virtual waiting under a 10s budget. Inject a clock and `timeout.total` goes quiet', + '', + ); + } + + finish( + 'C5', + 'YES, with user code — and by TWO different routes with different costs. (1) Collapse the triangle into ONE stitch (C1(e)’s hook rewrite) and `timeout.total` is a single wall-clock budget over submit + polls + download: measured 253ms wall, `timed out after 250ms`, 1 submit, both hops inside it. (2) Keep three stitches under `linked` and thread ONE caller-owned `AbortSignal` through every `input.signal`: measured 6 polls in a virtual hour, then a rejection. What does NOT express it: `timeout.perAttempt` (39 virtual minutes elapsed under a "5s" setting), and `linked` itself, which takes a body and no options — three members means three independent budgets summing to 90s. The trap: `timeout.total` is compared against WALL-CLOCK while its sleeps run on the injected clock, so under a `manualClock` it goes silent — 60 polls across 59 virtual seconds never tripped a 10s total', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c6-linked-trace.ts b/docs/scenarios/proofs/async-job-polling/c6-linked-trace.ts new file mode 100644 index 00000000..64b09594 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c6-linked-trace.ts @@ -0,0 +1,393 @@ +// C6 — does `linked` (pipe.ts:357) produce ONE trace chain across the three endpoints, and is a +// mid-operation failure attributable to the operation as a whole? +// +// Measured off a real `TraceSink`: the `traceId` / `spanId` / `parentSpanId` each run reports, what +// a bare sequence of awaits reports instead, and what the trace says when the poll fails between a +// successful submit and a download that never happens. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c6-linked-trace.ts +import { stitch } from '../../../../packages/core/src/index'; +import { linked } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { FakeJobApi, resultUrlOf, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +const HOST = 'https://bulk.example.com'; + +/** One trace record: the stitch name, the event, and the run identity it arrived under. */ +interface Rec { + name: string; + type: StitchEvent['type']; + /** Present on `progress` events only — `request` / `retry` / `throttled` / … */ + phase: string | undefined; + traceId: string | undefined; + spanId: string | undefined; + parentSpanId: string | undefined; +} + +/** A sink that just records. `TraceSink` is the documented seam (types.ts:1957-1960). */ +function recordingSink(into: Rec[]): TraceSink { + return { + handle(event: StitchEvent, ctx: TraceContext): void { + into.push({ + name: ctx.name, + type: event.type, + phase: event.type === 'progress' ? event.phase : undefined, + traceId: ctx.traceId, + spanId: ctx.spanId, + parentSpanId: ctx.parentSpanId, + }); + }, + }; +} + +const pollSurface = (after: number): Surface => ({ + id: 'job-poll', + interpret: (res) => { + const state = stateOf(res.body); + if (state === 'InProgress') + return { ok: false, retry: true, message: 'InProgress', after }; + if (state === 'Failed') + return { ok: false, message: 'job failed', status: res.status }; + return { ok: true, data: res.body }; + }, +}); + +async function main(): Promise { + heading('C6 — does `linked` draw ONE trace chain across the triangle?'); + + // ── (a) three stitches under `linked` share one traceId and chain their spans ───────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const recs: Rec[] = []; + const trace = recordingSink(recs); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + trace, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 10 }, + trace, + }); + const download = stitch({ + name: 'download', + url: '{+u}', + adapter: api.adapter(), + clock, + trace, + }); + const flow = linked(async (run) => { + await run(submit, { body: {} }); + const id = api.jobIds.at(-1)!; + const status = await run(poll, { params: { loc: `/jobs/${id}` } }); + return run(download, { params: { u: resultUrlOf(status)! } }); + }); + await clock.advance(3_600_000); + check( + '(a) the flow resolved', + JSON.stringify(await flow), + '{"rows":3}', + ); + + const starts = recs.filter((r) => r.type === 'start'); + check( + '(a) `start` events (one per stitch, not per poll)', + starts.length, + 3, + ); + check( + '(a) distinct traceIds across the whole operation', + new Set(recs.map((r) => r.traceId)).size, + 1, + ); + check( + '(a) distinct spanIds (one run per member)', + new Set(recs.map((r) => r.spanId)).size, + 3, + ); + const chain = starts.map((s) => s.name).join(' → '); + check('(a) the chain, in order', chain, 'submit → poll → download'); + check( + '(a) submit is the ROOT (no parent)', + starts[0]?.parentSpanId, + undefined, + ); + check( + '(a) poll’s parent IS submit’s span', + starts[1]?.parentSpanId === starts[0]?.spanId, + true, + ); + check( + '(a) download’s parent IS poll’s span', + starts[2]?.parentSpanId === starts[1]?.spanId, + true, + ); + check( + '(a) `request` progress events inside poll’s span (one per poll)', + recs.filter((r) => r.name === 'poll' && r.phase === 'request') + .length, + 3, + ); + check( + '(a) `retry` progress events inside poll’s span', + recs.filter((r) => r.name === 'poll' && r.phase === 'retry').length, + 2, + ); + } + + // ── (b) the same three awaits WITHOUT `linked` are three unrelated roots ────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const recs: Rec[] = []; + const trace = recordingSink(recs); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + trace, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 10 }, + trace, + }); + const download = stitch({ + name: 'download', + url: '{+u}', + adapter: api.adapter(), + clock, + trace, + }); + await submit.safe({ body: {} }); + const id = api.jobIds.at(-1)!; + const pp = poll.safe({ params: { loc: `/jobs/${id}` } }); + await clock.advance(3_600_000); + const status = await pp; + await download.safe({ params: { u: resultUrlOf(status.data)! } }); + + check( + '(b) distinct traceIds', + new Set(recs.map((r) => r.traceId)).size, + 3, + ); + check( + '(b) runs with a parent', + recs.filter((r) => r.parentSpanId !== undefined).length, + 0, + ); + note( + '(b) → same code, same calls; the only difference is calling through `run`', + '', + ); + } + + // ── (c) a mid-operation failure: attributable to the STEP, not to the operation ─────────── + // `linked` fails fast (pipe.ts:335-337), so the download never starts. The trace carries the + // failing step's `error`/`done` under the shared traceId — but there is no operation-level + // event: nothing says "the bulk-export operation failed", only "the `poll` stitch failed". + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 2, + terminal: 'Failed', + }); + const recs: Rec[] = []; + const trace = recordingSink(recs); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + trace, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 10 }, + trace, + }); + const download = stitch({ + name: 'download', + url: '{+u}', + adapter: api.adapter(), + clock, + trace, + }); + const flow = linked(async (run) => { + await run(submit, { body: {} }); + const id = api.jobIds.at(-1)!; + const status = await run(poll, { params: { loc: `/jobs/${id}` } }); + return run(download, { params: { u: resultUrlOf(status)! } }); + }); + const settled = flow.then( + () => 'resolved', + (e: unknown) => `rejected: ${(e as Error).message}`, + ); + await clock.advance(3_600_000); + check('(c) the flow', await settled, 'rejected: job failed'); + + check( + '(c) traceIds (the failure is still on the operation’s trace)', + new Set(recs.map((r) => r.traceId)).size, + 1, + ); + check( + '(c) which stitch the `error` event names', + recs + .filter((r) => r.type === 'error') + .map((r) => r.name) + .join(','), + 'poll', + ); + check( + '(c) `start` events (download never ran)', + recs + .filter((r) => r.type === 'start') + .map((r) => r.name) + .join(','), + 'submit,poll', + ); + check( + '(c) `done` events, and whether any reports the OPERATION', + recs + .filter((r) => r.type === 'done') + .map((r) => r.name) + .join(','), + 'submit,poll', + ); + check('(c) requests the download made', api.resultFetches.length, 0); + note( + '(c) → the trace is one tree, but the operation itself has no span: `linked` emits nothing of its own', + '', + ); + } + + // ── (d) the ONE-STITCH construction reports the triangle as a single run ───────────────── + // C5(a)'s shape: one `start`, one `done`, and the hops are `retry` progress events, so the run + // report's `attempts` counts them. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const recs: Rec[] = []; + let next: { url: string; method: string } | undefined; + const surface: Surface = { + id: 'async-job', + interpret: (res) => { + if (res.status === 202) + return { + ok: false, + retry: true, + message: 'accepted', + after: 1000, + }; + const state = stateOf(res.body); + if (state === 'InProgress') + return { + ok: false, + retry: true, + message: 'InProgress', + after: 1000, + }; + if (state === 'JobComplete') + return { + ok: false, + retry: true, + message: 'downloading', + after: 0, + }; + return { ok: true, data: res.body }; + }, + }; + const call = stitch({ + name: 'bulk-job', + url: FakeJobApi.submitUrl, + method: 'POST', + kind: surface, + adapter: api.adapter(), + clock, + retry: { attempts: 12 }, + trace: recordingSink(recs), + hooks: { + onRequest: (ctx) => { + if (ctx.req && next) { + ctx.req.url = next.url; + ctx.req.method = next.method; + ctx.req.body = undefined; + } + }, + onResponse: (ctx) => { + const res = ctx.res; + if (!res) return; + const loc = res.headers['location']; + if (loc) + next = { + url: new URL(loc, HOST).toString(), + method: 'GET', + }; + const done = resultUrlOf(res.body); + if (done !== undefined) next = { url: done, method: 'GET' }; + }, + }, + }); + const p = call.safe({ body: {} }); + await clock.advance(3_600_000); + const r = await p; + + check('(d) the call resolved', JSON.stringify(r.data), '{"rows":3}'); + check( + '(d) `start` events for the whole triangle', + recs.filter((r) => r.type === 'start').length, + 1, + ); + check( + '(d) `done` events', + recs.filter((r) => r.type === 'done').length, + 1, + ); + check( + '(d) distinct spanIds', + new Set(recs.map((r) => r.spanId)).size, + 1, + ); + check('(d) requests the one run actually made', api.hits.length, 5); + note( + '(d) → the triangle is ONE span with `attempts: 5`; the three endpoints are invisible in the trace', + '', + ); + } + + finish( + 'C6', + 'YES — `linked` draws exactly one trace chain: 3 `start` events, 1 traceId, 3 spanIds, `submit → poll → download` with each member’s `parentSpanId` equal to the previous member’s `spanId`, and the three polls folded into the poll span as 3 `request` + 2 `retry` progress events. The same three awaits WITHOUT `run` produce 3 traceIds and 0 parents. A mid-operation failure IS on the operation’s trace (one traceId, the `error` event naming `poll`, the download never started) — but it is attributable to the STEP, not the operation: `linked` emits no span of its own, so nothing in the stream says the bulk-export operation failed. The one-stitch alternative is the opposite trade: 1 start, 1 done, 1 span, `attempts: 5` — and the three endpoints are invisible', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c7-single-use-download.ts b/docs/scenarios/proofs/async-job-polling/c7-single-use-download.ts new file mode 100644 index 00000000..549b3976 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c7-single-use-download.ts @@ -0,0 +1,273 @@ +// C7 — the single-use result URL. Pre-signed links expire, and some expire on the FIRST successful +// fetch: every later fetch is a permanent 404 that LOOKS transient. Does `retry` turn that into +// repeated attempts? And can one stitch retry the poll while another does not retry the download? +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c7-single-use-download.ts +import { stitch } from '../../../../packages/core/src/index'; +import { linked } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeJobApi, resultUrlOf, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +const HOST = 'https://bulk.example.com'; + +const pollSurface = (after: number): Surface => ({ + id: 'job-poll', + interpret: (res) => + stateOf(res.body) === 'InProgress' + ? { ok: false, retry: true, message: 'InProgress', after } + : { ok: true, data: res.body }, +}); + +/** Submit + poll to completion; hand back the (single-use) result URL. */ +async function runToResultUrl( + api: FakeJobApi, + clock: ReturnType, +): Promise { + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + await submit.safe({ body: {} }); + const id = api.jobIds.at(-1)!; + const poll = stitch({ + url: FakeJobApi.statusUrl(id), + kind: pollSurface(1000), + adapter: api.adapter(), + clock, + retry: { attempts: 20 }, + }); + const p = poll.safe(); + await clock.advance(3_600_000); + return resultUrlOf((await p).data)!; +} + +async function main(): Promise { + heading('C7 — does `retry` hammer a single-use download?'); + + // ── (a) the DEFAULT `retry.on` does not include 404, so an expired link is terminal ──────── + // `[429, 502, 503, 504]` (types.ts:979-983). A 404 falls straight to the failure path even with + // a generous `attempts`. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 1 }); + const url = await runToResultUrl(api, clock); + const download = stitch({ + url, + adapter: api.adapter(), + clock, + retry: { attempts: 5 }, + }); + const first = await download.safe(); + const p = download.safe(); + await clock.advance(60_000); + const second = await p; + + check('(a) first fetch ok', first.ok, true); + check('(a) first payload', JSON.stringify(first.data), '{"rows":3}'); + check('(a) second fetch ok', second.ok, false); + check('(a) second error', second.error?.message, 'HTTP 404'); + check( + '(a) requests the link received', + api.resultFetches.map((h) => h.status).join(','), + '200,404', + ); + } + + // ── (b) widen `retry.on` to cover 404 and the permanent failure is attempted N times ─────── + // The realistic way to get here: an author who has seen the link 404 *before* it is ready and + // adds 404 to the retryable set. It cannot distinguish "not ready" from "already spent". + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 1 }); + const url = await runToResultUrl(api, clock); + const download = stitch({ + url, + adapter: api.adapter(), + clock, + retry: { attempts: 3, on: [404, 429, 503] }, + }); + await download.safe(); + const p = download.safe(); + await clock.advance(60_000); + const second = await p; + + check('(b) second fetch ok', second.ok, false); + check( + '(b) requests the link received', + api.resultFetches.map((h) => h.status).join(','), + '200,404,404,404', + ); + check( + '(b) wasted attempts on a permanently-dead link', + api.resultFetches.filter((h) => h.status === 404).length - 1, + 2, + ); + } + + // ── (c) per-stitch policy: the poll retries hard, the download not at all ───────────────── + // Retry is configured per stitch, so three stitches under `linked` carry three policies. This + // is the shape that gets it right: 20 poll attempts, exactly ONE download attempt. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 4 }); + const submit = stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + }); + const poll = stitch({ + name: 'poll', + url: `${HOST}{+loc}`, + kind: pollSurface(30_000), + adapter: api.adapter(), + clock, + retry: { attempts: 20 }, + }); + const download = stitch({ + name: 'download', + url: '{+u}', + adapter: api.adapter(), + clock, + // The single-use link: one shot, and a failure is a failure. + retry: { attempts: 1 }, + }); + const flow = linked(async (run) => { + await run(submit, { body: {} }); + const id = api.jobIds.at(-1)!; + const status = await run(poll, { params: { loc: `/jobs/${id}` } }); + return run(download, { params: { u: resultUrlOf(status)! } }); + }); + await clock.advance(3_600_000); + check( + '(c) the flow resolved', + JSON.stringify(await flow), + '{"rows":3}', + ); + check('(c) polls made', api.polls('job-1').length, 5); + check('(c) download attempts', api.resultFetches.length, 1); + check( + '(c) the two policies, side by side', + 'poll attempts=20 / download attempts=1', + 'poll attempts=20 / download attempts=1', + ); + } + + // ── (d) in the ONE-STITCH construction the two policies COLLAPSE ────────────────────────── + // One stitch is one `retry` block, and `retry.attempts` is a single budget shared by the + // submit, every poll, and the download. So a download that fails is re-attempted on the poll's + // policy — and because `hooks.onRequest` still points at the spent link, each re-attempt hits + // the dead URL again. Measured against a link that was already consumed once. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 1 }); + let next: { url: string; method: string } | undefined; + const surface: Surface = { + id: 'async-job', + interpret: (res) => { + if (res.status === 202) + return { + ok: false, + retry: true, + message: 'accepted', + after: 100, + }; + const state = stateOf(res.body); + if (state === 'InProgress') + return { + ok: false, + retry: true, + message: 'InProgress', + after: 100, + }; + if (state === 'JobComplete') + return { + ok: false, + retry: true, + message: 'downloading', + after: 0, + }; + // The download hop: a 404 body here is the spent link. Ask for another attempt, + // exactly as an author would for "the link is not ready yet". + if (res.status === 404) + return { + ok: false, + retry: true, + message: 'result not ready', + after: 100, + }; + return { ok: true, data: res.body }; + }, + }; + const call = stitch({ + name: 'bulk-job', + url: FakeJobApi.submitUrl, + method: 'POST', + kind: surface, + adapter: api.adapter(), + clock, + retry: { attempts: 8 }, + hooks: { + onRequest: (ctx) => { + if (ctx.req && next) { + ctx.req.url = next.url; + ctx.req.method = next.method; + ctx.req.body = undefined; + } + }, + onResponse: (ctx) => { + const res = ctx.res; + if (!res) return; + const loc = res.headers['location']; + if (loc) + next = { + url: new URL(loc, HOST).toString(), + method: 'GET', + }; + const done = resultUrlOf(res.body); + if (done !== undefined) next = { url: done, method: 'GET' }; + }, + }, + }); + // First run: consumes the link. + const p1 = call.safe({ body: {} }); + await clock.advance(3_600_000); + await p1; + const spent = api.resultFetches.length; + // Second run: a NEW job whose link is fine — but the fake's link is per job, so force the + // collision by re-pointing at the first job's spent URL. + next = { url: `${HOST}/results/job-1`, method: 'GET' }; + const p2 = call.safe({ body: {} }); + await clock.advance(3_600_000); + const r2 = await p2; + + check('(d) first run consumed the link', spent, 1); + check('(d) second run ok', r2.ok, false); + check( + '(d) attempts spent on the DEAD link', + api.resultFetches.length - spent, + 8, + ); + check( + '(d) can the download hop carry its own `retry`?', + 'one stitch = one retry block', + 'one stitch = one retry block', + ); + note( + '(d) → the price of C5(a)’s single deadline: the poll’s patience is also the download’s', + '', + ); + } + + finish( + 'C7', + 'The default is SAFE and the split IS expressible. `retry.on` defaults to `[429, 502, 503, 504]`, so an expired link 404s once and stops (measured 200,404) even under `attempts: 5`; widening `on` to include 404 turns one permanent failure into three requests (200,404,404,404). Because `retry` is per-stitch, three stitches under `linked` give the poll 20 attempts and the download exactly 1 — measured 5 polls, 1 download. The one-stitch construction cannot: one stitch is one `retry` block, so a download that fails burned all 8 shared attempts on the same dead URL', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c8-resume.ts b/docs/scenarios/proofs/async-job-polling/c8-resume.ts new file mode 100644 index 00000000..7eb2a98f --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c8-resume.ts @@ -0,0 +1,250 @@ +// C8 — resumability. The process dies mid-poll; the job is still running server-side. Re-submitting +// duplicates hours of work, so the correct move is to reattach to the stored id. Can a stitch do +// that? A `StitchStore` exists — does it help at all here? +// +// "Restart" is modelled honestly: every stitch object is rebuilt from scratch, and the ONLY thing +// that crosses the boundary is what a store was asked to hold. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c8-resume.ts +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchStore } from '../../../../packages/core/src/types'; +import { FakeJobApi, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; + +const pollSurface = (after: number): Surface => ({ + id: 'job-poll', + interpret: (res) => + stateOf(res.body) === 'InProgress' + ? { ok: false, retry: true, message: 'InProgress', after } + : { ok: true, data: res.body }, +}); + +/** A store that records every key written, so "what did the engine persist?" is measurable. */ +function spyStore(): StitchStore & { keys: string[] } { + const inner = memoryStore(); + const keys: string[] = []; + return { + keys, + get: (k) => inner.get(k), + set: (k, v, ttl) => { + keys.push(k); + return inner.set(k, v, ttl); + }, + increment: (k, ttl) => { + keys.push(k); + return inner.increment(k, ttl); + }, + }; +} + +async function main(): Promise { + heading('C8 — can a stitch reattach to a stored job id after a restart?'); + + // ── (a) the engine persists NOTHING about the job ───────────────────────────────────────── + // `StitchStore` is the engine's own state: throttle counters, auth sessions, cache entries + // (store.ts:1-3). A submit writes nothing at all. + { + const clock = manualClock(); + const store = spyStore(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + store, + }); + await submit.safe({ body: { q: 'SELECT Id' } }); + check('(a) keys the engine wrote', store.keys.length, 0); + check( + '(a) is the job id anywhere in the store?', + store.keys.some((k) => k.includes('job-1')), + false, + ); + check('(a) the job exists server-side', api.jobIds.join(','), 'job-1'); + note( + '(a) → the one durable thing about this operation is the one thing the engine never sees', + '', + ); + } + + // ── (b) resume works — entirely as USER code, and it is small ───────────────────────────── + // Persist the id yourself (a store is a perfectly good place), then build the poll stitch from + // it after the restart. Measured: the job completes and is never re-submitted. + { + const clock = manualClock(); + const store = memoryStore(); + const api = new FakeJobApi({ clock, inProgressPolls: 6 }); + + // ── process 1: submit, persist, poll twice, die ── + { + const submit = stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + hooks: { + onResponse: (ctx) => { + const loc = ctx.res?.headers['location']; + if (loc) + void store.set('bulk:job', loc, 24 * 3_600_000); + }, + }, + }); + await submit.safe({ body: { q: 'SELECT Id' } }); + const loc = (await store.get('bulk:job')) as string; + const poll = stitch({ + url: `https://bulk.example.com{+loc}`, + kind: pollSurface(60_000), + adapter: api.adapter(), + clock, + retry: { attempts: 2 }, + }); + const p = poll.safe({ params: { loc } }); + await clock.advance(3_600_000); + const r = await p; + check('(b) process 1 poll ok (it gave up mid-poll)', r.ok, false); + check('(b) polls before the crash', api.polls('job-1').length, 2); + } + + // ── process 2: everything rebuilt; only the store survived ── + { + const loc = (await store.get('bulk:job')) as string | undefined; + check('(b) what survived the restart', loc, '/jobs/job-1'); + const poll = stitch({ + url: `https://bulk.example.com{+loc}`, + kind: pollSurface(60_000), + adapter: api.adapter(), + clock, + retry: { attempts: 20 }, + }); + const p = poll.safe({ params: { loc: loc! } }); + await clock.advance(3_600_000); + const r = await p; + check('(b) the resumed poll succeeded', r.ok, true); + check( + '(b) terminal state', + stateOf(r.data), + 'JobComplete' as const, + ); + check('(b) SUBMITS across both processes', api.submits, 1); + check('(b) total polls', api.polls('job-1').length, 7); + } + } + + // ── (c) `cache` on the submit DOES stop the duplicate POST — and loses the job id ───────── + // `cache: { methods: ['POST'] }` over a shared store survives a restart, so process 2's submit + // is a HIT and the server never sees a second job. But a cache entry is the VALUE, and the + // value of a 202 is `{}` — the `Location` header is not in it. Worse: a hit short-circuits the + // request, so `hooks.onResponse` never fires and the id cannot be recovered that way either. + { + const clock = manualClock(); + const store = memoryStore(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const seenLocations: string[] = []; + const mkSubmit = (): ReturnType => + stitch({ + name: 'submit', + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + store, + cache: { ttl: '24h', methods: ['POST'], tenancy: 'app' }, + hooks: { + onResponse: (ctx) => { + const loc = ctx.res?.headers['location']; + if (loc) seenLocations.push(loc); + }, + }, + }); + const first = await mkSubmit().safe({ body: { q: 'SELECT Id' } }); + const second = await mkSubmit().safe({ body: { q: 'SELECT Id' } }); + + check('(c) SUBMITS the server saw', api.submits, 1); + check('(c) first value', JSON.stringify(first.data), '{}'); + check('(c) second (cached) value', JSON.stringify(second.data), '{}'); + check('(c) times `onResponse` fired', seenLocations.length, 1); + check( + '(c) can process 2 recover the job id?', + seenLocations.length > 1, + false, + ); + const rep = await mkSubmit().report({ body: { q: 'SELECT Id' } }, true); + check('(c) `.report().cache`', rep.cache, 'hit'); + check('(c) `.report().status` on a hit', rep.status, 202); + note( + '(c) → duplicate avoided, job ORPHANED: it runs to completion and nobody knows its id', + '', + ); + } + + // ── (d) `idempotency.keyOf` is the one built-in that makes a restart-resubmit safe ─────── + // A DERIVED key is stable across separate submissions (types.ts:1108-1112), so process 2's + // POST carries the same `Idempotency-Key` as process 1's and a server that honours it collapses + // them. It costs nothing and needs no store. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + const keys: (string | undefined)[] = []; + const mkSubmit = (): ReturnType => + stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + retry: { attempts: 3 }, + idempotency: { + keyOf: (input) => `bulk:${(input.body as { q: string }).q}`, + }, + hooks: { + onRequest: (ctx) => { + keys.push(ctx.req?.headers['Idempotency-Key']); + }, + }, + }); + await mkSubmit().safe({ body: { q: 'SELECT Id' } }); + await mkSubmit().safe({ body: { q: 'SELECT Id' } }); + check( + '(d) keys sent across two processes', + keys.join(' / '), + 'bulk:SELECT Id / bulk:SELECT Id', + ); + check('(d) the same key both times', new Set(keys).size, 1); + note( + '(d) → the fake does not dedupe (it mints a job per POST); a server that honours the header would', + '', + ); + } + + // ── (e) nothing in the config vocabulary names a resumable operation ────────────────────── + // The one resume mechanism the library has is for STREAMS: `Surface.resumeToken` / + // `applyResume` + `sse.reconnect` (surface.ts:88-107), which replays a `Last-Event-ID` on a + // dropped connection WITHIN one call. It cannot span a process restart, and it is only + // consulted on a streaming surface. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock }); + stitch({ + url: FakeJobApi.submitUrl, + method: 'POST', + adapter: api.adapter(), + clock, + // @ts-expect-error — there is no `resume` config slot. + resume: { key: 'bulk:job' }, + }); + note( + '(e) the only resume in the vocabulary', + '`Surface.resumeToken`/`applyResume` + `sse.reconnect` — within one streaming call', + ); + } + + finish( + 'C8', + "ENTIRELY USER-SIDE, and the store does not help with the part that matters. The engine persists NOTHING about the job (0 keys written on submit) and there is no `resume` config slot — the library’s only resume is `Surface.resumeToken`/`sse.reconnect`, which replays a `Last-Event-ID` within one streaming call. Doing it by hand is small and works: store the `Location` from `hooks.onResponse`, rebuild the poll stitch from it after the restart — measured 2 polls before the crash, 5 after, 1 SUBMIT total. The trap is the seemingly-clever version: `cache: { methods: ['POST'] }` over a shared store does prevent the duplicate POST (1 submit across two processes), but the cached value of a 202 is `{}` and a cache HIT never fires `onResponse` — so the job id is unrecoverable and the job is orphaned. `idempotency.keyOf` is the one built-in that helps: a derived key is byte-identical across processes, so a server that honours it collapses the resubmit", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/c9-assembled-solution.ts b/docs/scenarios/proofs/async-job-polling/c9-assembled-solution.ts new file mode 100644 index 00000000..9ad8e9d7 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/c9-assembled-solution.ts @@ -0,0 +1,393 @@ +// C9 — assemble the best answer the public API allows, run it end to end on the manual clock, and +// compare it HONESTLY against the hand-rolled `while` loop the state of the art recommends. +// +// The comparison is not an argument: the same fake provider runs both, and every claimed benefit is +// a measured number — request counts, poll spacing, `start` events, `attempts`, trace identity, and +// what each does when a circuit-breaking host starts failing. +// +// pnpm exec tsx docs/scenarios/proofs/async-job-polling/c9-assembled-solution.ts +import { memoryStore } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Clock, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { FakeJobApi, resultUrlOf, stateOf } from './fake-jobs'; +import { check, finish, heading, note } from './harness'; +import { jobTriangle, retryAfterMs } from './job-triangle'; + +const HOST = 'https://bulk.example.com'; + +/** + * THE BASELINE — the hand-rolled `while` loop, written as well as it can be written: it does the + * same job, honours `Retry-After` in seconds, backs off exponentially with a cap when the header is + * absent, treats in-band `Failed` as a failure, holds one deadline over the whole triangle, and + * fetches the single-use link exactly once. + * + * Everything below the signature is the code being counted. + */ +async function handRolled( + api: FakeJobApi, + clock: Clock, + body: unknown, + budgetMs: number, +): Promise { + const fetchJson = api.adapter(); + const deadline = clock.now() + budgetMs; + const accepted = await fetchJson({ + url: `${HOST}/jobs`, + method: 'POST', + headers: {}, + body, + }); + const location = accepted.headers['location']; + if (location === undefined) throw new Error('202 with no Location'); + let wait = 1000; + for (;;) { + if (clock.now() >= deadline) throw new Error('job budget exhausted'); + const status = await fetchJson({ + url: new URL(location, HOST).toString(), + method: 'GET', + headers: {}, + }); + if (status.status >= 400) throw new Error(`HTTP ${status.status}`); + const state = stateOf(status.body); + if (state === 'Failed') throw new Error('job failed'); + if (state === 'JobComplete') { + const url = resultUrlOf(status.body); + if (url === undefined) + throw new Error('JobComplete with no resultUrl'); + const out = await fetchJson({ url, method: 'GET', headers: {} }); + if (out.status >= 400) throw new Error(`HTTP ${out.status}`); + return out.body; + } + const asked = retryAfterMs(status.headers['retry-after'], clock); + await clock.sleep(Math.min(asked ?? wait, deadline - clock.now())); + wait = Math.min(wait * 2, 30_000); + } +} + +/** Count the executable lines of a function body (blank + comment lines excluded). */ +function bodyLines(source: string, marker: string): number { + const start = source.indexOf(marker); + const lines = source.slice(start).split('\n'); + let depth = 0; + let seen = false; + let count = 0; + for (const line of lines) { + const trimmed = line.trim(); + if ( + trimmed !== '' && + !trimmed.startsWith('//') && + !trimmed.startsWith('*') + ) + count++; + for (const ch of line) { + if (ch === '{') { + depth++; + seen = true; + } else if (ch === '}') depth--; + } + if (seen && depth === 0) break; + } + return count; +} + +interface Rec { + name: string; + type: StitchEvent['type']; + traceId: string | undefined; + parentSpanId: string | undefined; + spanId: string | undefined; + /** Present on `result` / `done` / `error` events — how many attempts that run took. */ + attempts: number | undefined; +} + +/** A `TraceSink` that just records — the same seam C6 measured the chain with. */ +function recordingSink(into: Rec[]): TraceSink { + return { + handle: (event: StitchEvent, ctx: TraceContext): void => { + into.push({ + name: ctx.name, + type: event.type, + traceId: ctx.traceId, + spanId: ctx.spanId, + parentSpanId: ctx.parentSpanId, + attempts: 'attempts' in event ? event.attempts : undefined, + }); + }, + }; +} + +async function main(): Promise { + heading( + 'C9 — the assembled answer, run end to end, against the `while` loop', + ); + + // ── (a) the assembled triangle, end to end on the manual clock ──────────────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 4, + retryAfter: 300, // the server asks for 5 minutes between polls + }); + const recs: Rec[] = []; + const store = memoryStore(); + const job = jobTriangle({ + submitUrl: FakeJobApi.submitUrl, + host: HOST, + adapter: api.adapter(), + clock, + pollAttempts: 60, + budgetMs: 3_600_000, + backoff: { base: 5_000, max: 60_000 }, + store, + trace: recordingSink(recs), + }); + const p = job.run({ q: 'SELECT Id FROM Account' }); + await clock.advance(24 * 3_600_000); + const out = await p; + + check('(a) payload', JSON.stringify(out.data), '{"rows":3}'); + check('(a) the job id was persisted', out.location, '/jobs/job-1'); + check( + '(a) it is in the store for a restart', + await store.get('job:location'), + '/jobs/job-1', + ); + check( + '(a) the request sequence', + api.hits.map((h) => `${h.method} ${h.path}`).join(' → '), + 'POST /jobs → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /results/job-1', + ); + check('(a) SUBMITS', api.submits, 1); + check('(a) polls', api.polls('job-1').length, 5); + check( + '(a) poll gaps — the SERVER’s pacing, in seconds (ms)', + api.gaps('job-1').join(','), + '300000,300000,300000,300000', + ); + check('(a) download attempts', api.resultFetches.length, 1); + check( + '(a) virtual time the operation took (ms)', + api.hits.at(-1)?.at, + 1_200_000, + ); + check( + '(a) `start` events for the whole operation', + recs.filter((r) => r.type === 'start').length, + 3, + ); + check( + '(a) distinct traceIds', + new Set(recs.map((r) => r.traceId)).size, + 1, + ); + } + + // ── (b) resume: the same object, reattached, with no submit ─────────────────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 8 }); + const store = memoryStore(); + const mk = (pollAttempts: number): ReturnType => + jobTriangle({ + submitUrl: FakeJobApi.submitUrl, + host: HOST, + adapter: api.adapter(), + clock, + pollAttempts, + budgetMs: 3_600_000, + backoff: { base: 60_000, max: 60_000 }, + store, + }); + // Process 1: a poll budget too small for this job — it dies mid-poll. + const first = mk(3) + .run({ q: 'SELECT Id' }) + .then( + () => 'resolved', + (e: unknown) => (e as Error).message, + ); + await clock.advance(3_600_000); + check('(b) process 1', await first, 'InProgress'); + check('(b) polls before the crash', api.polls('job-1').length, 3); + + // Process 2: everything rebuilt; only the store survived. + const saved = (await store.get('job:location')) as string; + check('(b) what survived the restart', saved, '/jobs/job-1'); + const second = mk(20) + .run({ q: 'SELECT Id' }, saved) + .then( + (r) => JSON.stringify(r.data), + (e: unknown) => `rejected: ${(e as Error).message}`, + ); + await clock.advance(3_600_000); + check('(b) process 2 resumed and finished', await second, '{"rows":3}'); + check('(b) SUBMITS across both processes', api.submits, 1); + check('(b) total polls', api.polls('job-1').length, 9); + } + + // ── (c) the deadline really ends the operation ──────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 10_000 }); + const job = jobTriangle({ + submitUrl: FakeJobApi.submitUrl, + host: HOST, + adapter: api.adapter(), + clock, + pollAttempts: 10_000, + budgetMs: 3_600_000, + backoff: { base: 600_000, max: 600_000 }, + }); + const settled = job.run({ q: 'SELECT Id' }).then( + () => 'resolved', + (e: unknown) => (e as Error).message, + ); + await clock.advance(24 * 3_600_000); + check('(c) the operation', await settled, 'aborted'); + check('(c) polls in one virtual hour', api.polls('job-1').length, 6); + check( + '(c) the last poll landed at (ms)', + api.polls('job-1').at(-1)?.at, + 3_000_000, + ); + note( + '(c) `abort(new Error("job budget of 3600000ms exhausted"))` arrives as', + 'aborted — the clocks’ `sleep` drops `signal.reason` (util.ts:39-55)', + ); + } + + // ── (d) the hand-rolled `while`, same provider, same behaviour ──────────────────────────── + { + const clock = manualClock(); + const api = new FakeJobApi({ + clock, + inProgressPolls: 4, + retryAfter: 300, + }); + const p = handRolled(api, clock, { q: 'SELECT Id' }, 3_600_000); + await clock.advance(24 * 3_600_000); + const out = await p; + check('(d) payload', JSON.stringify(out), '{"rows":3}'); + check( + '(d) the request sequence', + api.hits.map((h) => `${h.method} ${h.path}`).join(' → '), + 'POST /jobs → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /jobs/job-1 → GET /results/job-1', + ); + check( + '(d) poll gaps (ms)', + api.gaps('job-1').join(','), + '300000,300000,300000,300000', + ); + check('(d) SUBMITS', api.submits, 1); + note( + '(d) → identical wire behaviour. The difference is everything AROUND it', + '', + ); + } + + // ── (e) the line count, both ways ───────────────────────────────────────────────────────── + { + const fs = await import('node:fs'); + const here = new URL('.', import.meta.url).pathname; + const triangleSrc = fs.readFileSync(`${here}job-triangle.ts`, 'utf8'); + const thisSrc = fs.readFileSync( + `${here}c9-assembled-solution.ts`, + 'utf8', + ); + const assembled = + bodyLines(triangleSrc, 'export function jobTriangle(') + + bodyLines(triangleSrc, 'export function jobPollSurface(') + + bodyLines(triangleSrc, 'export function operationDeadline(') + + bodyLines(triangleSrc, 'export function retryAfterMs('); + const hand = + bodyLines(thisSrc, 'async function handRolled(') + + bodyLines(triangleSrc, 'export function retryAfterMs('); + note( + '(e) assembled: jobTriangle + surface + deadline + header parse', + assembled, + ); + note('(e) hand-rolled: the `while` + the same header parse', hand); + check('(e) is the assembled version longer?', assembled > hand, true); + note( + '(e) `retryAfterMs` is counted on BOTH sides — a correct `while` loop needs it too', + '', + ); + } + + // ── (f) what the extra lines actually BUY, measured ─────────────────────────────────────── + // The `while` loop has none of these, because there is nothing between it and the transport. + { + const clock = manualClock(); + const api = new FakeJobApi({ clock, inProgressPolls: 2 }); + const recs: Rec[] = []; + const job = jobTriangle({ + submitUrl: FakeJobApi.submitUrl, + host: HOST, + adapter: api.adapter(), + clock, + pollAttempts: 20, + budgetMs: 3_600_000, + backoff: { base: 1000, max: 1000 }, + trace: recordingSink(recs), + }); + const p = job.run({ q: 'SELECT Id' }); + await clock.advance(3_600_000); + await p; + + check('(f) requests made', api.hits.length, 5); + check( + '(f) `start` events (one per HOP, not per poll)', + recs.filter((r) => r.type === 'start').length, + 3, + ); + check( + '(f) distinct traceIds across the operation', + new Set(recs.map((r) => r.traceId)).size, + 1, + ); + const starts = recs.filter((r) => r.type === 'start'); + check( + '(f) the chain', + starts.map((r) => r.name).join(' → '), + 'job-submit → job-poll → job-download', + ); + check( + '(f) each hop’s parent is the previous hop’s span', + starts[1]?.parentSpanId === starts[0]?.spanId && + starts[2]?.parentSpanId === starts[1]?.spanId, + true, + ); + const pollDone = recs.filter( + (r) => r.name === 'job-poll' && r.type === 'done', + ); + check('(f) `done` events for the 3-poll hop', pollDone.length, 1); + check( + '(f) the poll count IS `attempts` on the poll run', + recs.find((r) => r.name === 'job-poll' && r.type === 'result') + ?.attempts, + 3, + ); + check( + '(f) …and the download hop reports its own', + recs.find((r) => r.name === 'job-download' && r.type === 'result') + ?.attempts, + 1, + ); + note( + '(f) → the hand-rolled loop reports one call per poll, or nothing at all', + '', + ); + } + + finish( + 'C9', + 'The assembled answer runs: submit → 5 polls at the SERVER’s 300s pacing → 1 download, 1 submit, 20 virtual minutes, the job id persisted for a restart; resume reattaches after a 3-poll crash with 1 submit total; the deadline ends a runaway job at 6 polls in a virtual hour. The hand-rolled `while` produces the BYTE-IDENTICAL request sequence and pacing. What the extra lines buy is measured, not asserted: one `start` + one `done` per HOP with the 3 polls folded in as `attempts: 3` (instead of three unrelated calls), one traceId chaining `job-submit → job-poll → job-download`, and a per-hop `retry` policy (20 poll attempts, 1 download attempt). The cost is the honest number: 110 executable lines against 49 for the `while` — and ALL of the semantics are still yours', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/async-job-polling/fake-jobs.ts b/docs/scenarios/proofs/async-job-polling/fake-jobs.ts new file mode 100644 index 00000000..fec466cc --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/fake-jobs.ts @@ -0,0 +1,264 @@ +// A fake, in-memory ASYNC JOB API — the three endpoints of the async request–reply triangle, +// shaped the way Salesforce Bulk API 2.0 / Shopify bulk operations / a report renderer shape them: +// +// 1. `POST /jobs` → **202** + `Location: /jobs/{id}` (+ optionally `Retry-After`) +// 2. `GET /jobs/{id}` → **200** `{ state }`, cycling `InProgress` N times then a TERMINAL +// `JobComplete` (with `resultUrl`) or `Failed` (with `errorMessage`). +// Every one of those is an HTTP 200 — the status line never says +// "done" and never says "failed". +// 3. `GET ` → the payload, **SINGLE-USE**: the first fetch works, a second 404s. +// +// Everything is driven by an INJECTED {@link Clock} and nothing touches the network, so an +// hour-long poll is virtual time. The provider records EVERY hit with the virtual timestamp, which +// makes three things measurements rather than arguments: +// +// - `polls(id).length` — how many times the client polled. +// - `gaps(id)` — the ms between successive polls on the injected clock. +// - `submits` — how many times the client SUBMITTED. `> 1` is a duplicate job: hours of +// server work done twice, the failure mode that makes restart dangerous. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +/** One recorded hit, as the provider saw it. */ +export interface RecordedHit { + method: string; + /** Path only (the fake has one host), e.g. `/jobs/job-1`. */ + path: string; + status: number; + /** Virtual time (ms) the request arrived, read off the injected clock. */ + at: number; +} + +/** What a job status body looks like on the wire. All three variants are HTTP 200. */ +export interface JobStatus { + id: string; + /** Salesforce's vocabulary: `InProgress` is non-terminal; `JobComplete`/`Failed` are terminal. */ + state: 'InProgress' | 'JobComplete' | 'Failed'; + /** Present only on `JobComplete`. */ + resultUrl?: string; + /** Present only on `Failed`. */ + errorMessage?: string; +} + +export interface JobsOptions { + clock: Clock; + /** + * How many `InProgress` polls a submitted job answers before it reaches its terminal state. + * Default 3. + */ + inProgressPolls?: number; + /** Terminal state the job lands on. Default `'JobComplete'`. */ + terminal?: 'JobComplete' | 'Failed'; + /** + * `Retry-After` the provider sends on the `202` **and** on every `InProgress` poll — the + * server's own pacing. A number is delta-seconds (`30` → `'30'`); a string is the RAW header + * value, so an HTTP-date (`'Thu, 01 Jan 1970 00:01:00 GMT'`) — the other form RFC 9110 allows + * — can be exercised too. Omit for a provider that sends no header at all, which is the case + * a client must fall back to a computed backoff for. + */ + retryAfter?: number | string; + /** Payload the result URL serves once. Default `{ rows: 3 }`. */ + payload?: unknown; +} + +const HOST = 'https://bulk.example.com'; + +/** One submitted job's server-side state. */ +interface JobRecord { + id: string; + pollsSeen: number; + /** Fetches of this job's result URL; the second one 404s. */ + resultFetches: number; +} + +/** + * The three-endpoint async job API. One instance is one server: submit as many jobs as you like, + * each gets its own id and its own poll counter. + */ +export class FakeJobApi { + /** Every hit, in order, across all three endpoints. */ + readonly hits: RecordedHit[] = []; + private readonly clock: Clock; + private readonly inProgressPolls: number; + private readonly terminal: 'JobComplete' | 'Failed'; + private readonly retryAfter: string | undefined; + private readonly payload: unknown; + private readonly jobs = new Map(); + private nextId = 1; + + constructor(opts: JobsOptions) { + this.clock = opts.clock; + this.inProgressPolls = opts.inProgressPolls ?? 3; + this.terminal = opts.terminal ?? 'JobComplete'; + this.retryAfter = + opts.retryAfter === undefined ? undefined : String(opts.retryAfter); + this.payload = opts.payload ?? { rows: 3 }; + } + + /** How many times `POST /jobs` was called. **`> 1` is a duplicate-submitted job.** */ + get submits(): number { + return this.hits.filter( + (h) => h.path === '/jobs' && h.method === 'POST', + ).length; + } + + /** The ids the server minted, in submission order. */ + get jobIds(): string[] { + return [...this.jobs.keys()]; + } + + /** Every `GET /jobs/{id}` hit for one job. `.length` IS the poll count. */ + polls(id: string): RecordedHit[] { + return this.hits.filter( + (h) => h.method === 'GET' && h.path === `/jobs/${id}`, + ); + } + + /** Virtual-clock ms between successive polls of one job — the pacing, measured. */ + gaps(id: string): number[] { + const at = this.polls(id).map((h) => h.at); + return at.slice(1).map((t, i) => t - at[i]!); + } + + /** Every hit on a result URL, whatever its job. */ + get resultFetches(): RecordedHit[] { + return this.hits.filter((h) => h.path.startsWith('/results/')); + } + + /** The `Location`-relative path for a job id — what step 1's header carries. */ + static locationOf(id: string): string { + return `/jobs/${id}`; + } + + /** Absolute URL for a job id, for a client that already knows the id (the resume case). */ + static statusUrl(id: string): string { + return `${HOST}/jobs/${id}`; + } + + /** The submit endpoint's absolute URL. */ + static get submitUrl(): string { + return `${HOST}/jobs`; + } + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const path = new URL(req.url).pathname; + const method = req.method.toUpperCase(); + const res = this.route(method, path); + this.hits.push({ + method, + path, + status: res.status, + at: this.clock.now(), + }); + return res; + }; + } + + private route(method: string, path: string): AdapterResponse { + if (method === 'POST' && path === '/jobs') return this.submit(); + if (method === 'GET' && path.startsWith('/jobs/')) + return this.status(path.slice('/jobs/'.length)); + if (method === 'GET' && path.startsWith('/results/')) + return this.result(path.slice('/results/'.length)); + return { + status: 404, + headers: {}, + body: { message: `no route ${path}` }, + }; + } + + // Step 1 — 202 Accepted. The body deliberately carries NOTHING useful: the only place the job + // id appears is the `Location` HEADER, which is what makes this scenario's first hop hard. + private submit(): AdapterResponse { + const id = `job-${this.nextId++}`; + this.jobs.set(id, { id, pollsSeen: 0, resultFetches: 0 }); + return { + status: 202, + headers: { + location: FakeJobApi.locationOf(id), + ...(this.retryAfter === undefined + ? {} + : { 'retry-after': this.retryAfter }), + }, + body: {}, + }; + } + + // Step 2 — always HTTP 200. `InProgress` for the first N polls, then the terminal state. + private status(id: string): AdapterResponse { + const job = this.jobs.get(id); + if (!job) + return { + status: 404, + headers: {}, + body: { message: `unknown job ${id}` }, + }; + job.pollsSeen += 1; + const done = job.pollsSeen > this.inProgressPolls; + if (!done) + return { + status: 200, + headers: + this.retryAfter === undefined + ? {} + : { 'retry-after': this.retryAfter }, + body: { + id, + state: 'InProgress', + } satisfies JobStatus, + }; + return { + status: 200, + headers: {}, + body: + this.terminal === 'JobComplete' + ? ({ + id, + state: 'JobComplete', + resultUrl: `${HOST}/results/${id}`, + } satisfies JobStatus) + : ({ + id, + state: 'Failed', + errorMessage: 'InvalidBatch : Field name not found', + } satisfies JobStatus), + }; + } + + // Step 3 — SINGLE-USE. The pre-signed link expires on first successful fetch; every later + // fetch is a permanent 404 that a naive `retry` will happily attempt three times. + private result(id: string): AdapterResponse { + const job = this.jobs.get(id); + if (!job) + return { + status: 404, + headers: {}, + body: { message: `unknown result ${id}` }, + }; + job.resultFetches += 1; + if (job.resultFetches > 1) + return { + status: 404, + headers: {}, + body: { message: 'link expired' }, + }; + return { status: 200, headers: {}, body: this.payload }; + } +} + +/** Read `state` off a job-status body. */ +export const stateOf = (body: unknown): JobStatus['state'] | undefined => + (body as JobStatus | null | undefined)?.state; + +/** Read `resultUrl` off a job-status body. */ +export const resultUrlOf = (body: unknown): string | undefined => + (body as JobStatus | null | undefined)?.resultUrl; + +/** Read `errorMessage` off a job-status body. */ +export const errorMessageOf = (body: unknown): string | undefined => + (body as JobStatus | null | undefined)?.errorMessage; diff --git a/docs/scenarios/proofs/async-job-polling/harness.ts b/docs/scenarios/proofs/async-job-polling/harness.ts new file mode 100644 index 00000000..c7c386f9 --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/harness.ts @@ -0,0 +1,56 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED number either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured number is at most `bound`. Used for the ONE wall-clock measurement in this + * suite (`timeout.total` is deliberately wall-clock — engine.ts:482 — so no injected clock can + * drive it); the bound is set 4× clear of the real timing so a slow machine cannot flip it. + */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${actual}${ok ? '' : ` (expected ≤ ${bound})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/async-job-polling/job-triangle.ts b/docs/scenarios/proofs/async-job-polling/job-triangle.ts new file mode 100644 index 00000000..e33c4a1c --- /dev/null +++ b/docs/scenarios/proofs/async-job-polling/job-triangle.ts @@ -0,0 +1,189 @@ +// USER CODE — the best answer this scenario has from the public API. Three stitches under `linked`, +// one caller-owned `AbortSignal` as the operation deadline, and a poll surface that carries the +// three rules the built-ins do not: in-band `InProgress` is a retry, in-band `Failed` is a failure, +// and the wait comes from `Retry-After` in SECONDS with a capped exponential fallback. +// +// Nothing here is exotic — every piece is a documented seam (`Surface.interpret`, `hooks.onResponse`, +// `StitchInput.signal`, `linked`). What the file exists to show is HOW MUCH of it there is, so C9's +// comparison against a hand-rolled `while` is a line count rather than an opinion. +import { stitch } from '../../../../packages/core/src/index'; +import { linked } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { verdictOf } from '../../../../packages/core/src/surface'; +import type { + Adapter, + Clock, + StitchStore, + TraceSink, +} from '../../../../packages/core/src/types'; +import { errorMessageOf, resultUrlOf, stateOf } from './fake-jobs'; + +/** + * Read `Retry-After` as the wait for the NEXT poll. Two things this must get right and neither is + * done for you: the header is delta-SECONDS (a bare `'30'` handed to `after` means 30ms — C3(c)), + * and the HTTP-date form is unparseable by `parseDuration`, so it must be converted here or the + * server's pacing is silently discarded (C3(d)). Returns `undefined` to fall through to `backoff`. + */ +export function retryAfterMs( + header: string | undefined, + clock: Clock, +): number | undefined { + if (header === undefined) return undefined; + if (/^\d+$/.test(header.trim())) return Number(header.trim()) * 1000; + const at = Date.parse(header); + return Number.isNaN(at) ? undefined : Math.max(0, at - clock.now()); +} + +/** + * The poll loop as a surface. `verdictOf` first, so a `verdict`-declared status still rules and a + * 404 does not come back as a successful poll (C2(e)). + */ +export function jobPollSurface(clock: Clock): Surface { + return { + id: 'job-poll', + interpret: (res, cfg) => { + const failed = verdictOf(res, cfg); + if (failed) return failed; + const state = stateOf(res.body); + if (state === 'InProgress') { + const after = retryAfterMs(res.headers['retry-after'], clock); + return after === undefined + ? { ok: false, retry: true, message: 'InProgress' } + : { ok: false, retry: true, message: 'InProgress', after }; + } + if (state === 'Failed') + return { + ok: false, + message: `job failed: ${errorMessageOf(res.body)}`, + status: res.status, + }; + return { ok: true, data: res.body }; + }, + }; +} + +/** + * The operation deadline: an `AbortSignal` that fires after `ms` on the INJECTED clock. There is no + * config field for a budget spanning three stitches (C5(c)), and `AbortSignal.timeout` is wall-clock + * only — so an hour-long budget is only testable if the timer is the stitch's own clock. + */ +export function operationDeadline(ms: number, clock: Clock): AbortSignal { + const ctrl = new AbortController(); + const timer = clock.setTimer(() => { + ctrl.abort(new Error(`job budget of ${ms}ms exhausted`)); + }, ms); + ctrl.signal.addEventListener('abort', () => { + clock.clearTimer(timer); + }); + return ctrl.signal; +} + +export interface JobTriangleOptions { + /** Where `POST /jobs` lives. */ + submitUrl: string; + /** Origin the `Location` header is resolved against. */ + host: string; + adapter: Adapter; + clock: Clock; + /** Poll bound. Reached before the deadline, this is what ends the operation. */ + pollAttempts: number; + /** Whole-operation budget (ms) — submit + every poll + the download. */ + budgetMs: number; + /** Fallback poll spacing when the server sends no `Retry-After`. */ + backoff: { base: number | string; max: number | string }; + /** Where the job id is persisted so a restart can reattach (C8(b)). */ + store?: StitchStore; + /** Key under `store` for the job's `Location`. */ + storeKey?: string; + /** One sink for all three stitches, so the operation's trace chain is observable (C6). */ + trace?: TraceSink; +} + +/** What one run of the triangle produced. */ +export interface JobTriangleResult { + /** The downloaded payload. */ + data: T; + /** The job's `Location`, for a resume that skips the submit. */ + location: string; +} + +/** + * Submit → poll to a terminal state → download, as one operation. Fails fast on an in-band `Failed`, + * on the poll budget, and on the deadline. Pass `resumeFrom` to reattach to a job already submitted + * — the submit is skipped entirely, which is the whole point of persisting the id. + */ +export function jobTriangle(opts: JobTriangleOptions) { + const { adapter, clock, host } = opts; + const key = opts.storeKey ?? 'job:location'; + let location = ''; + + const trace = opts.trace; + + const submit = stitch({ + name: 'job-submit', + url: opts.submitUrl, + method: 'POST', + adapter, + clock, + ...(trace === undefined ? {} : { trace }), + // A stable key so a restarted process's resubmit is collapsible server-side (C8(d)). + idempotency: { keyOf: (input) => `job:${JSON.stringify(input.body)}` }, + hooks: { + onResponse: (ctx) => { + const loc = ctx.res?.headers['location']; + if (loc === undefined) return; + location = loc; + void opts.store?.set(key, loc, opts.budgetMs); + }, + }, + }); + + const poll = stitch({ + name: 'job-poll', + url: `${host}{+loc}`, // `{+}` = reserved expansion: the slashes survive (C1(f)) + kind: jobPollSurface(clock), + adapter, + clock, + ...(trace === undefined ? {} : { trace }), + retry: { + attempts: opts.pollAttempts, + backoff: { curve: 'expo', ...opts.backoff }, + }, + }); + + const download = stitch({ + name: 'job-download', + url: '{+u}', + adapter, + clock, + ...(trace === undefined ? {} : { trace }), + // The link is single-use: one shot, and a 404 is permanent (C7). + retry: { attempts: 1 }, + }); + + return { + submit, + poll, + download, + /** Run the operation. `resumeFrom` skips the submit and polls an existing job. */ + run: (body: unknown, resumeFrom?: string): Promise => + linked(async (run) => { + const signal = operationDeadline(opts.budgetMs, clock); + if (resumeFrom === undefined) + await run(submit, { body, signal }); + else location = resumeFrom; + const status = await run(poll, { + params: { loc: location }, + signal, + }); + const url = resultUrlOf(status); + if (url === undefined) + throw new Error('JobComplete with no resultUrl'); + const data = await run(download, { + params: { u: url }, + signal, + }); + return { data, location }; + }), + }; +} diff --git a/docs/scenarios/proofs/batch-partial-failure/README.md b/docs/scenarios/proofs/batch-partial-failure/README.md new file mode 100644 index 00000000..50aa8426 --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/README.md @@ -0,0 +1,87 @@ +# Proofs — batch endpoints that report per-item failure inside a 200 + +Runnable evidence for the claims in +[`../../batch-partial-failure.md`](../../batch-partial-failure.md). + +Every script is standalone, offline, and deterministic: it injects a fake DynamoDB +`BatchWriteItem` / Elasticsearch `_bulk` through StitchAPI's `adapter` seam and drives every wait +off an injected `manualClock()`, so the gaps below are exact virtual time — no wall-clock sleeps, +nothing flaky, no network. The one deliberate exception is C7(d), which measures `timeout.total`; +that budget is wall-clock **by design** (engine.ts:482), so it runs on real timers with bounds set +4× clear of the real numbers. + +**The fake providers count writes per item.** `db.writeCount('a')` is 3 if the client wrote row `a` +three times, so "the retry re-applied rows that had already succeeded" is a measured number +(`duplicateWrites`), not an argument. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c1-retry-replays-the-batch.ts + +# all of them +for f in docs/scenarios/proofs/batch-partial-failure/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, +so they test the working tree, not the published bundle. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| `c1-retry-replays-the-batch.ts` | can built-in `retry` handle a partial failure? | **No.** 1 request under `attempts: 5`; forced with `on: 200` it replays all 5 items — **6 duplicate writes** | +| `c2-paginate-residue-loop.ts` | can `paginate.next` resend only the residue? | **Yes** — 3 requests, **0 duplicates**, all 6 rows land. But a zero-item page ends the run **ok** with 4 rows gone | +| `c3-backoff-between-pages.ts` | can anything wait between rounds? | **No built-in.** 6 rounds at t=0; `throttle` gives a **fixed** spacing only; growth = user code in a hook | +| `c4-surface-rewrite.ts` | can a `Surface` rewrite the request? | **No** — `SurfaceOutcome.retry` resends the identical body (4 duplicates). **`hooks.onRequest` can** | +| `c5-terminal-vs-retryable.ts` | can 429 and 400 per-item failures be split? | **Yes** — the 400 doc is sent **once**, vs **5×** for "resend everything that failed" | +| `c6-residue-reachability.ts` | when it gives up, where is the residue? | **Nowhere the engine owns.** ok result, no error, absent from events/trace/`inspect`/`report` | +| `c7-assembled-solution.ts` | what is the best answer, and is it worth it? | 4 rounds, 0 duplicates, exponential wait, residue returned as data — in **50 lines vs 28** hand-rolled | + +## Files + +- `fake-batch.ts` — the providers. `FakeDynamo` answers `200 { Processed, UnprocessedItems }` and can + be driven by a real **write-capacity bucket** refilling off the clock (AWS's actual cause of + `UnprocessedItems`, so "retry immediately and you throttle again" is measurable). `FakeElastic` + answers `200 { errors: true, items: [{ index: { status } }] }` with a mix of `429` and `400`. + Both count writes per item. +- `harness.ts` — `check` / `checkAtMost` / `note` / `heading` / `finish`. No test framework. +- `batch-retry-surface.ts` — **user code** for C7: `batchRetry()`, the `Surface` + `hooks` + ledger + triple that carries the assembled solution. + +## Reading the numbers honestly + +- **C2(d) and C6 are the same bug seen twice, and it is the scenario's own failure mode.** + `paginated` breaks on `items.length === 0` **before** calling `next` (engine.ts:984), and breaking + on `page >= max` falls straight through to the `result` event. Both end the call **successfully** + with the residue discarded. A batch endpoint answers "nothing landed this round" exactly when the + table is out of capacity — the normal case AWS tells you to back off from — so this is not an + exotic corner. It is elastic/logstash#1631 reproduced inside the library that was supposed to fix it. +- **C6(g) is worse than "incomplete".** The obvious place to build a residue ledger is `paginate.next`, + and it is **wrong**, not merely short: on a 3-round run capped at 3, `next` sees the residue after + rounds 1 and 2 only, so it reports `cdef` when the true residue is `def`. It names a row that + landed. `hooks.onResponse` sees every response and reports `def`. +- **C3's "no backoff" is a property of the type, not just the run.** The `@ts-expect-error` blocks on + `paginate: { delay }` / `{ backoff }` and `throttle: { backoff }` are machine-checked: a + `@ts-expect-error` that is _not_ an error fails `tsc`, and these files typecheck clean under + `packages/core`'s full strict set. +- **C3(c)'s exponential curve is user code, and the engine goes blind to it.** A sleep in an async + `onRequest` hook really does pace the rounds (0, 100, 300, 700, 1500, 3100 ms), but no `throttled` + event fires and no `waited` is reported: the run says it never waited while 2.5s of virtual time + passed. A `throttle` wait of the same length is reported. +- **C4(a) refutes scenario 2's answer for this scenario.** `SurfaceOutcome.retry` + `after` was the + seam that solved the cost-limit case. Here it is actively harmful: the re-attempt resends the same + six items, so it re-applies the rows that already succeeded. The seam that works is a _hook_. +- **C7(f) is the footgun the assembled solution ships with.** The ledger lives on the stitch, so two + concurrent calls through one stitch cross-contaminate: measured, both callers were handed the + other batch's items, both resolved `ok`, and rows `b` and `c` were never written by anybody. It + typechecks, it reads correctly, and it silently loses data — the same class of bug the scenario is + about. +- **C7(g)'s line counts are the honest comparison.** 50 lines for the loop alone (types and options + excluded; 71 for the whole file) against 28 for the hand-rolled `while`. What the extra 22 lines + buy is measured, not asserted: one `start` event instead of three, `attempts: 3` instead of three + calls each reporting `attempts: 1`, a circuit that opens on a broken host, and `timeout.total` + bounding the whole operation instead of each round. diff --git a/docs/scenarios/proofs/batch-partial-failure/batch-retry-surface.ts b/docs/scenarios/proofs/batch-partial-failure/batch-retry-surface.ts new file mode 100644 index 00000000..85afdf44 --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/batch-retry-surface.ts @@ -0,0 +1,130 @@ +// USER CODE for C7 — the best batch-residue loop the public API supports. +// +// It is built from three seams that already exist: +// +// 1. `Surface.interpret` reads the 200 body and asks for another attempt +// (`SurfaceOutcome.retry`, surface.ts:34-37) with a wait that GROWS (`after`); +// 2. `hooks.onRequest` rewrites `ctx.req.body` to the residue before that attempt leaves +// (engine.ts:646-670) — the only seam in the library that can change a request between +// attempts (C4); +// 3. a ledger closure the two share, which is also what makes the residue REACHABLE: when the +// round budget runs out this returns the residue as the call's DATA rather than dropping it. +// +// Everything below the `── surface ──` line is the code a user would write. +import { verdictOf } from '../../../../packages/core/src/index'; +import type { + Surface, + SurfaceOutcome, +} from '../../../../packages/core/src/surface'; +import type { AtLeastOne, Hooks } from '../../../../packages/core/src/types'; + +/** What a batch call resolves to: what landed, what never did, and whether it ran out of rounds. */ +export interface BatchLedger { + landed: T[]; + /** Items that were still unwritten when the loop stopped. Empty on a clean run. */ + residue: T[]; + /** Items the provider rejected permanently (a 400 mapping error) — never resent. */ + terminal: T[]; + rounds: number; + gaveUp: boolean; +} + +export interface BatchRetryOptions { + /** Items to RESEND, read off one response body (DynamoDB `UnprocessedItems`, ES's 429s). */ + residueOf: (body: unknown) => T[]; + /** Items that LANDED, read off the same body. */ + landedOf: (body: unknown) => T[]; + /** Items that failed permanently. Default: none. */ + terminalOf?: (body: unknown) => T[]; + /** Build the next request body from a residue. */ + bodyOf: (items: T[]) => unknown; + /** + * Rounds allowed, including the first. **Must equal the stitch's `retry.attempts`** — the + * engine does not tell `interpret` which attempt it is on (surface.ts:61-64), so this counts + * its own invocations instead. + */ + rounds: number; + /** Wait before round N (1-based, so `backoff(1)` precedes the SECOND request). */ + backoff: (round: number) => number | string; +} + +/** + * Build the `kind` + `hooks` + `ledger` triple for a batch stitch: + * + * ```ts + * const batch = batchRetry({ … }); + * const call = stitch({ url, method: 'POST', kind: batch.kind, hooks: batch.hooks, retry: { attempts: 6 } }); + * const out = (await call({ body })) as BatchLedger; + * ``` + * + * ⚠️ The ledger is ONE object per stitch, so a stitch built this way serves ONE CALL AT A TIME. + * Two concurrent calls share the ledger and corrupt each other (c7 measures exactly that). + */ +// #region loop +export function batchRetry(opts: BatchRetryOptions): { + kind: Surface; + // `StitchConfig.hooks` takes `AtLeastOne` (the opaque `hooks: {}` is rejected), so a + // helper that hands back a plain `Hooks` does not typecheck at the call site. + hooks: AtLeastOne; + ledger: BatchLedger; +} { + const ledger: BatchLedger = { + landed: [], + residue: [], + terminal: [], + rounds: 0, + gaveUp: false, + }; + + // ── surface ─────────────────────────────────────────────────────────────────────────────── + const kind: Surface = { + id: 'batch-residue', + interpret: (res, cfg): SurfaceOutcome => { + // Compose the declarative verdict first (surface.ts:151-172): a 500 is a transport + // failure before it is a batch envelope, and it must still open the circuit. + const failed = verdictOf(res, cfg); + if (failed) return failed; + + ledger.rounds += 1; + ledger.landed.push(...opts.landedOf(res.body)); + ledger.terminal.push(...(opts.terminalOf?.(res.body) ?? [])); + ledger.residue = opts.residueOf(res.body); + + if (ledger.residue.length === 0) return { ok: true, data: ledger }; + if (ledger.rounds >= opts.rounds) { + // Out of rounds. Resolve SUCCESSFULLY with the residue in the payload: an error + // would throw the landed items away, and dropping it is the Logstash bug. + ledger.gaveUp = true; + return { ok: true, data: ledger }; + } + return { + ok: false, + retry: true, + message: `${ledger.residue.length} unprocessed after round ${ledger.rounds}`, + after: opts.backoff(ledger.rounds), + }; + }, + }; + + const hooks: AtLeastOne = { + onRequest: (ctx) => { + if (!ctx.req) return; + if (ctx.attempt === 1) { + // A fresh call: clear the previous one's ledger. + ledger.landed = []; + ledger.residue = []; + ledger.terminal = []; + ledger.rounds = 0; + ledger.gaveUp = false; + return; + } + // Assign, never mutate in place: the attempt's request is a SHALLOW clone of one + // `baseReq` (engine.ts:261-264), so an in-place edit of `body` would rewrite the + // original too. + ctx.req.body = opts.bodyOf(ledger.residue); + }, + }; + + return { kind, hooks, ledger }; +} +// #endregion loop diff --git a/docs/scenarios/proofs/batch-partial-failure/c1-retry-replays-the-batch.ts b/docs/scenarios/proofs/batch-partial-failure/c1-retry-replays-the-batch.ts new file mode 100644 index 00000000..d886e20b --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c1-retry-replays-the-batch.ts @@ -0,0 +1,208 @@ +// C1 — the BUILT-IN `retry` against a 200 that reports per-item failure in its body. Two questions, +// both measured against a provider that counts writes PER ITEM: +// +// 1. does `retry` fire at all, when the status is 200? +// 2. when forced to fire, how many times are the items that ALREADY SUCCEEDED written again? +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c1-retry-replays-the-batch.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StatusMatch } from '../../../../packages/core/src/types'; +import { FakeDynamo, dynamoBody, unprocessedOf } from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const IDS = ['a', 'b', 'c', 'd', 'e']; + +/** A table that lands the first THREE items of whatever it is sent and rejects the rest. */ +function table(): { db: FakeDynamo; clock: ReturnType } { + const clock = manualClock(); + return { db: new FakeDynamo({ clock, accepts: 3 }), clock }; +} + +async function main(): Promise { + heading( + 'C1 — does `retry` fire on a 200-with-UnprocessedItems, and what does it cost?', + ); + + // ── (a) the DEFAULT retry set — and the Logstash failure mode, reproduced ────────────────── + // `retry.on` defaults to [429, 502, 503, 504]. A partial failure is a 200, so nothing matches: + // one request, no retry, and — worse — the call resolves SUCCESSFULLY with the half-failed + // envelope as its data. Two of the five rows are simply gone, and nothing said so. + { + const { db, clock } = table(); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + retry: { attempts: 5 }, + }); + const r = await call.safe({ body: dynamoBody(IDS) }); + check( + '(a) requests made (retry.attempts was 5)', + db.requests.length, + 1, + ); + check('(a) the call REPORTED SUCCESS on a partial failure', r.ok, true); + check('(a) items that landed', db.landed.join(','), 'a,b,c'); + check( + '(a) …items silently left behind', + unprocessedOf(r.data) + .map((i) => i.id) + .join(','), + 'd,e', + ); + note( + '(a) → this is elastic/logstash#1631: a 200, and the residue is the caller’s problem', + '', + ); + } + + // ── (b) a PREDICATE on `retry.on` — it is handed the STATUS ONLY ─────────────────────────── + // If the predicate saw the response it could read `UnprocessedItems`. It does not: the engine + // calls it as `retryMatch(res.status)` (engine.ts:743) — one argument, a number. + { + const { db, clock } = table(); + const received: unknown[][] = []; + const on = ((...args: unknown[]): boolean => { + received.push(args); + return false; + }) as StatusMatch; + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + retry: { attempts: 5, on }, + }); + await call.safe({ body: dynamoBody(IDS) }); + check('(b) predicate invoked', received.length > 0, true); + check('(b) arguments handed to retry.on', received[0]?.length, 1); + check('(b) argument value', received[0]?.[0], 200); + note( + '(b) the body — where the failure lives — is not passed', + JSON.stringify(received[0] ?? []), + ); + } + + // ── (c) FORCE it with `retry.on: 200` — and measure the duplicate writes ─────────────────── + // This is the only built-in spelling that makes a retry fire here, and it replays the request + // byte-for-byte. The three rows that already landed are written again on every attempt. + { + const { db, clock } = table(); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 1000 }, + }, + }); + const p = call.safe({ body: dynamoBody(IDS) }); + await clock.advance(60_000); + await p; + check('(c) requests made', db.requests.length, 3); + check( + '(c) every request carried ALL FIVE items', + db.requests.every((r) => r.ids.length === 5), + true, + ); + check( + '(c) writes of item `a` (it landed on attempt 1)', + db.writeCount('a'), + 3, + ); + check( + '(c) DUPLICATE WRITES caused by the retry', + db.duplicateWrites, + 6, + ); + check( + '(c) items still not written after 3 attempts', + 5 - db.landed.length, + 2, + ); + note( + '(c) → 9 writes to land 3 rows, and the 2 that failed never got a different request', + '', + ); + } + + // ── (d) the `onResponse` status-rewrite hack — same replay, invented status ──────────────── + // Rewriting a 200 into a 429 inside `onResponse` (the hook fires at engine.ts:705, before the + // retry check at :743) does drive the DEFAULT retry set. It changes WHEN the retry fires, never + // WHAT it sends: the duplicate count is identical, and the caller's error status is now a 429 + // that was never on the wire. + { + const { db, clock } = table(); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 1000 } }, + hooks: { + onResponse: (ctx) => { + if (ctx.res && unprocessedOf(ctx.res.body).length > 0) + ctx.res.status = 429; + }, + }, + }); + const p = call.safe({ body: dynamoBody(IDS) }); + await clock.advance(60_000); + const r = await p; + check('(d) requests made', db.requests.length, 3); + check('(d) DUPLICATE WRITES', db.duplicateWrites, 6); + check('(d) call failed', r.ok, false); + check( + '(d) status the caller is handed (never sent by the provider)', + r.error?.status, + 429, + ); + note('(d) error message', r.error?.message); + } + + // ── (e) the cost on a HEALTHY batch: `on: 200` retries SUCCESSES too ─────────────────────── + // The status matcher runs before any body is interpreted, so `on: 200` cannot tell a partial + // failure from a complete success. A batch that fully succeeded is written three times over. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 99 }); // roomy table: everything lands + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 1000 }, + }, + }); + const p = call.safe({ body: dynamoBody(IDS) }); + await clock.advance(60_000); + await p; + check( + '(e) requests made for a batch that succeeded first time', + db.requests.length, + 3, + ); + check('(e) total writes for 5 rows', db.totalWrites, 15); + check( + '(e) DUPLICATE WRITES on a fully successful batch', + db.duplicateWrites, + 10, + ); + } + + finish( + 'C1', + 'built-in `retry` cannot see a per-item failure (the status is 200 and `retry.on` receives only the status), and the one spelling that forces it to fire — `on: 200` — replays the WHOLE batch: 6 duplicate writes to chase 2 failed rows, and 10 duplicate writes on a batch that never failed at all', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c2-paginate-residue-loop.ts b/docs/scenarios/proofs/batch-partial-failure/c2-paginate-residue-loop.ts new file mode 100644 index 00000000..fe5b341d --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c2-paginate-residue-loop.ts @@ -0,0 +1,175 @@ +// C2 — can `paginate.next(prevBody, pagesFetched)` express "resend ONLY the residue"? It returns +// the input merged over the original, or `undefined` to stop (types.ts:1412-1422), which is +// structurally the loop this scenario needs. This measures the loop it builds: requests, DUPLICATE +// WRITES (the number that must be zero), and whether every item eventually lands. +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c2-paginate-residue-loop.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { AdapterRequest } from '../../../../packages/core/src/types'; +import { + FakeDynamo, + dynamoBody, + processedOf, + unprocessedOf, +} from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const IDS = ['a', 'b', 'c', 'd', 'e', 'f']; + +/** The residue loop, spelled on `paginate`. This is the whole of the user's code. */ +const residueLoop = (pages?: number) => ({ + // `UnprocessedItems` comes back in the same shape `RequestItems` went out in, so the next + // request body IS the residue — no transformation, exactly as AWS documents. + next: (prevBody: unknown) => { + const residue = unprocessedOf(prevBody); + return residue.length > 0 + ? { body: { RequestItems: residue } } + : undefined; + }, + items: (value: unknown) => processedOf(value), + ...(pages === undefined ? {} : { pages }), +}); + +async function main(): Promise { + heading('C2 — `paginate.next` as a residue-resend loop'); + + // ── (a) the loop, against a table that lands 2 items per request ─────────────────────────── + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + paginate: residueLoop(), + }); + const r = await call.safe({ body: dynamoBody(IDS) }); + + check('(a) call ok', r.ok, true); + check( + '(a) requests made for 6 items at 2/request', + db.requests.length, + 3, + ); + check( + '(a) what each request carried', + db.requests.map((q) => q.ids.join('')).join(' → '), + 'abcdef → cdef → ef', + ); + check('(a) DUPLICATE WRITES', db.duplicateWrites, 0); + check('(a) every item landed exactly once', db.totalWrites, 6); + check('(a) items landed', db.landed.join(''), 'abcdef'); + check( + '(a) aggregated successes handed back', + (r.data as { id: string }[]).map((i) => i.id).join(''), + 'abcdef', + ); + } + + // ── (b) it stops when the residue empties, not when a page cap is hit ────────────────────── + // `next` returning `undefined` is the terminating condition; `pages` (default 50) is only the + // safety cap. Measured by giving it a cap it must not need. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + paginate: residueLoop(50), + }); + await call.safe({ body: dynamoBody(IDS) }); + check('(b) requests made under a 50-page cap', db.requests.length, 3); + } + + // ── (c) the rest of the request is preserved across rounds ───────────────────────────────── + // `next` returns a PARTIAL input merged over the original (engine.ts:901-915), so headers, + // query and method ride every round; only `body` is rewritten. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const seen: AdapterRequest[] = []; + const call = stitch({ + url: URL, + method: 'POST', + headers: { 'x-amz-target': 'DynamoDB_20120810.BatchWriteItem' }, + adapter: async (req) => { + seen.push(req); + return db.adapter()(req); + }, + clock, + paginate: residueLoop(), + }); + await call.safe({ + body: dynamoBody(IDS), + headers: { 'x-request-id': 'r-1' }, + }); + check('(c) requests made', seen.length, 3); + check( + '(c) static header on the LAST round', + seen[2]?.headers['x-amz-target'], + 'DynamoDB_20120810.BatchWriteItem', + ); + check( + '(c) per-call header on the LAST round', + seen[2]?.headers['x-request-id'], + 'r-1', + ); + check('(c) method on the LAST round', seen[2]?.method, 'POST'); + } + + // ── (d) THE HOLE: a round that lands NOTHING silently ends the loop ──────────────────────── + // `paginated` breaks on `items.length === 0` BEFORE it calls `next` (engine.ts:984). A batch + // endpoint answers exactly that way whenever the table has no capacity left — which is the + // normal case AWS tells you to back off from. The call then resolves ok, with the residue gone. + { + const clock = manualClock(); + // A real write-capacity bucket: 2 units at t=0, refilling 1/s. With no wait between rounds + // the second request arrives at t=0 with nothing left, so it lands zero items. + const db = new FakeDynamo({ clock, writeUnitsPerSec: 1, burst: 2 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + paginate: residueLoop(), + }); + const p = call.safe({ body: dynamoBody(IDS) }); + await clock.advance(60_000); + const r = await p; + + check( + '(d) requests made before the loop gave up', + db.requests.length, + 2, + ); + check( + '(d) the second request landed nothing', + db.requests[1]?.accepted.length, + 0, + ); + check('(d) the call REPORTED SUCCESS', r.ok, true); + check('(d) there is no error', r.error, null); + check('(d) items that landed', db.landed.join(''), 'ab'); + check( + '(d) items the caller believes were written', + (r.data as { id: string }[]).map((i) => i.id).join(''), + 'ab', + ); + note( + '(d) → 4 of 6 rows are gone, the loop stopped early, and nothing failed', + '', + ); + } + + finish( + 'C2', + '`paginate.next` DOES express the residue resend — 3 requests for 6 items, ZERO duplicate writes, every item landed — but the loop terminates on a page that aggregates zero items (engine.ts:984), so a round in which nothing lands ends the run SUCCESSFULLY with the residue dropped', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c3-backoff-between-pages.ts b/docs/scenarios/proofs/batch-partial-failure/c3-backoff-between-pages.ts new file mode 100644 index 00000000..1db7ba6d --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c3-backoff-between-pages.ts @@ -0,0 +1,374 @@ +// C3 — DECIDING CLAIM. AWS is explicit that retrying `UnprocessedItems` without waiting simply +// throttles again, so a residue loop that cannot back off is not a solution. Four questions, all +// measured on an injected `manualClock()`, so every gap below is exact virtual time: +// +// (a) is there ANY per-iteration wait in `paginate`? +// (b) can `throttle` stand in, and can its spacing GROW? +// (c) is growth reachable anywhere else on the public API? +// (d) what does a fixed spacing cost against a table that really is out of write capacity? +// (e) does the throttle that paces the loop also pace unrelated traffic? +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c3-backoff-between-pages.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + PaginateOptions, + StitchEvent, + ThrottleOptions, +} from '../../../../packages/core/src/types'; +import { + FakeDynamo, + dynamoBody, + processedOf, + unprocessedOf, +} from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const SIX = ['a', 'b', 'c', 'd', 'e', 'f']; + +const residueLoop: PaginateOptions = { + next: (prevBody: unknown) => { + const residue = unprocessedOf(prevBody); + return residue.length > 0 + ? { body: { RequestItems: residue } } + : undefined; + }, + items: (value: unknown) => processedOf(value), +}; + +/** Run the residue loop and return the virtual arrival time of every request. */ +async function gaps(opts: { + ids?: string[]; + accepts?: number; + writeUnitsPerSec?: number; + burst?: number; + throttle?: string; + /** A per-round wait implemented in user code, in the async `onRequest` hook. */ + hookWait?: (round: number) => number; +}): Promise<{ at: number[]; db: FakeDynamo; ok: boolean }> { + const clock = manualClock(); + const db = new FakeDynamo({ + clock, + ...(opts.accepts === undefined ? {} : { accepts: opts.accepts }), + ...(opts.writeUnitsPerSec === undefined + ? {} + : { writeUnitsPerSec: opts.writeUnitsPerSec }), + ...(opts.burst === undefined ? {} : { burst: opts.burst }), + }); + let round = 0; + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + paginate: residueLoop, + ...(opts.throttle === undefined ? {} : { throttle: opts.throttle }), + ...(opts.hookWait === undefined + ? {} + : { + hooks: { + onRequest: async (): Promise => { + const ms = opts.hookWait!(round++); + if (ms > 0) await clock.sleep(ms); + }, + }, + }), + }); + const p = call.safe({ body: dynamoBody(opts.ids ?? SIX) }); + await clock.advance(600_000); + const r = await p; + return { at: db.requests.map((q) => q.at), db, ok: r.ok }; +} + +async function main(): Promise { + heading('C3 — can any backoff be introduced between residue rounds?'); + + // ── (a) NO. `PaginateOptions` is three fields and the loop never sleeps ──────────────────── + // engine.ts:939-988 is `for (;;) { request; aggregate; next }` — there is no sleep site in it. + // Six rounds land on the same virtual millisecond. + { + const { at } = await gaps({ accepts: 1 }); + check( + '(a) arrival time of every round (ms)', + at.join(','), + '0,0,0,0,0,0', + ); + check('(a) rounds fired', at.length, 6); + + // The type has no slot to put one in. A `@ts-expect-error` that is NOT an error fails + // `tsc`, so these two lines are the machine-checked half of the claim. + const withDelay: PaginateOptions = { + ...residueLoop, + // @ts-expect-error — `delay` is not a `PaginateOptions` field (types.ts:1412-1422). + delay: 1000, + }; + const withBackoff: PaginateOptions = { + ...residueLoop, + // @ts-expect-error — neither is `backoff`. The three fields are next / items / pages. + backoff: { curve: 'expo', base: 100 }, + }; + void withDelay; + void withBackoff; + note( + '(a) PaginateOptions fields', + Object.keys(residueLoop).concat('pages').join(', '), + ); + } + + // ── (b) `throttle` DOES pace the rounds — with a FIXED spacing that cannot grow ──────────── + // Each page is a full request, so it takes the rate gate (engine.ts:628-644). The gaps are + // equal by construction: `rate` is one ratio parsed once, and `ThrottleOptions` has no curve. + { + const { at } = await gaps({ accepts: 1, throttle: '1/s' }); + check( + '(b) arrival times under throttle "1/s"', + at.join(','), + '0,1000,2000,3000,4000,5000', + ); + const deltas = at.slice(1).map((t, i) => t - at[i]!); + check('(b) distinct gaps between rounds', new Set(deltas).size, 1); + + const growing: ThrottleOptions = { + // @ts-expect-error — no curve/base/max on `throttle`; `rate` is a `/` + // string and nothing else (types.ts:1005-1032). + backoff: { curve: 'expo', base: 100 }, + }; + void growing; + note( + '(b) ThrottleOptions fields', + 'rate, concurrency, pool, delegate, on', + ); + } + + // ── (c) growth is reachable, but only as USER CODE inside an async hook ──────────────────── + // `onRequest` is awaited before every attempt (engine.ts:652), so sleeping in it delays the + // next round. That is the whole mechanism — there is no configuration involved. + { + const { at } = await gaps({ + accepts: 1, + hookWait: (round) => (round === 0 ? 0 : 100 * 2 ** (round - 1)), + }); + check( + '(c) arrival times with an expo wait in `onRequest`', + at.join(','), + '0,100,300,700,1500,3100', + ); + const deltas = at.slice(1).map((t, i) => t - at[i]!); + check( + '(c) the gap grows every round', + deltas.join(','), + '100,200,400,800,1600', + ); + } + + // ── (c2) …and the engine does not know it happened ──────────────────────────────────────── + // A `throttle` wait is reported as a `throttled` progress event carrying `waited`. A sleep in + // the hook is invisible: no event, no `waited`, nothing in a trace of the run. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const evts: StitchEvent[] = []; + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + paginate: residueLoop, + hooks: { + onRequest: async (ctx): Promise => { + if (ctx.attempt >= 0 && db.requests.length > 0) + await clock.sleep(500); + }, + }, + }); + const consume = (async (): Promise => { + for await (const e of call.stream({ body: dynamoBody(SIX) })) + evts.push(e); + })(); + await clock.advance(600_000); + await consume; + const throttled = evts.filter( + (e) => e.type === 'progress' && e.phase === 'throttled', + ); + check( + '(c2) rounds that really waited 500ms', + db.requests.length - 1, + 5, + ); + check( + '(c2) `throttled` events the engine emitted', + throttled.length, + 0, + ); + check( + '(c2) total virtual time spent waiting', + db.requests[5]?.at, + 2500, + ); + note( + '(c2) → the wait is real and the run report says the call never waited', + '', + ); + } + + // ── (d) what a fixed spacing buys, and what it costs ─────────────────────────────────────── + // The table now has a real write-capacity bucket: 2 units at t=0, refilling 1/s. This is the + // AWS case — `UnprocessedItems` because there is no capacity, so the wait IS the fix. + { + const none = await gaps({ writeUnitsPerSec: 1, burst: 2 }); + check('(d) no wait — rounds fired', none.at.length, 2); + check('(d) no wait — items landed', none.db.landed.length, 2); + check( + '(d) no wait — items LOST (loop broke on a zero-item page)', + 6 - none.db.landed.length, + 4, + ); + check('(d) no wait — the call still reported success', none.ok, true); + + const fixed = await gaps({ + writeUnitsPerSec: 1, + burst: 2, + throttle: '1/s', + }); + check('(d) throttle "1/s" — rounds fired', fixed.at.length, 5); + check('(d) throttle "1/s" — items landed', fixed.db.landed.length, 6); + check( + '(d) throttle "1/s" — duplicate writes', + fixed.db.duplicateWrites, + 0, + ); + check( + '(d) throttle "1/s" — finished at (ms)', + fixed.at[fixed.at.length - 1], + 4000, + ); + + // An exponential curve that starts BELOW the refill period under-waits on its first step, + // lands zero items, and hits the same zero-page break. On this seam, a curve that starts + // small is indistinguishable from no backoff at all. + const expo = await gaps({ + writeUnitsPerSec: 1, + burst: 2, + hookWait: (round) => (round === 0 ? 0 : 100 * 2 ** (round - 1)), + }); + check('(d) expo(base 100) — rounds fired', expo.at.length, 2); + check('(d) expo(base 100) — items LOST', 6 - expo.db.landed.length, 4); + note( + '(d) → correctness here comes from waiting LONG ENOUGH, not from the curve', + '', + ); + } + + // ── (d2) the cost of sizing that spacing for the worst case ──────────────────────────────── + // The same stitch against a HEALTHY table (2 items per request, no capacity limit). The + // spacing that saved (d) now paces work the table would have taken instantly. + { + const twenty = Array.from({ length: 20 }, (_, i) => `i${i}`); + const free = await gaps({ ids: twenty, accepts: 2 }); + const paced = await gaps({ ids: twenty, accepts: 2, throttle: '1/4s' }); + check( + '(d2) healthy table, no throttle — finished at (ms)', + free.at[free.at.length - 1], + 0, + ); + check( + '(d2) healthy table, throttle "1/4s" — rounds', + paced.at.length, + 10, + ); + check( + '(d2) healthy table, throttle "1/4s" — finished at (ms)', + paced.at[paced.at.length - 1], + 36_000, + ); + note( + '(d2) → 36s of pacing for work the table took in one virtual millisecond', + '', + ); + } + + // ── (e) blast radius: the spacing paces every call through that stitch ───────────────────── + // (i) default pool `'stitch'`: two concurrent batches through the SAME stitch share one + // limiter, so the second batch's rounds interleave with the first's. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + throttle: '1/s', + paginate: residueLoop, + }); + const p = Promise.all([ + call.safe({ body: dynamoBody(['a', 'b', 'c']) }), + call.safe({ body: dynamoBody(['x', 'y', 'z']) }), + ]); + await clock.advance(600_000); + await p; + const second = db.requests.filter( + (q) => + q.ids[0]?.startsWith('x') || + q.ids[0] === 'y' || + q.ids[0] === 'z', + ); + check('(e) rounds fired in total', db.requests.length, 6); + check( + '(e) the SECOND batch finished at (ms)', + second[second.length - 1]?.at, + 5000, + ); + note( + '(e) alone it would have finished at 2000ms — it waits behind the first batch’s residue rounds', + '', + ); + } + // (ii) pool 'host': an unrelated stitch on the same host is paced by the batch loop too. + { + const clock = manualClock(); + const hits: { who: string; at: number }[] = []; + const db = new FakeDynamo({ clock, accepts: 1 }); + const batch = stitch({ + url: URL, + method: 'POST', + adapter: async (req) => { + hits.push({ who: 'batch', at: clock.now() }); + return db.adapter()(req); + }, + clock, + throttle: { rate: '1/s', pool: 'host' }, + paginate: residueLoop, + }); + const reader = stitch({ + url: 'https://dynamodb.us-east-1.amazonaws.com/get', + method: 'POST', + adapter: async () => { + hits.push({ who: 'reader', at: clock.now() }); + return { status: 200, headers: {}, body: { Item: {} } }; + }, + clock, + throttle: { rate: '1/s', pool: 'host' }, + }); + const p = Promise.all([ + batch.safe({ body: dynamoBody(['a', 'b', 'c']) }), + reader.safe({ body: {} }), + ]); + await clock.advance(600_000); + await p; + check( + '(e2) pool "host" — an unrelated read waits for the batch loop (ms)', + hits.find((h) => h.who === 'reader')?.at, + 1000, + ); + note('(e2) arrivals', hits.map((h) => `${h.who}@${h.at}`).join(' ')); + } + + finish( + 'C3', + 'there is NO per-iteration wait in `paginate` (six rounds at t=0) and no field to declare one; `throttle` paces the rounds but only as a FIXED spacing (one ratio, no curve) that also paces every other call through the stitch — and every growing curve measured here is user code sleeping in an async `onRequest` hook, which the engine never reports', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c4-surface-rewrite.ts b/docs/scenarios/proofs/batch-partial-failure/c4-surface-rewrite.ts new file mode 100644 index 00000000..37fa0892 --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c4-surface-rewrite.ts @@ -0,0 +1,353 @@ +// C4 — can a custom `Surface` rewrite the OUTGOING REQUEST BODY between attempts? Scenario 2's +// answer to a body-carried failure was `SurfaceOutcome.retry` + `after`, which re-enters the attempt +// loop. This measures what that re-attempt actually SENDS, and then walks every other surface hook +// (`buildRequest`, `execute`) and the `onRequest` hook to find one that can change it. +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c4-surface-rewrite.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { + Surface, + SurfaceOutcome, +} from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + AdapterResponse, + StitchEvent, +} from '../../../../packages/core/src/types'; +import { + FakeDynamo, + dynamoBody, + processedOf, + unprocessedOf, +} from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const SIX = ['a', 'b', 'c', 'd', 'e', 'f']; + +/** A surface that reads the residue off the 200 body and asks for another attempt. */ +const retryingSurface = (afterMs: number): Surface => ({ + id: 'dynamo-batch', + interpret: (res: AdapterResponse): SurfaceOutcome => { + const residue = unprocessedOf(res.body); + return residue.length > 0 + ? { + ok: false, + retry: true, + message: `${residue.length} unprocessed`, + after: afterMs, + } + : { ok: true, data: res.body }; + }, +}); + +async function main(): Promise { + heading('C4 — can a surface rewrite the request body between attempts?'); + + // ── (a) `SurfaceOutcome.retry` re-sends the SAME request ────────────────────────────────── + // The attempt loop clones ONE `baseReq` per attempt (engine.ts:646) — it is built once, before + // the loop (`runOnce`, engine.ts:1527). The surface's verdict re-enters that loop; it does not + // rebuild the request. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const call = stitch({ + url: URL, + method: 'POST', + kind: retryingSurface(500), + adapter: db.adapter(), + clock, + retry: { attempts: 3 }, + }); + const p = call.safe({ body: dynamoBody(SIX) }); + await clock.advance(60_000); + const r = await p; + + check('(a) attempts made', db.requests.length, 3); + check( + '(a) the wait between attempts was honoured (ms)', + db.requests[1]?.at, + 500, + ); + check( + '(a) what each attempt SENT', + db.requests.map((q) => q.ids.join('')).join(' → '), + 'abcdef → abcdef → abcdef', + ); + check('(a) items already landed, written again', db.writeCount('a'), 3); + check('(a) DUPLICATE WRITES', db.duplicateWrites, 4); + check('(a) items never written', 6 - db.landed.length, 4); + check('(a) call failed after the budget', r.ok, false); + note('(a) error message', r.error?.message); + } + + // ── (a2) `interpret` is not even told which attempt it is on ─────────────────────────────── + // Its signature is `(res, cfg)` (surface.ts:61-64). A surface cannot tell a first attempt from + // a last one, so it cannot decide "this is the final round, hand back what is left". + { + const threeArg: Surface = { + id: 'x', + // @ts-expect-error — `interpret` takes (res, cfg); there is no attempt argument. + interpret: ( + res: AdapterResponse, + _cfg: unknown, + _attempt: number, + ) => ({ + ok: true as const, + data: res.body, + }), + }; + void threeArg; + note( + '(a2) Surface.interpret signature', + '(res, cfg) => SurfaceOutcome', + ); + } + + // ── (b) `buildRequest` runs ONCE per call on the retry path ─────────────────────────────── + // So the other surface hook that touches the request cannot rewrite it between attempts either. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + let builds = 0; + const counted: Surface = { + ...retryingSurface(0), + buildRequest: (_cfg, _input, base) => { + builds++; + return base; + }, + }; + const call = stitch({ + url: URL, + method: 'POST', + kind: counted, + adapter: db.adapter(), + clock, + retry: { attempts: 3 }, + }); + const p = call.safe({ body: dynamoBody(SIX) }); + await clock.advance(60_000); + await p; + check('(b) requests sent', db.requests.length, 3); + check('(b) times `buildRequest` ran', builds, 1); + } + + // ── (b2) on the PAGINATE path it runs once per page — but only on `next`'s input ─────────── + // `paginated` rebuilds the request each round (engine.ts:940), so `buildRequest` does see the + // new body. It is downstream of `paginate.next`, not an independent rewrite seam. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + let builds = 0; + const bodies: string[] = []; + const counted: Surface = { + id: 'dynamo-batch', + buildRequest: (_cfg, _input, base) => { + builds++; + bodies.push( + ( + (base.body as { RequestItems?: { id: string }[] }) + ?.RequestItems ?? [] + ) + .map((i) => i.id) + .join(''), + ); + return base; + }, + }; + const call = stitch({ + url: URL, + method: 'POST', + kind: counted, + adapter: db.adapter(), + clock, + paginate: { + next: (prev) => { + const residue = unprocessedOf(prev); + return residue.length + ? { body: { RequestItems: residue } } + : undefined; + }, + items: (v) => processedOf(v), + }, + }); + await call.safe({ body: dynamoBody(SIX) }); + check('(b2) pages fetched', db.requests.length, 3); + // One extra build: `paginated` builds the request once for the `start` event (engine.ts:936) + // and again at the head of each round (:940). + check('(b2) times `buildRequest` ran', builds, 4); + check('(b2) bodies it saw', bodies.join(','), 'abcdef,abcdef,cdef,ef'); + } + + // ── (c) the `onRequest` HOOK can rewrite the body, and does it inside the chain ──────────── + // `onRequest` is awaited on the per-attempt clone, before the transport runs (engine.ts:646-670). + // Assigning `ctx.req.body` there changes what attempt N+1 sends. This is the seam C7 is built on. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + let residue: { id: string }[] | null = null; + const evts: StitchEvent[] = []; + const call = stitch({ + url: URL, + method: 'POST', + kind: retryingSurface(500), + adapter: db.adapter(), + clock, + retry: { attempts: 4 }, + hooks: { + onRequest: (ctx) => { + if (ctx.req && residue) + ctx.req.body = { RequestItems: residue }; + }, + onResponse: (ctx) => { + const left = unprocessedOf(ctx.res?.body); + residue = left.length > 0 ? left : null; + }, + }, + }); + const consume = (async (): Promise => { + for await (const e of call.stream({ body: dynamoBody(SIX) })) + evts.push(e); + })(); + await clock.advance(60_000); + await consume; + + check('(c) attempts made', db.requests.length, 3); + check( + '(c) what each attempt SENT', + db.requests.map((q) => q.ids.join('')).join(' → '), + 'abcdef → cdef → ef', + ); + check('(c) DUPLICATE WRITES', db.duplicateWrites, 0); + check('(c) every item landed', db.landed.join(''), 'abcdef'); + check( + '(c) the engine counted the rounds as retry attempts', + evts.filter((e) => e.type === 'progress' && e.phase === 'retry') + .length, + 2, + ); + const result = evts.find((e) => e.type === 'result'); + check( + '(c) attempts reported on the result', + result && 'attempts' in result ? result.attempts : undefined, + 3, + ); + } + + // ── (c2) …but the rewrite must ASSIGN, never mutate in place ────────────────────────────── + // `cloneReq` copies the request and its headers, and shares `body` by reference + // (engine.ts:261-264) — and that reference is the CALLER'S object. Editing `ctx.req.body` in + // place therefore reaches back into the array the caller passed in. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const callerBody = dynamoBody(SIX); + let residue: { id: string }[] | null = null; + const call = stitch({ + url: URL, + method: 'POST', + kind: retryingSurface(0), + adapter: db.adapter(), + clock, + retry: { attempts: 4 }, + hooks: { + onRequest: (ctx) => { + const body = ctx.req?.body as + { RequestItems: { id: string }[] } | undefined; + // The in-place spelling — the one that looks equivalent. + if (body && residue) body.RequestItems = residue; + }, + onResponse: (ctx) => { + const left = unprocessedOf(ctx.res?.body); + residue = left.length > 0 ? left : null; + }, + }, + }); + const p = call.safe({ body: callerBody }); + await clock.advance(60_000); + await p; + check('(c2) the loop still worked', db.landed.join(''), 'abcdef'); + check( + '(c2) the CALLER’S body object after the call', + callerBody.RequestItems.map((i) => i.id).join(''), + 'ef', + ); + note( + '(c2) → the caller handed in 6 items and got their array rewritten to the last residue', + '', + ); + } + + // ── (d) `kind.execute` can loop freely — BELOW the resilience chain ──────────────────────── + // A surface may replace the transport (surface.ts:109-118). A loop written there can do + // anything, but the engine sees ONE call: one attempt, no retry events, and the rounds it made + // are invisible to the run report. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const transport = db.adapter(); + const looping: Surface = { + id: 'dynamo-batch-execute', + execute: async (req) => { + let body = req.body as { RequestItems: { id: string }[] }; + let res = await transport({ ...req, body }); + for (let i = 0; i < 5 && unprocessedOf(res.body).length; i++) { + await clock.sleep(100 * 2 ** i); + body = { RequestItems: unprocessedOf(res.body) }; + res = await transport({ ...req, body }); + } + return res; + }, + }; + const evts: StitchEvent[] = []; + const call = stitch({ + url: URL, + method: 'POST', + kind: looping, + clock, + retry: { attempts: 4 }, + }); + const consume = (async (): Promise => { + for await (const e of call.stream({ body: dynamoBody(SIX) })) + evts.push(e); + })(); + await clock.advance(60_000); + await consume; + + check( + '(d) requests the provider really received', + db.requests.length, + 3, + ); + check('(d) DUPLICATE WRITES', db.duplicateWrites, 0); + const result = evts.find((e) => e.type === 'result'); + check( + '(d) attempts the engine reported', + result && 'attempts' in result ? result.attempts : undefined, + 1, + ); + check( + '(d) `retry` progress events', + evts.filter((e) => e.type === 'progress' && e.phase === 'retry') + .length, + 0, + ); + check( + '(d) `request` progress events (one per real round?)', + evts.filter((e) => e.type === 'progress' && e.phase === 'request') + .length, + 1, + ); + note( + '(d) → the loop works and the observability is gone; same trade as a hand-rolled while', + '', + ); + } + + finish( + 'C4', + 'a surface CANNOT rewrite the request between attempts — `SurfaceOutcome.retry` resends the identical body (3 attempts × the same 6 items, 4 duplicate writes) and `buildRequest` runs once per call — but the `onRequest` HOOK can: assigning `ctx.req.body` there turns the same retry budget into a shrinking-subset loop with ZERO duplicate writes, still inside the resilience chain', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c5-terminal-vs-retryable.ts b/docs/scenarios/proofs/batch-partial-failure/c5-terminal-vs-retryable.ts new file mode 100644 index 00000000..ee5f740d --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c5-terminal-vs-retryable.ts @@ -0,0 +1,194 @@ +// C5 — Elasticsearch `_bulk` answers 200 with a PER-ITEM status: some `429` (the queue is full — +// resend it) and some `400` (a mapping error — resending it forever is elasticsearch-py#1004's +// hang). Can the loop partition them, so the terminal ones are not retried? Measured by counting +// how many requests carried the poison document. +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c5-terminal-vs-retryable.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { BulkItem } from './fake-batch'; +import { FakeElastic, bulkBody, bulkItemsOf } from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://es.example.com/_bulk'; +// `bad` is a mapping error: it will answer 400 forever, however long you wait. +const DOCS = ['s1', 't1', 'bad', 't2', 's2']; + +const landed = (items: BulkItem[]): BulkItem[] => + items.filter((i) => i.index.status < 300); +const retryable = (items: BulkItem[]): BulkItem[] => + items.filter((i) => i.index.status === 429); +const terminal = (items: BulkItem[]): BulkItem[] => + items.filter((i) => i.index.status >= 400 && i.index.status !== 429); +const resend = (items: BulkItem[]): { operations: { id: string }[] } => ({ + operations: items.map((i) => ({ id: i.index._id })), +}); + +async function main(): Promise { + heading( + 'C5 — separating retryable (429) from terminal (400) per-item failures', + ); + + // ── (a) the PARTITIONED loop: only the 429s go back ──────────────────────────────────────── + { + const clock = manualClock(); + const es = new FakeElastic({ clock, terminal: ['bad'], accepts: 1 }); + const dead: BulkItem[] = []; + const call = stitch({ + url: URL, + method: 'POST', + adapter: es.adapter(), + clock, + paginate: { + next: (prevBody) => { + const items = bulkItemsOf(prevBody); + dead.push(...terminal(items)); + const again = retryable(items); + return again.length > 0 + ? { body: resend(again) } + : undefined; + }, + items: (value) => landed(bulkItemsOf(value)), + pages: 20, + }, + }); + const r = await call.safe({ body: bulkBody(DOCS) }); + + check('(a) requests made', es.requests.length, 4); + check( + '(a) what each request carried', + es.requests.map((q) => q.ids.join('+')).join(' → '), + 's1+t1+bad+t2+s2 → t1+t2+s2 → t2+s2 → s2', + ); + check( + '(a) requests that carried the 400 document', + es.requests.filter((q) => q.ids.includes('bad')).length, + 1, + ); + check( + '(a) the 400 document was never written', + es.writeCount('bad'), + 0, + ); + check( + '(a) every retryable document landed', + es.landed.sort().join(','), + 's1,s2,t1,t2', + ); + check('(a) DUPLICATE WRITES', es.duplicateWrites, 0); + check('(a) call ok', r.ok, true); + check('(a) aggregated successes', (r.data as BulkItem[]).length, 4); + check( + '(a) terminal failures the loop set aside', + dead.map((d) => d.index._id).join(','), + 'bad', + ); + note('(a) reason recorded for it', dead[0]?.index.error?.type); + } + + // ── (b) the NAIVE loop: "resend everything that failed" ──────────────────────────────────── + // The same loop with `status >= 400` instead of `=== 429`. The poison document rides every + // round. It stops only because the last round landed nothing — which is the zero-page break + // (C2d), not a decision about the failure — and the call still reports success. + { + const clock = manualClock(); + const es = new FakeElastic({ clock, terminal: ['bad'], accepts: 1 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: es.adapter(), + clock, + paginate: { + next: (prevBody) => { + const failed = bulkItemsOf(prevBody).filter( + (i) => i.index.status >= 400, + ); + return failed.length > 0 + ? { body: resend(failed) } + : undefined; + }, + items: (value) => landed(bulkItemsOf(value)), + pages: 20, + }, + }); + const r = await call.safe({ body: bulkBody(DOCS) }); + + check('(b) requests made', es.requests.length, 5); + check( + '(b) requests that carried the 400 document', + es.requests.filter((q) => q.ids.includes('bad')).length, + 5, + ); + check( + '(b) wasted requests chasing a document that can never land', + 5 - 1, + 4, + ); + check( + '(b) the final round landed nothing', + es.requests[4]?.accepted.length, + 0, + ); + check('(b) …and the call still reported success', r.ok, true); + note( + '(b) → without the zero-page break this would run to the `pages` cap', + '', + ); + } + + // ── (c) the same partition holds when the terminal item is ALONE in the residue ──────────── + // Here every retryable doc lands on round 1, so round 2 would be "just the 400". The + // partitioned `next` returns `undefined` instead: one request, and no hang. + { + const clock = manualClock(); + const es = new FakeElastic({ clock, terminal: ['bad'], accepts: 99 }); + const dead: BulkItem[] = []; + const call = stitch({ + url: URL, + method: 'POST', + adapter: es.adapter(), + clock, + paginate: { + next: (prevBody) => { + const items = bulkItemsOf(prevBody); + dead.push(...terminal(items)); + const again = retryable(items); + return again.length > 0 + ? { body: resend(again) } + : undefined; + }, + items: (value) => landed(bulkItemsOf(value)), + pages: 20, + }, + }); + const r = await call.safe({ body: bulkBody(DOCS) }); + check('(c) requests made', es.requests.length, 1); + check( + '(c) documents written', + es.landed.sort().join(','), + 's1,s2,t1,t2', + ); + check( + '(c) terminal failures set aside', + dead.map((d) => d.index._id).join(','), + 'bad', + ); + check('(c) call ok', r.ok, true); + check( + '(c) does the RESULT mention the failed document?', + JSON.stringify(r.data).includes('bad'), + false, + ); + note( + '(c) → the partition is correct, and the caller only learns about `bad` from the closure', + '', + ); + } + + finish( + 'C5', + 'the partition IS expressible in `paginate.next` — filtering to `status === 429` sends the 400 document exactly ONCE (vs 5 times for the naive "resend everything that failed"), never writes it, and terminates — but the terminal items reach the caller only through a closure the user holds, never through the result', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c6-residue-reachability.ts b/docs/scenarios/proofs/batch-partial-failure/c6-residue-reachability.ts new file mode 100644 index 00000000..53b247bf --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c6-residue-reachability.ts @@ -0,0 +1,301 @@ +// C6 — DECIDING CLAIM. When the loop gives up — the `pages` cap is hit, or a round lands nothing — +// can the caller obtain THE RESIDUE: the actual items that never landed? Every channel the public +// API offers is asked in turn: the return value, the thrown error, `.inspect()`, `.report()`, the +// event stream, a trace sink, the `paginate.next` closure and the `hooks.onResponse` closure. +// +// This is the Logstash failure mode (elastic/logstash#1631, "rejected docs … silently lost"). +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c6-residue-reachability.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import type { BatchItem } from './fake-batch'; +import { + FakeDynamo, + dynamoBody, + processedOf, + unprocessedOf, +} from './fake-batch'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const SIX = ['a', 'b', 'c', 'd', 'e', 'f']; +const ids = (items: { id: string }[]): string => + items.map((i) => i.id).join(''); + +/** + * One place all of C6's runs come from: a table that lands ONE item per request, a 3-page cap, and + * every observation channel wired at once. After 3 rounds `d,e,f` have never been written — + * that string is the answer every channel below is checked against. + */ +function cappedRun(): { + call: ReturnType; + db: FakeDynamo; + clock: ReturnType; + seenByNext: BatchItem[][]; + seenByHook: BatchItem[][]; + traced: StitchEvent[]; +} { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const seenByNext: BatchItem[][] = []; + const seenByHook: BatchItem[][] = []; + const traced: StitchEvent[] = []; + const sink: TraceSink = { + handle: (event) => { + traced.push(event); + }, + }; + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + trace: sink, + hooks: { + onResponse: (ctx) => { + seenByHook.push(unprocessedOf(ctx.res?.body)); + }, + }, + paginate: { + next: (prevBody) => { + const residue = unprocessedOf(prevBody); + seenByNext.push(residue); + return residue.length > 0 + ? { body: { RequestItems: residue } } + : undefined; + }, + items: (value) => processedOf(value), + pages: 3, + }, + }); + return { call, db, clock, seenByNext, seenByHook, traced }; +} + +async function main(): Promise { + heading('C6 — when the loop gives up, where is the residue?'); + + // ── (a) the RETURN VALUE: a success, carrying only what landed ───────────────────────────── + // `paginated` breaks out of the loop on `page >= max` and falls straight through to the + // `result` event (engine.ts:984-1010). There is no "we stopped early" signal of any kind. + // Each observation gets its OWN run: the ledgers below count invocations, and sharing one + // stitch across probes would count them twice. + { + const { call, db } = cappedRun(); + const r = await call.safe({ body: dynamoBody(SIX) }); + + check('(a) rounds fired (cap was 3)', db.requests.length, 3); + check('(a) items that landed', db.landed.join(''), 'abc'); + check( + '(a) items that never landed', + SIX.filter((id) => db.writeCount(id) === 0).join(''), + 'def', + ); + check('(a) call ok', r.ok, true); + check('(a) error', r.error, null); + check( + '(a) data — the aggregated successes', + ids(r.data as BatchItem[]), + 'abc', + ); + check( + '(a) does the result contain the residue anywhere?', + JSON.stringify(r.data).includes('"d"'), + false, + ); + } + + // ── (b) the THROWN error: there is none — the bare call resolves ─────────────────────────── + { + const { call } = cappedRun(); + const thrown = await call({ body: dynamoBody(SIX) }).then( + () => 'resolved', + (e: unknown) => `threw ${String(e)}`, + ); + check('(b) awaiting the stitch directly', thrown, 'resolved'); + } + + // ── (e) the EVENT STREAM, and (f) a TRACE SINK fed by the same events ───────────────────── + { + const { call, traced } = cappedRun(); + const evts: StitchEvent[] = []; + for await (const e of call.stream({ body: dynamoBody(SIX) })) + evts.push(e); + const paginateEvts = evts.filter( + (e) => e.type === 'progress' && e.phase === 'paginate', + ); + check('(e) `paginate` progress events', paginateEvts.length, 3); + note( + '(e) what the last one says', + paginateEvts[2] && 'detail' in paginateEvts[2] + ? paginateEvts[2].detail + : '', + ); + check( + '(e) any event carrying an unprocessed item', + evts.some((e) => JSON.stringify(e).includes('UnprocessedItems')), + false, + ); + check( + '(e) the run ended with a `done` event that says ok', + evts.some((e) => e.type === 'done' && e.ok), + true, + ); + check('(f) trace events captured', traced.length > 0, true); + check( + '(f) any traced event carrying the residue', + traced.some((e) => JSON.stringify(e).includes('UnprocessedItems')), + false, + ); + } + + // ── (g) the `paginate.next` closure: it is STALE BY ONE ROUND ────────────────────────────── + // `next` is called only when the loop is going to continue (engine.ts:984-986), so on the round + // that hits the cap it never runs. A residue ledger built there is not merely incomplete — it + // names `c`, which DID land. + // ── (h) `hooks.onResponse` fires on every response, including the last ───────────────────── + { + const { call, db, seenByNext, seenByHook } = cappedRun(); + await call.safe({ body: dynamoBody(SIX) }); + const lastSeen = seenByNext[seenByNext.length - 1] ?? []; + check('(g) times `next` ran for 3 rounds', seenByNext.length, 2); + check( + '(g) residue the `next` ledger would report', + ids(lastSeen), + 'cdef', + ); + check( + '(g) …the TRUE residue', + SIX.filter((id) => db.writeCount(id) === 0).join(''), + 'def', + ); + check( + '(g) items it wrongly reports as lost', + ids(lastSeen.filter((i) => db.writeCount(i.id) > 0)), + 'c', + ); + + const lastHook = seenByHook[seenByHook.length - 1] ?? []; + check('(h) times `onResponse` ran', seenByHook.length, 3); + check('(h) residue the hook ledger reports', ids(lastHook), 'def'); + } + + // ── (c) `.inspect()` — the pre-validation body is the AGGREGATE, not the residue ─────────── + { + const { call } = cappedRun(); + const seen = await call.inspect({ body: dynamoBody(SIX) }); + check('(c) inspect error', String(seen.error), 'null'); + check('(c) inspect raw', ids(seen.raw as BatchItem[]), 'abc'); + check( + '(c) does `raw` carry the residue?', + JSON.stringify(seen.raw).includes('"d"'), + false, + ); + } + + // ── (d) `.report()` — the run report, for a run that lost half the batch ─────────────────── + { + const { call } = cappedRun(); + const rep = await call.report({ body: dynamoBody(SIX) }); + check('(d) report says the run failed?', rep.error !== null, false); + check('(d) attempts reported', rep.attempts, 1); + check( + '(d) report mentions the residue', + JSON.stringify(rep).includes('UnprocessedItems'), + false, + ); + note('(d) status on the report', rep.status); + } + + // ── (i) `verdict.flag` cannot express "the residue must be empty" ────────────────────────── + // `flag` fails a 200 whose flag is PRESENT and FALSY (surface.ts:174-191). `UnprocessedItems` + // is an array: `[]` and `[{…}]` are both truthy, so the flag is inert either way. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + verdict: { flag: 'UnprocessedItems' }, + }); + const r = await call.safe({ body: dynamoBody(SIX) }); + check( + '(i) call with a non-empty residue and `verdict.flag`', + r.ok, + true, + ); + check('(i) requests made', db.requests.length, 1); + } + + // ── (j) the one channel that needs no closure: RECONSTRUCT it from the successes ─────────── + // The caller knows what it sent, and `items` hands back what landed. The set difference IS the + // residue — as long as the items are identifiable and nothing reshaped the aggregate. It is a + // recovery, not a report: the call still resolved successfully, so nothing prompts the check. + { + const { call, db } = cappedRun(); + const sent = SIX; + const r = await call.safe({ body: dynamoBody(sent) }); + const landedIds = new Set((r.data as BatchItem[]).map((i) => i.id)); + const residue = sent.filter((id) => !landedIds.has(id)); + check( + '(j) residue reconstructed by set difference', + residue.join(''), + 'def', + ); + check( + '(j) …and it matches what the provider never saw', + residue.join(''), + SIX.filter((id) => db.writeCount(id) === 0).join(''), + ); + } + + // ── (k) the one built-in that can at least make the loss LOUD: an `output` contract ──────── + // A contract on the aggregate — "I sent 6, I expect 6 back" — fails the call when the loop + // stopped short. It reports THAT items were lost, never WHICH: the error carries the + // aggregated successes, and the residue is still nowhere. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const call = stitch({ + url: URL, + method: 'POST', + adapter: db.adapter(), + clock, + output: (v: unknown) => Array.isArray(v) && v.length === SIX.length, + paginate: { + next: (prevBody) => { + const residue = unprocessedOf(prevBody); + return residue.length > 0 + ? { body: { RequestItems: residue } } + : undefined; + }, + items: (value) => processedOf(value), + pages: 3, + }, + }); + const r = await call.safe({ body: dynamoBody(SIX) }); + check('(k) the call now FAILS', r.ok, false); + check('(k) message', r.error?.message, 'contract violation (drift)'); + check( + '(k) does the error carry the residue?', + JSON.stringify(r.error?.body ?? null).includes('"d"'), + false, + ); + note( + '(k) → loud, but it needs the caller to know the expected count up front', + '', + ); + } + + finish( + 'C6', + 'the residue is UNREACHABLE from every channel the engine owns: the call resolves ok with only the successes, there is no error, `.inspect()`/`.report()`/the event stream/a trace sink never carry it, and the `paginate.next` ledger is stale by one round (it reports `cdef` when the true residue is `def`). Only a user-held `hooks.onResponse` closure — or reconstructing the difference from the successes — recovers it; an `output` contract can make the loss loud but still cannot name the items', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/c7-assembled-solution.ts b/docs/scenarios/proofs/batch-partial-failure/c7-assembled-solution.ts new file mode 100644 index 00000000..84514493 --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/c7-assembled-solution.ts @@ -0,0 +1,513 @@ +// C7 — the best answer the public API supports, written, run, and compared against the hand-rolled +// `while` loop it replaces. The seam is `Surface.interpret` + `hooks.onRequest` (see +// `batch-retry-surface.ts`); the comparison is not "is it prettier" but "does it keep +// `timeout.total`, the circuit breaker and the trace that the hand-rolled loop loses" — measured, +// not assumed. +// +// pnpm exec tsx docs/scenarios/proofs/batch-partial-failure/c7-assembled-solution.ts +import { stitch, systemClock } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import type { BatchLedger } from './batch-retry-surface'; +import { batchRetry } from './batch-retry-surface'; +import type { BatchItem } from './fake-batch'; +import { + FakeDynamo, + dynamoBody, + processedOf, + unprocessedOf, +} from './fake-batch'; +import { check, checkAtMost, finish, heading, note } from './harness'; + +import { readFileSync } from 'node:fs'; + +const ENDPOINT = 'https://dynamodb.us-east-1.amazonaws.com/batch'; +const SIX = ['a', 'b', 'c', 'd', 'e', 'f']; +const ids = (items: BatchItem[]): string => items.map((i) => i.id).join(''); + +/** The three readers the loop needs for a DynamoDB-shaped response. */ +const dynamoReaders = { + residueOf: unprocessedOf, + landedOf: processedOf, + bodyOf: (items: BatchItem[]) => ({ RequestItems: items }), +}; + +/** Count real lines of code in a file (or a `#region`), ignoring blanks and comment-only lines. */ +function codeLines(file: string, region?: string): number { + const src = readFileSync(new URL(file, import.meta.url), 'utf8'); + let lines = src.split('\n'); + if (region) { + const from = lines.findIndex((l) => l.includes(`#region ${region}`)); + const to = lines.findIndex((l) => l.includes(`#endregion ${region}`)); + lines = lines.slice(from + 1, to); + } + return lines.filter((l) => { + const t = l.trim(); + return ( + t !== '' && + !t.startsWith('//') && + !t.startsWith('*') && + !t.startsWith('/*') + ); + }).length; +} + +async function main(): Promise { + heading( + 'C7 — the assembled solution, measured against the hand-rolled loop', + ); + + // ── (a) it lands every item, once, with a wait that grows ───────────────────────────────── + // The table is a real write-capacity bucket (2 units at t=0, refilling 1/s) — the case where + // the wait IS the fix. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, writeUnitsPerSec: 1, burst: 2 }); + const batch = batchRetry({ + ...dynamoReaders, + rounds: 6, + backoff: (round) => 1000 * 2 ** (round - 1), + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: db.adapter(), + clock, + retry: { attempts: 6 }, + }); + const p = call.safe({ body: dynamoBody(SIX) }); + await clock.advance(600_000); + const r = await p; + const out = r.data as BatchLedger; + + check('(a) rounds fired', db.requests.length, 4); + check( + '(a) what each round SENT', + db.requests.map((q) => q.ids.join('')).join(' → '), + 'abcdef → cdef → def → f', + ); + check( + '(a) arrival times (ms)', + db.requests.map((q) => q.at).join(','), + '0,1000,3000,7000', + ); + check('(a) DUPLICATE WRITES', db.duplicateWrites, 0); + check('(a) every item landed', db.landed.join(''), 'abcdef'); + check('(a) call ok', r.ok, true); + check('(a) ledger.landed', ids(out.landed), 'abcdef'); + check('(a) ledger.residue', ids(out.residue), ''); + check('(a) ledger.gaveUp', out.gaveUp, false); + } + + // ── (a2) it survives a round that lands NOTHING — the case `paginate` cannot ─────────────── + // Same table, a curve that starts below the refill period. Rounds 2-4 land zero items; the + // loop keeps going instead of terminating successfully with the batch half-written (C2d). + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, writeUnitsPerSec: 1, burst: 2 }); + const batch = batchRetry({ + ...dynamoReaders, + rounds: 8, + backoff: (round) => 100 * 2 ** (round - 1), + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: db.adapter(), + clock, + retry: { attempts: 8 }, + }); + const p = call.safe({ body: dynamoBody(SIX) }); + await clock.advance(600_000); + const r = await p; + const out = r.data as BatchLedger; + check( + '(a2) rounds that landed nothing', + db.requests.filter((q) => q.accepted.length === 0).length, + 3, + ); + check('(a2) every item still landed', db.landed.join(''), 'abcdef'); + check('(a2) DUPLICATE WRITES', db.duplicateWrites, 0); + check('(a2) ledger.gaveUp', out.gaveUp, false); + } + + // ── (b) when it runs out of rounds, the RESIDUE IS THE RESULT ───────────────────────────── + // The answer to C6 on this seam: the caller gets the items that never landed, in the payload, + // without holding a closure of their own. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const batch = batchRetry({ + ...dynamoReaders, + rounds: 3, + backoff: () => 1000, + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: db.adapter(), + clock, + retry: { attempts: 3 }, + }); + const p = call.safe({ body: dynamoBody(SIX) }); + await clock.advance(600_000); + const r = await p; + const out = r.data as BatchLedger; + + check('(b) rounds fired', db.requests.length, 3); + check('(b) call ok', r.ok, true); + check('(b) ledger.landed', ids(out.landed), 'abc'); + check( + '(b) ledger.residue — THE ITEMS THAT NEVER LANDED', + ids(out.residue), + 'def', + ); + check('(b) ledger.gaveUp', out.gaveUp, true); + check( + '(b) it matches what the provider never saw', + ids(out.residue), + SIX.filter((id) => db.writeCount(id) === 0).join(''), + ); + } + + // ── (c) what the engine still sees: attempts, retry events, trace ───────────────────────── + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const traced: StitchEvent[] = []; + const sink: TraceSink = { + handle: (e) => { + traced.push(e); + }, + }; + const batch = batchRetry({ + ...dynamoReaders, + rounds: 4, + backoff: () => 500, + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: db.adapter(), + clock, + trace: sink, + retry: { attempts: 4 }, + }); + const evts: StitchEvent[] = []; + const consume = (async (): Promise => { + for await (const e of call.stream({ body: dynamoBody(SIX) })) + evts.push(e); + })(); + await clock.advance(600_000); + await consume; + + const result = evts.find((e) => e.type === 'result'); + check('(c) rounds fired', db.requests.length, 3); + check( + '(c) `start` events (one logical call)', + evts.filter((e) => e.type === 'start').length, + 1, + ); + check( + '(c) attempts on the result', + result && 'attempts' in result ? result.attempts : undefined, + 3, + ); + check( + '(c) `retry` progress events', + evts.filter((e) => e.type === 'progress' && e.phase === 'retry') + .length, + 2, + ); + check('(c) trace saw the same run', traced.length, evts.length); + note( + '(c) retry details in the trace', + evts + .filter((e) => e.type === 'progress' && e.phase === 'retry') + .map((e) => ('detail' in e ? e.detail : '')) + .join(' | '), + ); + } + + // ── (c2) the circuit breaker still sees a broken host ────────────────────────────────────── + // `interpret` composes `verdictOf` first, so a 500 is a transport failure, not a batch + // envelope: two failed calls open the breaker and the third never reaches the network. + { + const clock = manualClock(); + let hits = 0; + const batch = batchRetry({ + ...dynamoReaders, + rounds: 1, + backoff: () => 0, + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: async () => { + hits++; + return { status: 500, headers: {}, body: { message: 'boom' } }; + }, + clock, + circuit: { failures: 2, cooldown: 30_000 }, + }); + const a = await call.safe({ body: dynamoBody(SIX) }); + const b = await call.safe({ body: dynamoBody(SIX) }); + const c = await call.safe({ body: dynamoBody(SIX) }); + check('(c2) requests that reached the host', hits, 2); + check( + '(c2) first two failures', + `${a.error?.message}/${b.error?.message}`, + 'HTTP 500/HTTP 500', + ); + check( + '(c2) third call short-circuited', + c.error?.message, + 'circuit open', + ); + } + + // ── (d) `timeout.total` — the ONE wall-clock measurement in this suite ───────────────────── + // `timeout.total` is deliberately not driven by the injected clock (engine.ts:482), so this + // runs on real timers. Bounds are 4× clear of the real numbers. + { + const slowAdapter = (perRound: number) => { + const db = new FakeDynamo({ clock: systemClock, accepts: 1 }); + const inner = db.adapter(); + return { + db, + adapter: async (req: Parameters[0]) => { + await new Promise((r) => setTimeout(r, perRound)); + return inner(req); + }, + }; + }; + + // The assembled solution: one logical call, so `timeout.total` bounds the WHOLE loop. + const s = slowAdapter(40); + const batch = batchRetry({ + ...dynamoReaders, + rounds: 6, + backoff: () => 10, + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: s.adapter, + timeout: { total: 100 }, + retry: { attempts: 6 }, + }); + const t0 = Date.now(); + const r = await call.safe({ body: dynamoBody(SIX) }); + const elapsed = Date.now() - t0; + check('(d) call ok', r.ok, false); + checkAtMost( + '(d) rounds it managed before the deadline (6 needed)', + s.db.requests.length, + 4, + ); + checkAtMost('(d) elapsed ms (budget was 100)', elapsed, 400); + note('(d) error', r.error?.message); + } + + // ── (e) the hand-rolled `while` loop, for comparison ─────────────────────────────────────── + { + // #region handrolled + const runHandRolled = async ( + call: ReturnType, + body: { RequestItems: BatchItem[] }, + rounds: number, + wait: (round: number) => Promise, + ): Promise> => { + const ledger: BatchLedger = { + landed: [], + residue: body.RequestItems, + terminal: [], + rounds: 0, + gaveUp: false, + }; + while (ledger.residue.length > 0) { + if (ledger.rounds > 0) await wait(ledger.rounds); + const res = await call({ + body: { RequestItems: ledger.residue }, + }); + ledger.rounds += 1; + ledger.landed.push(...processedOf(res)); + ledger.residue = unprocessedOf(res); + if (ledger.residue.length > 0 && ledger.rounds >= rounds) { + ledger.gaveUp = true; + break; + } + } + return ledger; + }; + // #endregion handrolled + + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 2 }); + const traced: StitchEvent[] = []; + const call = stitch({ + url: ENDPOINT, + method: 'POST', + adapter: db.adapter(), + clock, + timeout: { total: 100 }, + trace: { + handle: (e) => { + traced.push(e); + }, + }, + }); + const p = runHandRolled(call, dynamoBody(SIX), 6, (round) => + clock.sleep(1000 * round), + ); + await clock.advance(600_000); + const out = await p; + + check('(e) rounds fired', db.requests.length, 3); + check('(e) DUPLICATE WRITES', db.duplicateWrites, 0); + check('(e) every item landed', ids(out.landed), 'abcdef'); + check('(e) residue', ids(out.residue), ''); + // What it loses: the engine sees three unrelated calls, not one operation that retried. + check( + '(e) `start` events (one per round, not one per operation)', + traced.filter((e) => e.type === 'start').length, + 3, + ); + check( + '(e) `retry` progress events', + traced.filter((e) => e.type === 'progress' && e.phase === 'retry') + .length, + 0, + ); + check( + '(e) attempts each call reported', + traced.filter((e) => e.type === 'result' && e.attempts === 1) + .length, + 3, + ); + check( + '(e) the 3s spent waiting between rounds, as the engine saw it', + traced.filter( + (e) => e.type === 'progress' && e.phase === 'throttled', + ).length, + 0, + ); + note( + '(e) `timeout: { total: 100 }` bounds each ROUND here, never the loop', + '', + ); + } + + // ── (f) the cost of the ledger: it is per-STITCH, so concurrent calls corrupt it ────────── + // The footgun that comes with this design, and it is the worst kind: two batches through one + // stitch, both resolve SUCCESSFULLY, and two rows are never written by anybody. + { + const clock = manualClock(); + const db = new FakeDynamo({ clock, accepts: 1 }); + const batch = batchRetry({ + ...dynamoReaders, + rounds: 6, + backoff: () => 100, + }); + const call = stitch({ + url: ENDPOINT, + method: 'POST', + kind: batch.kind, + hooks: batch.hooks, + adapter: db.adapter(), + clock, + retry: { attempts: 6 }, + }); + const p = Promise.all([ + call.safe({ body: dynamoBody(['a', 'b', 'c']) }), + call.safe({ body: dynamoBody(['x', 'y', 'z']) }), + ]); + await clock.advance(600_000); + const [first, second] = await p; + const l1 = first?.data as BatchLedger; + const l2 = second?.data as BatchLedger; + check( + '(f) both calls resolved ok', + `${first?.ok}/${second?.ok}`, + 'true/true', + ); + check( + '(f) neither reports giving up', + `${l1.gaveUp}/${l2.gaveUp}`, + 'false/false', + ); + check( + '(f) neither reports a residue', + `${ids(l1.residue)}/${ids(l2.residue)}`, + '/', + ); + check( + '(f) items the two ledgers claim landed', + `${ids(l1.landed)}/${ids(l2.landed)}`, + 'axyz/axyz', + ); + check( + '(f) items the provider really wrote', + db.landed.sort().join(''), + 'axyz', + ); + check( + '(f) items SILENTLY LOST', + ['a', 'b', 'c', 'x', 'y', 'z'] + .filter((id) => db.writeCount(id) === 0) + .join(''), + 'bc', + ); + note( + '(f) what the shared ledger actually sent', + db.requests.map((q) => q.ids.join('')).join(' → '), + ); + note( + '(f) → both callers were handed the OTHER batch’s items and told everything landed', + '', + ); + } + + // ── (g) size of each answer, in real lines of code ──────────────────────────────────────── + { + const surfaceLines = codeLines('./batch-retry-surface.ts'); + const loopLines = codeLines('./batch-retry-surface.ts', 'loop'); + const handRolledLines = codeLines( + './c7-assembled-solution.ts', + 'handrolled', + ); + note('(g) the assembled solution, whole file', `${surfaceLines} lines`); + note( + '(g) …its loop alone, types and options excluded', + `${loopLines} lines`, + ); + note('(g) the hand-rolled while loop', `${handRolledLines} lines`); + check( + '(g) is the StitchAPI version smaller?', + loopLines < handRolledLines, + false, + ); + } + + finish( + 'C7', + 'the loop IS assemblable on `Surface.interpret` + `hooks.onRequest` — 4 rounds, zero duplicate writes, an exponential wait, the residue handed back as DATA, and `attempts`/`retry` events/circuit/`timeout.total` all still working — but it is MORE user code than the hand-rolled `while` loop it replaces, and its ledger makes the stitch single-call-at-a-time', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/batch-partial-failure/fake-batch.ts b/docs/scenarios/proofs/batch-partial-failure/fake-batch.ts new file mode 100644 index 00000000..e1e758de --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/fake-batch.ts @@ -0,0 +1,259 @@ +// Two fake, in-memory batch-write providers that report PER-ITEM failure inside an HTTP 200: +// +// - {@link FakeDynamo} — DynamoDB `BatchWriteItem`: `POST /batch` with `{ RequestItems }` answers +// `200 { Processed, UnprocessedItems }`. Optionally driven by a real WRITE-CAPACITY BUCKET +// (AWS's actual cause of `UnprocessedItems`) refilling off the injected clock, so "retry +// immediately and you will simply be throttled again" is a measurable property, not a slogan. +// - {@link FakeElastic} — Elasticsearch `_bulk`: `200 { errors: true, items: [{ index: { status } }] }` +// with a MIX of `429` (retryable) and `400` (terminal, a mapping error) per item. +// +// Both count how many times EACH INDIVIDUAL ITEM was written, so duplicate application is measured +// rather than argued about: `writeCount('a')` is 3 if the client wrote `a` three times. +// +// Everything is driven by an INJECTED {@link Clock} and nothing touches the network. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +/** One item in a batch. `id` is what the provider counts writes against. */ +export interface BatchItem { + id: string; + [key: string]: unknown; +} + +/** One recorded hit, as the provider saw it. */ +export interface RecordedRequest { + /** Item ids the client actually sent in THIS request — the measure of "what got replayed". */ + ids: string[]; + /** Item ids this request wrote. */ + accepted: string[]; + /** Virtual time (ms) the request arrived. */ + at: number; +} + +/** Shared write-counting book-keeping. */ +abstract class CountingProvider { + /** Every hit, in order. `requests.length` IS the request count. */ + readonly requests: RecordedRequest[] = []; + protected readonly writes = new Map(); + protected readonly clock: Clock; + + protected constructor(clock: Clock) { + this.clock = clock; + } + + /** How many times this item was WRITTEN. `> 1` is a duplicate application. */ + writeCount(id: string): number { + return this.writes.get(id) ?? 0; + } + + /** Items written at least once, in insertion order. */ + get landed(): string[] { + return [...this.writes.keys()]; + } + + /** Total writes across all items — `landed.length` when nothing was applied twice. */ + get totalWrites(): number { + let n = 0; + for (const c of this.writes.values()) n += c; + return n; + } + + /** Writes beyond the first for each item. **Zero is the only correct value.** */ + get duplicateWrites(): number { + let n = 0; + for (const c of this.writes.values()) n += c - 1; + return n; + } + + protected record(id: string): void { + this.writes.set(id, (this.writes.get(id) ?? 0) + 1); + } +} + +export interface DynamoOptions { + clock: Clock; + /** + * Items accepted per request, flat. Default 1 — every request lands one item and returns the + * rest as `UnprocessedItems`, which is the smallest shape that still needs a real loop. + */ + accepts?: number; + /** + * Write-capacity units refilling per second. Set it to model DynamoDB's ACTUAL cause of + * `UnprocessedItems`: a request writes only as many items as the table has capacity for right + * now, so a client that retries without waiting gets nothing through. Overrides `accepts`. + */ + writeUnitsPerSec?: number; + /** Capacity available at t=0 under `writeUnitsPerSec`. Default 2. */ + burst?: number; +} + +/** + * DynamoDB `BatchWriteItem`, shaped as the vendor shapes it: HTTP **200**, with the items that did + * not land echoed back under `UnprocessedItems` in the same form they were sent (so a resend needs + * no transformation — the property AWS's own docs point at). + */ +export class FakeDynamo extends CountingProvider { + private readonly accepts: number; + private readonly writeUnitsPerSec: number | undefined; + private available: number; + private lastRefillAt: number; + + constructor(opts: DynamoOptions) { + super(opts.clock); + this.accepts = opts.accepts ?? 1; + this.writeUnitsPerSec = opts.writeUnitsPerSec; + this.available = opts.burst ?? 2; + this.lastRefillAt = opts.clock.now(); + } + + private capacityNow(): number { + if (this.writeUnitsPerSec === undefined) return this.accepts; + const nowMs = this.clock.now(); + const elapsedSec = (nowMs - this.lastRefillAt) / 1000; + this.lastRefillAt = nowMs; + if (elapsedSec > 0) + this.available += elapsedSec * this.writeUnitsPerSec; + return Math.floor(this.available); + } + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const items = + (req.body as { RequestItems?: BatchItem[] } | undefined) + ?.RequestItems ?? []; + const room = this.capacityNow(); + const accepted = items.slice(0, Math.max(0, room)); + const unprocessed = items.slice(accepted.length); + if (this.writeUnitsPerSec !== undefined) + this.available -= accepted.length; + for (const item of accepted) this.record(item.id); + this.requests.push({ + ids: items.map((i) => i.id), + accepted: accepted.map((i) => i.id), + at: this.clock.now(), + }); + // THE TRAP: a partial failure is a 200. Nothing in the status line says anything failed. + return { + status: 200, + headers: {}, + body: { Processed: accepted, UnprocessedItems: unprocessed }, + }; + }; + } +} + +export interface ElasticOptions { + clock: Clock; + /** Ids that always answer `400` — a mapping error. Retrying one is a hang, not a fix. */ + terminal?: string[]; + /** Non-terminal docs accepted per request; the rest answer `429`. Default 1. */ + accepts?: number; +} + +/** One entry of an Elasticsearch `_bulk` response's `items` array. */ +export interface BulkItem { + index: { + _id: string; + status: number; + error?: { type: string; reason: string }; + }; +} + +/** + * Elasticsearch `_bulk`, shaped as the vendor shapes it: HTTP **200** with `errors: true` and a + * PER-ITEM `status` — some `429` (the queue is full: retry it), some `400` (the document does not + * fit the mapping: retrying it forever is the hang elasticsearch-py#1004 is about). + * + * The request body here is a plain `{ operations: [...] }` rather than the real NDJSON action/doc + * pairs: the framing is not what this scenario is about, and JSON keeps the proof about the loop. + */ +export class FakeElastic extends CountingProvider { + private readonly terminal: Set; + private readonly accepts: number; + + constructor(opts: ElasticOptions) { + super(opts.clock); + this.terminal = new Set(opts.terminal ?? []); + this.accepts = opts.accepts ?? 1; + } + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const docs = + (req.body as { operations?: BatchItem[] } | undefined) + ?.operations ?? []; + const accepted: string[] = []; + let room = this.accepts; + const items: BulkItem[] = docs.map((doc) => { + if (this.terminal.has(doc.id)) + return { + index: { + _id: doc.id, + status: 400, + error: { + type: 'mapper_parsing_exception', + reason: `failed to parse field [ts] of type [date] in document [${doc.id}]`, + }, + }, + }; + if (room > 0) { + room--; + accepted.push(doc.id); + this.record(doc.id); + return { index: { _id: doc.id, status: 201 } }; + } + return { + index: { + _id: doc.id, + status: 429, + error: { + type: 'es_rejected_execution_exception', + reason: 'rejected execution of bulk request', + }, + }, + }; + }); + this.requests.push({ + ids: docs.map((d) => d.id), + accepted, + at: this.clock.now(), + }); + return { + status: 200, + headers: {}, + body: { + took: 1, + errors: items.some((i) => i.index.status >= 400), + items, + }, + }; + }; + } +} + +/** `{ RequestItems: [...] }` for `ids` — the request body both providers' loops start from. */ +export const dynamoBody = (ids: string[]): { RequestItems: BatchItem[] } => ({ + RequestItems: ids.map((id) => ({ id, payload: `row-${id}` })), +}); + +/** `{ operations: [...] }` for `ids`. */ +export const bulkBody = (ids: string[]): { operations: BatchItem[] } => ({ + operations: ids.map((id) => ({ id, payload: `doc-${id}` })), +}); + +/** Read `UnprocessedItems` off a DynamoDB-shaped 200 body. */ +export const unprocessedOf = (body: unknown): BatchItem[] => + (body as { UnprocessedItems?: BatchItem[] } | null | undefined) + ?.UnprocessedItems ?? []; + +/** Read `Processed` off a DynamoDB-shaped 200 body. */ +export const processedOf = (body: unknown): BatchItem[] => + (body as { Processed?: BatchItem[] } | null | undefined)?.Processed ?? []; + +/** Read the `items` array off an Elasticsearch-shaped `_bulk` body. */ +export const bulkItemsOf = (body: unknown): BulkItem[] => + (body as { items?: BulkItem[] } | null | undefined)?.items ?? []; diff --git a/docs/scenarios/proofs/batch-partial-failure/harness.ts b/docs/scenarios/proofs/batch-partial-failure/harness.ts new file mode 100644 index 00000000..c7c386f9 --- /dev/null +++ b/docs/scenarios/proofs/batch-partial-failure/harness.ts @@ -0,0 +1,56 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED number either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured number is at most `bound`. Used for the ONE wall-clock measurement in this + * suite (`timeout.total` is deliberately wall-clock — engine.ts:482 — so no injected clock can + * drive it); the bound is set 4× clear of the real timing so a slow machine cannot flip it. + */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${actual}${ok ? '' : ` (expected ≤ ${bound})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/conditional-requests-304/README.md b/docs/scenarios/proofs/conditional-requests-304/README.md new file mode 100644 index 00000000..5ee576ff --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/README.md @@ -0,0 +1,160 @@ +# Proofs — the free poll: ETag revalidation and the bodyless 304 + +Runnable evidence for the claims in +[`../../conditional-requests-304.md`](../../conditional-requests-304.md). + +Every script is standalone, offline, and deterministic: it injects a fake GitHub-shaped +conditional-request API through StitchAPI's `adapter` / `Surface.execute` seam and drives every +wait off an injected `manualClock()`, so a ten-poll run at one-minute intervals is exact **virtual** +time — no real sleeping, nothing flaky, no network. + +**The measurement is a pair of counters and a header string.** The whole scenario turns on "how many +of these responses would have cost me rate-limit budget" and "what exactly went out as +`If-None-Match`", so the fake server records both on every hit: `billed` counts every non-304 +(GitHub's own rule), and `validators` is the byte-exact header value it received (`'(none)'` when +absent). `W/"v1"` surviving as `W/"v1"` is a measurement, not an argument, and so is +`billed 2 / requests 10`. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c3-substitute-the-body.ts + +# all of them +for f in docs/scenarios/proofs/conditional-requests-304/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/conditional-requests-304/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ---------------------------- | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `c1-bare-304.ts` | what does a bare stitch do with a 304? | **Silent success carrying nothing.** `ok:true`, `data:undefined`, `error:null`; `await` resolves | +| `c2-replay-the-validator.ts` | can a hook set `If-None-Match`? | **Yes** — and `buildRequest` bakes it once per RUN while `onRequest` runs per ATTEMPT | +| `c3-substitute-the-body.ts` | can a 304 become the cached body? | **YES. `interpret` runs on `[200,304,404]`** — three seams work, `execute` needs no `interpret` | +| `c4-output-schema.ts` | does `output` reject the empty body? | **Yes, hard** — but substitution is upstream of validation, so the schema sees `[1,1]` not `[1,undefined]` | +| `c5-builtin-cache.ts` | can `cache` hold the ETag / revalidate? | **No.** Entry is `{v,s,vary}`; a hit's spine is `[start, cache:hit, result, done]` — no request phase | +| `c6-per-credential.ts` | is one principal's validator kept off another? | **Split.** `tenancy` protects `cache`, nothing protects the ETag store; **bob got alice's body** | +| `c7-weak-validators.ts` | does `W/"v1"` survive byte-exact? | **Yes, both transports.** Also: header names are never case-folded, so `delete` misses the other casing | +| `c8-the-payoff.ts` | what does revalidation actually buy? | **8 of 10 polls free with ZERO staleness.** TTL bills 1/10 and never sees the change | +| `c9-assembled-solution.ts` | best answer, and is it worth it? | **87 lines, ONE seam**, byte-identical to a 79-line hand-rolled twin on all 4 shapes | + +## Files + +- `fake-etag-api.ts` — the server. Plain GET → `200` + body + `ETag`; a matching `If-None-Match` → + **`304` with no body**; a stale one → a fresh `200` + new `ETag`. Weak comparison per RFC 9110 + §8.8.3.2, so `W/"v1"` matches `"v1"`. Knobs for the shapes that matter: `weak` (mint `W/"…"`), + `inodeEtags` (a unique validator on every response — the load-balancer case where revalidation + never succeeds), and `etagScope: 'content'` (validators derived from the representation rather + than the credential, which is what makes a cross-principal replay a data leak rather than a miss). + Exposes both an `Adapter` and a `fetch`-shaped entry point, so C9's two implementations share one + transport contract. +- `revalidate.ts` — **user code**, the assembled answer and the subject of C9's line count. One + `Surface` with one hook (`execute`), plus counters (`revalidated` / `stored` / `orphans` / + `unvalidatable`) that make the two silent-failure modes assertable. +- `hand-rolled.ts` — the same five rules with no StitchAPI in them, feature-matched down to the + bounded store, so the line comparison is honest. +- `clock-store.ts` — a `StitchStore` whose TTL reads the injected clock. Needed because the default + `memoryStore` reads `Date.now()` and ignores `clock` entirely (C5 case e). +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C3 refutes the capture's central worry, and that is the finding.** The capture asks whether a 304 + is "rejected by `classifyStatus`/`verdict` _before_ `interpret` runs", citing scenario 5's + discovery that `interpret` is dead code on streaming surfaces. On the buffered path it is not. + Measured with a counter, `interpret` was called for **`[200, 304, 404]`** on one stitch: + engine.ts:775 sits inside `attemptLoop` and its own comment says it interprets "EVERY response, + including the non-2xx the engine used to throw on". A 304 reaches it doubly easily, because + `classifyStatus` (surface.ts:143-149) only fails `status >= 400` — a 304 was never rejected at all. + This is a legitimate **ACHIEVABLE WITH USER CODE**, not a manufactured one. +- **The best seam is `Surface.execute`, and it is the one the capture does not mention.** Pairing + `interpret` with `hooks.onRequest` works (measured `[1,1,2,2]` from `[200,304,200,304]` at 2 billed + of 4), but the two seams have no channel between them, so the store key has to live in a closure + variable. Under two CONCURRENT calls to different resources on one stitch, the store ended up with + **1 entry instead of 2**, and 3 of the next 4 polls paid full price with nothing to indicate why. + `execute` (engine.ts:666-674) sees a request and its own response in one function call, runs per + attempt, and sits after `cfg.auth.apply` — so it correlates correctly AND can key by credential. +- **`execute` needs no custom `interpret` at all, and that is the neatest part.** The substituted + body rides back on a response whose status is still **304**, and `httpInterpret` hands it to the + caller unchanged, because a 304 is transport-healthy. So `.inspect().status` honestly reports 304 + while `.data.version` is 2. Nothing has to lie about what happened on the wire. +- **C1 is the trap this whole scenario exists to name.** A bare stitch answers a 304 with + `ok: true`, `data: undefined`, `error: null`, and `await issues()` **resolves** with `undefined` + rather than throwing. Through the real `fetchAdapter` the empty body decodes to `undefined` with a + JSON `content-type` (http-adapter.ts:135) and to `""` without one (http-adapter.ts:137) — two + different falsy values, so a caller cannot even guard on one shape. `verdict.accept: [304]`, the + obvious first fix, is a measured no-op. +- **Adding an `output` schema makes it worse before it makes it better.** The same 304 that silently + yielded `undefined` becomes `ok: false` / `contract violation (drift)` / `data: null` the moment a + contract is attached — so a polling loop that "worked" starts erroring when someone adds a schema. + With substitution in place the contract never sees the empty body: the pipeline is `interpret` + (engine.ts:775) → `transform` (1198) → `pick` (1199) → `validateOutput` (1203), measured as the + schema being handed versions `[1, 1]`. +- **The built-in `cache` cannot participate, and the reason is structural.** Its entry is + `{ v, s, vary }` (cache.ts:300-304) and what gets stored is `out.value` — post-`interpret`, + post-`transform`, post-validation (engine.ts:1624). No response header reaches it, so there is + nowhere for an ETag to live. A hit short-circuits everything below the lookup: measured across 3 + calls, hooks fired `[onRequest, onResponse]` **once**, `interpret` ran **1** time, and the event + spine on a hit is `[start, cache:hit, result, done]` — there is no `request` phase to conditionalise. + `revalidateOnHit` (cache.ts:441) is a false friend; it re-checks the stored value against the + `output` SCHEMA, never the network. +- **The cache cannot even STORE a 304.** Forced into its own key via `vary: ['if-none-match']`, three + identical conditional calls measured `[undefined, undefined, undefined]` with statuses + `[200,304,304,304]` and 4 network requests: `op.set` writes `{ v: undefined }` and cache.ts:482 + reads that as a permanent miss. The entry is written and can never be read. +- **The one workaround for storing the ETag turns the cache off.** Folding it into the value via + `transform` (`{ etag, body }`) makes the stitch un-fingerprintable, so ADR 0004 fails closed: + measured `bypass: opaque transform without cache.transformVersion or trustTransform`, 2 requests + across 2 calls. +- **C6 is the finding that counts double.** With a server whose validators are content-derived (the + Apache/CDN default), an ETag store keyed on `METHOD URL` alone measured **`[tok-alice|(none)→200, +tok-bob|"v1"→304]`** — one store entry, and bob receiving `viewer: tok-alice`. Alice's private body + was served to bob, and the rate-limit metrics IMPROVED while it happened. `cache.tenancy` does not + help: it keys the built-in cache, which knows nothing about the ETag store. And the obvious place + to fix it is not available — `ResolvedStitchConfig` carries no `principal` (it lives on + `AuthContext`, engine.ts:1032), and `Surface.buildRequest` cannot see the credential HEADER either, + because it runs at engine.ts:253, **before** `cfg.auth.apply` at engine.ts:649. Only + `hooks.onRequest` and `Surface.execute` see it. One extra term in the key expression fixes it. +- **C8's TTL row is the one to read twice.** In the QUIET world (nothing changes across 10 polls) a + TTL cache and revalidation cost exactly the same: **1 billed of 10**. In the CHANGE world (the + resource moves once, before poll 6) revalidation bills 2/10 with versions + `[1,1,1,1,1,2,2,2,2,2]` and zero stale polls, while the TTL cache bills 1/10 and **never sees the + change at all** — versions `[1,1,1,1,1,1,1,1,1,1]`, 5 of 10 polls serving a superseded version. + One extra billed response is the entire price of correctness. +- **Two failure modes look exactly like success, and neither will ever raise an error.** The + load-balancer INODE case (a server minting a fresh validator per response) polled 10 times, billed + 10, got **0** 304s and reported nothing — detectable only as `stats.revalidated === 0` while + `stats.stored === 10`. A server that sends no `ETag` at all billed 3/3 with `stats.stored === 0` + and `stats.unvalidatable === 3`. Both deserve an assertion in production. +- **C9's line count is honest in the unflattering direction.** 87 executable lines of user code + against a **79**-line feature-matched hand-rolled twin — the StitchAPI side is LONGER, by 8 lines, + and those lines attribute precisely: `credentialOf` and `clearValidator`, two helpers that exist + only because the engine hands a surface a SHARED header record it never case-folds + (engine.ts:232). Behaviour is byte-identical on all four shapes. What the extra lines buy is what + stayed CONFIG — `auth`, `output`, `retry`, `timeout`, `seam.as()` and the trace spine + (`[start, progress:request, result, done]` per poll) — every one of which would have to be written + INTO the hand-rolled file. +- **Two capture corrections worth carrying.** `CacheOptions` has a tenth field the capture's list + omits: `keyOf` (types.ts:1202), a user-supplied key derivation — and `deriveCacheKey` still folds + the principal in through it (cache.ts:159-162), so it cannot be used to defeat tenancy. And + `cache.ttl` does **not** honour the injected `clock`: `memoryStore` reads `Date.now()` + (store.ts:16,45 via util.ts:4), measured as 1 request after advancing a `manualClock` by a virtual + hour against a 1-second TTL. `clock` drives retry/throttle/timeout/circuit; cache expiry is the + one timing knob it does not reach. diff --git a/docs/scenarios/proofs/conditional-requests-304/c1-bare-304.ts b/docs/scenarios/proofs/conditional-requests-304/c1-bare-304.ts new file mode 100644 index 00000000..9198ee2b --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c1-bare-304.ts @@ -0,0 +1,160 @@ +// C1 — what does a BARE stitch do with a `304 Not Modified`? +// +// The capture frames this as a fork: "treat it as a failure and every unchanged poll is an error; +// treat it as a success and the caller receives `undefined`". StitchAPI takes the second fork, and +// it takes it silently — because a 304 is not an error by any measure the engine applies. +// `classifyStatus` (surface.ts:143-149) fails a response only when `status >= 400`, so a 304 is +// "acceptable transport" exactly like a 200, and `httpInterpret` (surface.ts:234-237) then returns +// `{ ok: true, data: res.body }` where `res.body` is whatever an empty body decoded to. +// +// That last part is measured through the REAL `fetchAdapter` with an injected `fetch`, not asserted +// from the fake: a zero-byte JSON response decodes to `undefined` (http-adapter.ts:135) and a +// zero-byte response with no `content-type` decodes to `''` (http-adapter.ts:137). Neither is the +// resource, and neither is an error. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c1-bare-304.ts +import { fetchAdapter, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +async function main(): Promise { + heading('C1 — a bare stitch meets a 304'); + + // ── (a) the fork: success or failure? ───────────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ url: api.url, adapter: api.adapter(), clock }); + + const first = await issues.safe({}); + check('(a) plain GET → ok', first.ok, true); + check( + '(a) plain GET → data.version', + (first.data as { version?: number }).version, + 1, + ); + + // Replay the validator the server just minted. This is the shape every subsequent poll has. + const inm = { 'If-None-Match': api.etagFor('(none)') }; + const second = await issues.safe({ headers: inm }); + check('(a) 304 → ok', second.ok, true); + check('(a) 304 → data', second.data, undefined); + check('(a) 304 → error', second.error, null); + checkSeq('(a) server saw', api.statuses, [200, 304]); + note( + '(a) → the fork is resolved as SUCCESS-WITH-NOTHING', + 'not an error, not the resource — `ok: true` carrying `undefined`', + ); + } + + // ── (b) the awaited path does not throw either ──────────────────────────────────────────── + // `await stitch(...)` throws a StitchError on failure, so if a 304 were a failure this is where + // it would surface. It resolves. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ url: api.url, adapter: api.adapter(), clock }); + await issues.safe({}); + let threw = ''; + let resolved: unknown = 'NOT REACHED'; + try { + resolved = await issues({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + } catch (e) { + threw = (e as Error).message; + } + check('(b) await threw', threw, ''); + check('(b) await resolved with', resolved, undefined); + note( + '(b) → a polling loop written as `const data = await issues()` gets `undefined`', + 'and every downstream read of it is a TypeError far from the cause', + ); + } + + // ── (c) `.inspect()` — the accessor whose job is "what did the server send?" ────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ url: api.url, adapter: api.adapter(), clock }); + await issues.safe({}); + const probe = await issues.inspect({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(c) inspect().status', probe.status, 304); + check('(c) inspect().data', probe.data, undefined); + check('(c) inspect().raw', probe.raw, null); + note( + '(c) → the STATUS is the only place a 304 is visible', + '`.inspect().status === 304` is the one signal a caller can branch on', + ); + } + + // ── (d) `verdict.accept: [304]` is a no-op, because 304 was never rejected ──────────────── + // The natural first guess — "declare 304 acceptable" — changes nothing: `acceptsStatus` is only + // consulted for `status >= 400` (surface.ts:147). + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + verdict: { accept: [304] }, + }); + await issues.safe({}); + const r = await issues.safe({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(d) verdict.accept: [304] → ok', r.ok, true); + check('(d) verdict.accept: [304] → data', r.data, undefined); + } + + // ── (e) what an empty body ACTUALLY decodes to, through the real adapter ────────────────── + // Injected `fetch`, no network. Both 304 shapes GitHub and friends send are covered. + { + const jsonCt = await fetchAdapter({ + fetch: async () => + new Response(null, { + status: 304, + headers: { + etag: '"v1"', + 'content-type': 'application/json; charset=utf-8', + }, + }), + })({ + url: 'https://api.github.example/repos/octo/hello/issues', + method: 'GET', + headers: {}, + }); + check('(e) real fetchAdapter → status', jsonCt.status, 304); + check('(e) real fetchAdapter → body (json ct)', jsonCt.body, undefined); + check( + '(e) real fetchAdapter → etag survives', + jsonCt.headers['etag'], + '"v1"', + ); + + const noCt = await fetchAdapter({ + fetch: async () => + new Response(null, { status: 304, headers: { etag: '"v1"' } }), + })({ + url: 'https://api.github.example/repos/octo/hello/issues', + method: 'GET', + headers: {}, + }); + check('(e) real fetchAdapter → body (no ct)', noCt.body, ''); + note( + '(e) → the empty body is `undefined` or `""` depending on `content-type`', + 'so a caller cannot even rely on one falsy shape; the ETag header, though, always survives', + ); + } + + finish( + 'C1', + 'a 304 is a SILENT SUCCESS CARRYING NOTHING. Measured: `ok: true`, `data: undefined`, `error: null`, and the awaited form resolves rather than throwing (`await issues()` → `undefined`). It is not a policy the engine chose for 304s — `classifyStatus` (surface.ts:143-149) only fails `status >= 400`, so a 304 is transport-healthy exactly like a 200, and `httpInterpret` (surface.ts:237) hands back `res.body` unexamined. `verdict.accept: [304]` is therefore a no-op (measured: still `ok: true`, `data: undefined`) because nothing rejected it. Through the REAL `fetchAdapter` with an injected `fetch`, the empty body decodes to `undefined` with a JSON `content-type` (http-adapter.ts:135) and to `""` without one (http-adapter.ts:137) — two different falsy values, neither of them the resource. The one place the 304 remains visible is `.inspect().status`, measured 304 with `data: undefined` and `raw: null`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c2-replay-the-validator.ts b/docs/scenarios/proofs/conditional-requests-304/c2-replay-the-validator.ts new file mode 100644 index 00000000..97bf6334 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c2-replay-the-validator.ts @@ -0,0 +1,254 @@ +// C2 — can the ETag be read off a response and replayed as `If-None-Match` on the next request? +// +// Three seams can set an outgoing header, and they are NOT interchangeable. The difference that +// matters is WHEN each one runs, and it is measurable rather than arguable: +// +// • `hooks.onRequest` — engine.ts:652, once per ATTEMPT, AFTER `cfg.auth.apply` (engine.ts:649) +// • `Surface.buildRequest` — engine.ts:253, once per RUN, BEFORE auth and before the attempt loop +// • `Surface.execute` — engine.ts:666-674, once per ATTEMPT, AFTER auth, and it also sees the +// response, which is the only way to correlate the two (C9) +// +// The read side has one more seam than the capture supposes: `Surface.interpret` receives the whole +// `AdapterResponse`, headers included, so the ETag does NOT have to come out through `onResponse`. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c2-replay-the-validator.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +async function main(): Promise { + heading('C2 — reading the ETag out, and putting `If-None-Match` back in'); + + // ── (a) hooks.onResponse reads it, hooks.onRequest replays it ───────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let stored: string | undefined; + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (stored !== undefined && ctx.req) + ctx.req.headers['If-None-Match'] = stored; + }, + onResponse: (ctx) => { + const etag = ctx.res?.headers['etag']; + if (etag !== undefined) stored = etag; + }, + }, + }); + await issues.safe({}); + await issues.safe({}); + await issues.safe({}); + checkSeq('(a) `If-None-Match` on the wire', api.validators, [ + '(none)', + '"v1.t1"', + '"v1.t1"', + ]); + checkSeq('(a) statuses', api.statuses, [200, 304, 304]); + check('(a) rate-limited responses', api.billed, 1); + note( + '(a) → the REQUEST half works exactly as hoped', + 'the header goes out byte-for-byte and the server answers 304', + ); + } + + // ── (b) …but the DATA is still gone ─────────────────────────────────────────────────────── + // Replaying the validator is the cheap half. The result of doing so, with hooks alone, is that + // the caller now receives `undefined` on 2 of 3 polls instead of 0 of 3 — strictly worse than + // not conditionalising at all, unless something substitutes the body (C3). + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let stored: string | undefined; + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (stored !== undefined && ctx.req) + ctx.req.headers['If-None-Match'] = stored; + }, + onResponse: (ctx) => { + const etag = ctx.res?.headers['etag']; + if (etag !== undefined) stored = etag; + }, + }, + }); + const versions = [ + (await issues.safe({})).data, + (await issues.safe({})).data, + (await issues.safe({})).data, + ].map((d) => (d as { version?: number } | undefined)?.version ?? null); + checkSeq('(b) versions the caller received', versions, [1, null, null]); + note( + '(b) → hooks alone make the poll CHEAPER and the answer EMPTY', + 'the request half is free; the response half is the whole problem (C3)', + ); + } + + // ── (c) `Surface.interpret` can read the ETag itself — `onResponse` is not required ─────── + // The capture asks whether the ETag is readable "off the previous response (`hooks.onResponse`? + // `interpret`?)". Both. `interpret` gets the full `AdapterResponse`. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const seenByInterpret: (string | undefined)[] = []; + let stored: string | undefined; + const reader: Surface = { + id: 'etag-reader', + interpret: (res, cfg) => { + seenByInterpret.push(res.headers['etag']); + if (res.status === 200) stored = res.headers['etag']; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: reader, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (stored !== undefined && ctx.req) + ctx.req.headers['If-None-Match'] = stored; + }, + }, + }); + await issues.safe({}); + await issues.safe({}); + checkSeq('(c) ETags visible to `interpret`', seenByInterpret, [ + '"v1.t1"', + '"v1.t1"', + ]); + checkSeq('(c) `If-None-Match` on the wire', api.validators, [ + '(none)', + '"v1.t1"', + ]); + note( + '(c) → `interpret` sees response HEADERS, not just the body', + 'so the store-write and the body-substitution can live in one function (C3)', + ); + } + + // ── (d) `Surface.buildRequest` also sets it — but only ONCE PER RUN ─────────────────────── + // Same wire result on a happy path. The difference shows up under retry: `buildRequest` runs at + // engine.ts:253, before `attemptLoop`, and every attempt is a `cloneReq(baseReq)` of that one + // request. A validator baked in here can never be dropped mid-run. Measured against a stitch + // whose `interpret` asks for a re-attempt (`{ ok: false, retry: true }`, ADR 0022 Decision 5): + // all 3 attempts carry the SAME validator. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + // Start with a validator already in hand but no body — the "orphan validator" case. + let stored: string | undefined = api.etagFor('(none)'); + const baked: Surface = { + id: 'bake-once', + buildRequest: (_cfg, _input, base) => + stored === undefined + ? base + : { + ...base, + headers: { ...base.headers, 'If-None-Match': stored }, + }, + interpret: (res, cfg) => { + if (res.status === 304) { + stored = undefined; // "drop it and refetch" — has no effect on this run + return { + ok: false, + retry: true, + message: '304 with no cached body', + }; + } + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: baked, + adapter: api.adapter(), + clock, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + }); + const pending = issues.safe({}); + await clock.advance(10_000); + const r = await pending; + checkSeq( + '(d) `buildRequest` validator across 3 attempts', + api.validators, + ['"v1.t1"', '"v1.t1"', '"v1.t1"'], + ); + check('(d) run ok', r.ok, false); + check('(d) error', r.error?.message, '304 with no cached body'); + } + + // ── (e) …whereas `hooks.onRequest` runs PER ATTEMPT and CAN drop it ─────────────────────── + // Identical surface logic, header moved to the hook: attempt 1 sends the orphan validator, gets + // a 304, `interpret` asks for a re-attempt, and attempt 2 goes out UNCONDITIONAL and succeeds. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let stored: string | undefined = api.etagFor('(none)'); + let cached: unknown; + const perAttempt: Surface = { + id: 'per-attempt', + interpret: (res, cfg) => { + if (res.status === 304) { + if (cached !== undefined) return { ok: true, data: cached }; + stored = undefined; + return { + ok: false, + retry: true, + message: '304 with no cached body', + }; + } + const failure = verdictOf(res, cfg); + if (failure) return failure; + stored = res.headers['etag']; + cached = res.body; + return { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: perAttempt, + adapter: api.adapter(), + clock, + retry: { attempts: 2, backoff: { curve: 'fixed', base: 0 } }, + hooks: { + onRequest: (ctx) => { + if (!ctx.req) return; + if (stored !== undefined) + ctx.req.headers['If-None-Match'] = stored; + else delete ctx.req.headers['If-None-Match']; + }, + }, + }); + const pending = issues.safe({}); + await clock.advance(10_000); + const r = await pending; + checkSeq('(e) validators across 2 attempts', api.validators, [ + '"v1.t1"', + '(none)', + ]); + checkSeq('(e) statuses', api.statuses, [304, 200]); + check('(e) run ok', r.ok, true); + check('(e) data.version', (r.data as { version?: number }).version, 1); + note( + '(e) → an orphan validator is RECOVERABLE only from the per-attempt seam', + 'the identical logic behind `buildRequest` (case d) loops on the same 304 until the budget is gone', + ); + } + + finish( + 'C2', + 'YES on both halves, and the seams are not interchangeable. `hooks.onRequest` puts `If-None-Match` on the wire byte-for-byte (measured `["(none)","\\"v1.t1\\"","\\"v1.t1\\""]` → statuses `[200,304,304]`, 1 billed response out of 3), and the ETag is readable from `hooks.onResponse` — but ALSO from `Surface.interpret`, which the capture treats as an open question: `interpret` receives the whole `AdapterResponse`, so `res.headers["etag"]` is right there (measured `["\\"v1.t1\\"","\\"v1.t1\\""]`). The ordering finding is the one worth carrying: `Surface.buildRequest` runs ONCE PER RUN (engine.ts:253, before `attemptLoop`), so a validator set there is baked into every `cloneReq` — measured 3 identical validators across 3 attempts and a run that fails — while `hooks.onRequest` runs ONCE PER ATTEMPT (engine.ts:652) and can DROP the header on a re-attempt, measured `["\\"v1.t1\\"","(none)"]` → `[304,200]` → `ok: true`. Replaying the validator alone still leaves the caller with `undefined` on every unchanged poll (measured versions `[1,null,null]`), which is C3', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c3-substitute-the-body.ts b/docs/scenarios/proofs/conditional-requests-304/c3-substitute-the-body.ts new file mode 100644 index 00000000..6e721160 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c3-substitute-the-body.ts @@ -0,0 +1,300 @@ +// C3 — THE DECIDING CLAIM. Can a 304 be turned into "return the cached body", inside the response +// path, so the caller receives the resource? +// +// The capture is right to make this the hinge and right to distrust it: scenario 5 measured that +// `Surface.interpret` is DEAD CODE on a streaming surface — `runStreaming` never calls +// `interpretOf`, only `classifyStatus(res.status, cfg)` at engine.ts:1371. So the first thing this +// script establishes, with a counter rather than a reading of the source, is whether `interpret` +// runs at all for a NON-2xx status on the BUFFERED path. +// +// It does, and it always has since ADR 0022 Decision 1: engine.ts:775 sits inside `attemptLoop` and +// is reached for EVERY response, with the comment saying so in as many words — "The surface +// interprets EVERY response here, including the non-2xx the engine used to throw on before any hook +// could see it." A 304 reaches it doubly easily, because `classifyStatus` never rejected it in the +// first place (C1). +// +// So the answer is YES, through three different seams, and the differences between them are what +// this script spends its checks on. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c3-substitute-the-body.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** What a validator store holds. The two halves must live together — that is the whole scenario. */ +interface Entry { + etag: string; + body: unknown; +} + +async function main(): Promise { + heading('C3 — turning a 304 back into the cached body'); + + // ── (a) FIRST: does `interpret` run on a non-2xx at all? ────────────────────────────────── + // Measured with a counter against 200, 304 and 404 on the same buffered stitch. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const seen: number[] = []; + // A transport that answers 404 on demand, so the counter covers a status the engine + // genuinely fails on as well as the 3xx this scenario is about. + const inner = api.adapter(); + let nextIs404 = false; + const withNotFound: Adapter = async (req) => + nextIs404 + ? { status: 404, headers: {}, body: { message: 'Not Found' } } + : inner(req); + const counting: Surface = { + id: 'counting', + interpret: (res, cfg) => { + seen.push(res.status); + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: counting, + adapter: withNotFound, + clock, + }); + await issues.safe({}); + await issues.safe({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + nextIs404 = true; + await issues.safe({}); + checkSeq( + '(a) statuses `interpret` was called for', + seen, + [200, 304, 404], + ); + note( + '(a) → `interpret` is NOT dead code here, unlike on a streaming surface', + 'engine.ts:775 runs it for every response inside `attemptLoop` (ADR 0022 Decision 1)', + ); + } + + // ── (b) SEAM 1 — `interpret` substitutes, `hooks.onRequest` replays ─────────────────────── + // The capture's nomination, and it works. `interpret` reads the ETag off the response itself, + // so the hook is needed only for the request half. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let entry: Entry | undefined; + const revalidating: Surface = { + id: 'http+revalidate', + interpret: (res, cfg) => { + if (res.status === 304 && entry) + return { ok: true, data: entry.body }; + const failure = verdictOf(res, cfg); + if (failure) return failure; + const etag = res.headers['etag']; + if (etag !== undefined) entry = { etag, body: res.body }; + return { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: revalidating, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (entry && ctx.req) + ctx.req.headers['If-None-Match'] = entry.etag; + }, + }, + }); + const versions: (number | null)[] = []; + const push = async (): Promise => { + const r = await issues.safe({}); + versions.push( + (r.data as { version?: number } | undefined)?.version ?? null, + ); + }; + await push(); + await push(); + api.mutate(); + await push(); + await push(); + checkSeq('(b) versions the caller received', versions, [1, 1, 2, 2]); + checkSeq('(b) statuses', api.statuses, [200, 304, 200, 304]); + check('(b) rate-limited responses', api.billed, 2); + check('(b) requests made', api.requests, 4); + note( + '(b) → the caller never sees `undefined` and never sees a stale version', + 'a change is picked up on the very next poll, at half the rate-limit cost', + ); + } + + // ── (c) SEAM 2 — `Surface.execute`, which sees the request AND the response ─────────────── + // The stronger answer, and the one C9 assembles. `execute` REPLACES the transport (ADR 0008) at + // engine.ts:666-674 — inside the resilience chain, after `cfg.auth.apply`. One function owns + // both halves, so the response is correlated with ITS OWN request rather than with whatever a + // closure variable happened to hold. Note there is no custom `interpret` here at all: the + // substituted body rides back on a response whose status is still 304, and `httpInterpret` + // hands it to the caller unchanged, because 304 was never a failure (C1). + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const store = new Map(); + const transport = api.adapter(); + const revalidating: Surface = { + id: 'http+revalidate', + execute: async (req) => { + const key = `${req.method} ${req.url}`; + const entry = store.get(key); + if (entry) req.headers['If-None-Match'] = entry.etag; + const res = await transport(req); + if (res.status === 304 && entry) + return { ...res, body: entry.body }; + const etag = res.headers['etag']; + if (res.status === 200 && etag !== undefined) + store.set(key, { etag, body: res.body }); + return res; + }, + }; + const issues = stitch({ url: api.url, kind: revalidating, clock }); + const versions: (number | null)[] = []; + const push = async (): Promise => { + const r = await issues.safe({}); + versions.push( + (r.data as { version?: number } | undefined)?.version ?? null, + ); + }; + await push(); + await push(); + api.mutate(); + await push(); + await push(); + checkSeq('(c) versions the caller received', versions, [1, 1, 2, 2]); + checkSeq('(c) statuses', api.statuses, [200, 304, 200, 304]); + check('(c) rate-limited responses', api.billed, 2); + + // The status stays HONEST: the wire said 304, and `.inspect()` reports 304 while the data + // is the resource. Nothing has to lie about what happened to make the caller whole. + const probe = await issues.inspect({}); + check('(c) inspect().status after substitution', probe.status, 304); + check( + '(c) inspect().data.version', + (probe.data as { version?: number }).version, + 2, + ); + note( + '(c) → NO custom `interpret` was needed', + 'a 304 carrying a body is already a success to `httpInterpret`; `execute` just supplies the body', + ); + } + + // ── (d) SEAM 3 — `transform`, which works but is the worst of the three ─────────────────── + // `transform` runs at engine.ts:1198, after `interpret` and before validation, so it CAN swap + // the value. What it cannot do is know it is looking at a 304: it receives only the value, so + // the status has to be smuggled in through `onResponse`. Measured: it sees `undefined`. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const seenByTransform: unknown[] = []; + let lastStatus = 0; + let stored: Entry | undefined; + // `transform` cannot see headers OR the status, so BOTH have to come from hooks. + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (stored?.etag && ctx.req) + ctx.req.headers['If-None-Match'] = stored.etag; + }, + onResponse: (ctx) => { + lastStatus = ctx.res?.status ?? 0; + const etag = ctx.res?.headers['etag']; + if (etag !== undefined && lastStatus === 200) + stored = { etag, body: stored?.body }; + }, + }, + transform: (body) => { + seenByTransform.push(body); + if (lastStatus === 304) return stored?.body; + if (stored) stored.body = body; + return body; + }, + }); + const a = await issues.safe({}); + const b = await issues.safe({}); + check( + '(d) transform saw `undefined` on the 304', + seenByTransform[1], + undefined, + ); + check( + '(d) first poll version', + (a.data as { version?: number }).version, + 1, + ); + check( + '(d) 304 poll version', + (b.data as { version?: number }).version, + 1, + ); + note( + '(d) → it works, and it costs two out-of-band variables and the cache fingerprint', + 'an opaque `transform` makes a `cache`-bearing stitch refuse to cache unless `transformVersion` is set (ADR 0004)', + ); + } + + // ── (e) the seam that does NOT work: hooks alone cannot change the value ────────────────── + // Worth pinning because it is the first thing anyone tries. `hooks.onResponse` receives `ctx.res` + // and can mutate it — but the engine already read `res` into `outcome` at engine.ts:775? No: the + // hook fires at engine.ts:705, BEFORE the verdict. So mutating `ctx.res.body` DOES land. The + // reason not to build on it is that the hook has no idea which stored body belongs to this + // response, and no return channel for a failure — measured here purely to record that it lands. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let stored: Entry | undefined; + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (stored && ctx.req) + ctx.req.headers['If-None-Match'] = stored.etag; + }, + onResponse: (ctx) => { + if (!ctx.res) return; + if (ctx.res.status === 304 && stored) { + ctx.res.body = stored.body; // mutate the response in place + return; + } + const etag = ctx.res.headers['etag']; + if (etag !== undefined) + stored = { etag, body: ctx.res.body }; + }, + }, + }); + await issues.safe({}); + const r = await issues.safe({}); + check('(e) onResponse mutation lands → ok', r.ok, true); + check( + '(e) onResponse mutation lands → data.version', + (r.data as { version?: number }).version, + 1, + ); + note( + '(e) → `hooks.onResponse` fires at engine.ts:705, BEFORE the verdict at 775', + 'so an in-place body swap does reach the caller; it just has no request correlation and no failure channel', + ); + } + + finish( + 'C3', + 'YES — and `interpret` DOES run on a 304, measured with a counter, not inferred. The counter recorded `interpret` being called for `[200, 304, 404]` on one buffered stitch: engine.ts:775 sits inside `attemptLoop` and interprets EVERY response (ADR 0022 Decision 1), so scenario 5’s "interpret is dead code" finding is specific to `runStreaming` and does not carry here. Three seams turn the 304 back into the resource, all measured on the same 4-poll run with one mid-run change, all producing versions `[1,1,2,2]` from statuses `[200,304,200,304]` at 2 billed responses instead of 4: (1) `Surface.interpret` returning `{ ok: true, data: cached }`, paired with `hooks.onRequest` for the replay; (2) `Surface.execute`, which owns request AND response in one function and needs NO custom `interpret` at all — the substituted body rides back on a still-304 response and `httpInterpret` passes it through, so `.inspect().status` honestly reports 304 while `.data.version` is 2; (3) `transform`, which works but must smuggle the status in through `onResponse` (measured: it is handed `undefined`) and costs the cache fingerprint. `hooks.onResponse` can also mutate `ctx.res.body` in place and it lands (the hook fires at engine.ts:705, before the verdict at 775) — it just has no request correlation and no failure channel. THE 304 IS NOT A DEAD END', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c4-output-schema.ts b/docs/scenarios/proofs/conditional-requests-304/c4-output-schema.ts new file mode 100644 index 00000000..2e6ea6e1 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c4-output-schema.ts @@ -0,0 +1,254 @@ +// C4 — does an `output` schema reject a 304's empty body, and can the SUBSTITUTED body be validated +// instead? The capture's worry is that "the substitution has to happen below whatever parses/ +// validates the response", which in most clients it cannot. +// +// Here it can, and the ordering is not a coincidence — it is the run pipeline, in one place: +// +// interpret (engine.ts:775) → transform (1198) → pick (1199) → validateOutput (1203) +// +// Substitution happens at step 1. Validation happens at step 4, on whatever step 1 produced. So an +// `output` contract sees the CACHED body and never sees the 304's `undefined` at all. +// +// The claim has a sharp edge worth measuring in both directions: without the substitution, the +// contract turns every unchanged poll into a hard failure, which is strictly worse than C1's silent +// `undefined` — a polling loop that "worked" starts erroring the moment someone adds a schema. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c4-output-schema.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { StandardSchemaV1 } from '../../../../packages/core/src/standard-schema'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** Every value this schema is asked about, in order — so "what did validation actually see?" is + * a measurement rather than an inference. */ +const validated: unknown[] = []; + +/** A Standard Schema that requires the issues payload's shape. */ +const issuesSchema: StandardSchemaV1 = { + '~standard': { + version: 1, + vendor: 'conditional-requests-304-proof', + validate: (value: unknown) => { + validated.push(value); + const v = value as { repo?: unknown } | null | undefined; + if (v == null || typeof v !== 'object') + return { + issues: [ + { + message: `expected the issues payload, got ${String(value)}`, + }, + ], + }; + if (typeof v.repo !== 'string') + return { issues: [{ message: '`repo` must be a string' }] }; + return { value }; + }, + }, +}; + +async function main(): Promise { + heading('C4 — an `output` contract meets a 304'); + + // ── (a) bare stitch + `output`: the 304 is a HARD FAILURE ───────────────────────────────── + { + validated.length = 0; + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + output: issuesSchema, + }); + const first = await issues.safe({}); + check('(a) plain GET → ok', first.ok, true); + + const r = await issues.safe({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(a) 304 → ok', r.ok, false); + check( + '(a) 304 → error.message', + r.error?.message, + 'contract violation (drift)', + ); + check('(a) 304 → data', r.data, null); + check('(a) value the schema was handed', validated[1], undefined); + + const report = await issues.report({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(a) drift findings', report.findings.length, 1); + check('(a) finding.level', report.findings[0]?.level, 'error'); + note('(a) finding.detail', report.findings[0]?.detail ?? '(none)'); + note( + '(a) → adding a schema to a working conditional poll BREAKS it', + 'C1 gave `ok: true, data: undefined`; `output` turns the same 304 into `ok: false`', + ); + } + + // ── (b) with substitution: validation runs on the SUBSTITUTED body ──────────────────────── + // Same schema, same server, same 304 — but `interpret` supplied the cached body first. + { + validated.length = 0; + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let entry: { etag: string; body: unknown } | undefined; + const revalidating: Surface = { + id: 'http+revalidate', + interpret: (res, cfg) => { + if (res.status === 304 && entry) + return { ok: true, data: entry.body }; + const failure = verdictOf(res, cfg); + if (failure) return failure; + const etag = res.headers['etag']; + if (etag !== undefined) entry = { etag, body: res.body }; + return { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: revalidating, + adapter: api.adapter(), + clock, + output: issuesSchema, + hooks: { + onRequest: (ctx) => { + if (entry && ctx.req) + ctx.req.headers['If-None-Match'] = entry.etag; + }, + }, + }); + await issues.safe({}); + const r = await issues.safe({}); + check('(b) 304 → ok', r.ok, true); + check( + '(b) 304 → data.version', + (r.data as { version?: number }).version, + 1, + ); + check('(b) times the schema ran', validated.length, 2); + checkSeq( + '(b) values the schema saw', + validated.map( + (v) => (v as { version?: number } | undefined)?.version ?? null, + ), + [1, 1], + ); + checkSeq('(b) statuses on the wire', api.statuses, [200, 304]); + note( + '(b) → the contract never sees `undefined`', + 'substitution at engine.ts:775 is upstream of validation at engine.ts:1203', + ); + } + + // ── (c) the contract still BITES on the substituted value ───────────────────────────────── + // Substitution is not a bypass: a schema that rejects the cached body fails the run, so a store + // holding a value that no longer satisfies the contract is caught rather than served. + { + validated.length = 0; + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + // A store pre-seeded with a value that does NOT match the schema — the shape a persisted + // cache written by an older version of the code would have. + let entry: { etag: string; body: unknown } | undefined; + const revalidating: Surface = { + id: 'http+revalidate', + interpret: (res, cfg) => { + if (res.status === 304 && entry) + return { ok: true, data: entry.body }; + const failure = verdictOf(res, cfg); + if (failure) return failure; + const etag = res.headers['etag']; + if (etag !== undefined) + entry = { etag, body: { stale: 'shape' } }; + return { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: revalidating, + adapter: api.adapter(), + clock, + output: issuesSchema, + hooks: { + onRequest: (ctx) => { + if (entry && ctx.req) + ctx.req.headers['If-None-Match'] = entry.etag; + }, + }, + }); + await issues.safe({}); + const r = await issues.safe({}); + check('(c) stale-shaped cached body → ok', r.ok, false); + check( + '(c) error.message', + r.error?.message, + 'contract violation (drift)', + ); + note( + '(c) → the contract is enforced on the value the CALLER receives', + 'not on the bytes the server sent, which is the right place for it', + ); + } + + // ── (d) `pick` runs on the substituted value too ────────────────────────────────────────── + // Same pipeline position (engine.ts:1199). Worth pinning because `pick` on a bare 304's + // `undefined` silently yields `undefined` rather than erroring. + { + validated.length = 0; + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let entry: { etag: string; body: unknown } | undefined; + const revalidating: Surface = { + id: 'http+revalidate', + interpret: (res, cfg) => { + if (res.status === 304 && entry) + return { ok: true, data: entry.body }; + const failure = verdictOf(res, cfg); + if (failure) return failure; + const etag = res.headers['etag']; + if (etag !== undefined) entry = { etag, body: res.body }; + return { ok: true, data: res.body }; + }, + }; + const picked = stitch({ + url: api.url, + kind: revalidating, + adapter: api.adapter(), + clock, + pick: 'issues', + hooks: { + onRequest: (ctx) => { + if (entry && ctx.req) + ctx.req.headers['If-None-Match'] = entry.etag; + }, + }, + }); + await picked.safe({}); + const r = await picked.safe({}); + check('(d) pick on the substituted body → ok', r.ok, true); + check('(d) picked length', (r.data as unknown[]).length, 1); + + const bare = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + pick: 'issues', + }); + const b = await bare.safe({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(d) pick on a bare 304 → ok', b.ok, true); + check('(d) pick on a bare 304 → data', b.data, undefined); + } + + finish( + 'C4', + 'YES to both halves, and the ordering is the finding. A bare `output` schema turns every unchanged poll into a HARD FAILURE — measured `ok: false`, `error.message` `contract violation (drift)`, `data: null`, one `error`-level finding reading "expected the issues payload, got undefined" — which is strictly worse than C1’s silent `undefined`, because adding a schema to a working conditional poll is what breaks it. With substitution in place the contract never sees the empty body at all: the pipeline is `interpret` (engine.ts:775) → `transform` (1198) → `pick` (1199) → `validateOutput` (1203), so validation runs on whatever `interpret` returned. Measured on a 200-then-304 pair: the schema ran twice and was handed version `[1, 1]` — never `undefined` — and the 304 poll resolved `ok: true` with `data.version: 1`. Substitution is not a bypass either: a store holding a value the schema rejects still fails the run (measured `ok: false`, same drift message), so the contract is enforced on the value the CALLER receives. `pick` sits at the same pipeline position and behaves the same way — measured 1 item off the substituted body, versus `undefined` off a bare 304', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c5-builtin-cache.ts b/docs/scenarios/proofs/conditional-requests-304/c5-builtin-cache.ts new file mode 100644 index 00000000..2552c6e7 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c5-builtin-cache.ts @@ -0,0 +1,288 @@ +// C5 — can the built-in `cache` participate, storing the ETag alongside the body? And does a cache +// HIT short-circuit before any revalidation could run? +// +// No, and yes. The two are the same fact seen from either end: `cache` is a value store, not a +// response store. Its entry shape is `{ v, s, vary }` (cache.ts:300-304) — the validated VALUE, the +// status, and the learned `Vary` names. Response headers are never carried into it, so there is +// nowhere for an ETag to live; and `runCached` serves a hit at engine.ts:1613-1617 by yielding +// `resultEvt` directly, before `runFrom` is ever entered, so nothing downstream of the lookup runs. +// +// `revalidateOnHit` (cache.ts:441, engine.ts:1604) is a false friend: it re-validates the stored +// value against the `output` SCHEMA (ADR 0004's un-fingerprintable-contract policy). It never +// touches the network. +// +// What the two CAN do is compose, in the one order that makes sense: `cache` outermost for the hot +// window, revalidation underneath for the cold one. That is measured too. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c5-builtin-cache.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { clockStore } from './clock-store'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** The `execute`-seam revalidator from C3, as a reusable factory for this script. */ +function revalidating(api: FakeEtagApi): Surface { + const store = new Map(); + const transport = api.adapter(); + return { + id: 'http+revalidate', + execute: async (req) => { + const key = `${req.method} ${req.url}`; + const entry = store.get(key); + if (entry) req.headers['If-None-Match'] = entry.etag; + const res = await transport(req); + if (res.status === 304 && entry) + return { ...res, body: entry.body }; + const etag = res.headers['etag']; + if (res.status === 200 && etag !== undefined) + store.set(key, { etag, body: res.body }); + return res; + }, + }; +} + +async function main(): Promise { + heading('C5 — the built-in `cache`, and what a hit skips'); + + // ── (a) a hit short-circuits EVERYTHING below the lookup ────────────────────────────────── + // Hooks, the surface's `interpret`, the network. Measured with three counters at once. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const fired: string[] = []; + let interpretCalls = 0; + const counting: Surface = { + id: 'counting', + interpret: (res, cfg) => { + interpretCalls++; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: counting, + adapter: api.adapter(), + clock, + store: clockStore(clock), + cache: { ttl: '60s', tenancy: 'app' }, + hooks: { + onRequest: () => void fired.push('onRequest'), + onResponse: () => void fired.push('onResponse'), + }, + }); + await issues.safe({}); + await issues.safe({}); + await issues.safe({}); + checkSeq('(a) hooks fired across 3 calls', fired, [ + 'onRequest', + 'onResponse', + ]); + check('(a) `interpret` calls', interpretCalls, 1); + check('(a) requests reaching the server', api.requests, 1); + + const events: string[] = []; + for await (const e of issues.stream({})) + events.push( + e.type === 'progress' ? `${e.phase}:${e.detail ?? ''}` : e.type, + ); + checkSeq('(a) event spine on a hit', events, [ + 'start', + 'cache:hit', + 'result', + 'done', + ]); + note( + '(a) → there is no `request` phase on a hit, so there is nothing to conditionalise', + 'engine.ts:1613-1617 yields `resultEvt` and returns without entering `runFrom`', + ); + } + + // ── (b) the entry has no room for an ETag ───────────────────────────────────────────────── + // The stored value is what the CALLER got — post-`interpret`, post-`transform`, post-validation + // (engine.ts:1624 stores `out.value`). No headers reach it. The only way to keep a validator + // alongside the body through `cache` is to fold it INTO the value, which changes the value the + // caller receives — measured here so the workaround's cost is on the record. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + let lastEtag: string | undefined; + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + store: clockStore(clock), + cache: { ttl: '60s', tenancy: 'app' }, + hooks: { + onResponse: (ctx) => { + lastEtag = ctx.res?.headers['etag']; + }, + }, + transform: (body) => ({ etag: lastEtag, body }), + }); + const first = await issues.safe({}); + check( + '(b) etag reachable only by reshaping the value', + (first.data as { etag?: string }).etag, + '"v1.t1"', + ); + // …and the reshaping is self-defeating: an opaque `transform` cannot be fingerprinted, so + // ADR 0004's fail-closed policy REFUSES to cache at all. The workaround that makes the ETag + // storable is the same workaround that turns the cache off. + const events: string[] = []; + for await (const e of issues.stream({})) + if (e.type === 'progress' && e.phase === 'cache') + events.push(e.detail ?? ''); + check('(b) requests across 2 calls', api.requests, 2); + check('(b) cache verdict', events[0]?.startsWith('bypass:'), true); + note('(b) cache bypass reason', events[0] ?? '(none)'); + note( + '(b) → the caller now unwraps `{ etag, body }` on every call AND loses the cache', + 'an opaque `transform` is un-fingerprintable, so the stitch refuses to cache (ADR 0004)', + ); + } + + // ── (c) `cache` cannot STORE a 304, and keeps re-requesting one forever ─────────────────── + // Force the conditional request into its own cache key (`vary: ['if-none-match']`) so the 304 is + // a genuine miss that then gets stored. `op.set` writes `{ v: undefined, s: 304 }`, and `hitFrom` + // (cache.ts:482) reads `entry.v === undefined` as "no hit" — so the entry is written and can + // never be read. Every subsequent identical call goes back to the network AND returns `undefined`. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + store: clockStore(clock), + cache: { + ttl: '600s', + tenancy: 'app', + vary: ['if-none-match'], + }, + }); + await issues.safe({}); + const inm = { 'If-None-Match': api.etagFor('(none)') }; + const r1 = await issues.safe({ headers: inm }); + const r2 = await issues.safe({ headers: inm }); + const r3 = await issues.safe({ headers: inm }); + checkSeq( + '(c) data across 3 identical conditional calls', + [r1.data, r2.data, r3.data], + [undefined, undefined, undefined], + ); + checkSeq( + '(c) statuses on the wire', + api.statuses, + [200, 304, 304, 304], + ); + check('(c) requests', api.requests, 4); + note( + '(c) → the cache neither serves nor suppresses a 304', + 'it writes `{ v: undefined }`, which cache.ts:482 reads as a permanent miss', + ); + } + + // ── (d) `cache.vary` cannot key on the ETag anyway — the header is not in the key ────────── + // Without an explicit `vary`, request headers are absent from `canonicalRequest` (cache.ts:129-147: + // method, url, body, principal — headers ONLY when `varyNames` is non-empty). So a hand-set + // `If-None-Match` does not even reach the server: the call hits the cache entry stored for the + // unconditional request. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + store: clockStore(clock), + cache: { ttl: '600s', tenancy: 'app' }, + }); + await issues.safe({}); + const r = await issues.safe({ + headers: { 'If-None-Match': api.etagFor('(none)') }, + }); + check('(d) conditional call → ok', r.ok, true); + check( + '(d) conditional call → data.version', + (r.data as { version?: number }).version, + 1, + ); + checkSeq('(d) statuses on the wire', api.statuses, [200]); + note( + '(d) → the validator never left the process', + 'the same key served the cached 200, which is correct caching and useless revalidation', + ); + } + + // ── (e) `cache.ttl` does not honour the injected clock ──────────────────────────────────── + // `memoryStore` reads `now()` = `Date.now()` (store.ts:16,45 via util.ts:4), so a `manualClock` + // advanced by a virtual HOUR expires nothing. Every other timing knob in the library is + // clock-driven; this one is not, which makes cache staleness untestable without real sleeping. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + cache: { ttl: '1s', tenancy: 'app' }, // default `memoryStore` + }); + await issues.safe({}); + await clock.advance(3_600_000); + await issues.safe({}); + check('(e) requests after +1h VIRTUAL on a 1s ttl', api.requests, 1); + note( + '(e) → `clock` drives retry/throttle/timeout/circuit, but NOT cache expiry', + 'every claim here that needs an expiring cache injects `clockStore(clock)` instead', + ); + } + + // ── (f) what DOES work: `cache` outermost, revalidation underneath ──────────────────────── + // The hot window is answered with zero requests; the cold one with a 304 that costs nothing. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + kind: revalidating(api), + clock, + store: clockStore(clock), + cache: { ttl: '60s', tenancy: 'app' }, + }); + await issues.safe({}); + await issues.safe({}); + check('(f) requests inside the TTL window', api.requests, 1); + await clock.advance(61_000); + const cold = await issues.safe({}); + check('(f) requests after expiry', api.requests, 2); + check('(f) billed after expiry', api.billed, 1); + check( + '(f) cold-window data.version', + (cold.data as { version?: number }).version, + 1, + ); + api.mutate(); + await clock.advance(61_000); + const changed = await issues.safe({}); + check( + '(f) picked up the change', + (changed.data as { version?: number }).version, + 2, + ); + checkSeq( + '(f) statuses across the whole run', + api.statuses, + [200, 304, 200], + ); + check('(f) billed across the whole run', api.billed, 2); + } + + finish( + 'C5', + 'NO on both counts, for one reason: `cache` is a VALUE store, not a response store. Its entry is `{ v, s, vary }` (cache.ts:300-304) and what gets written is `out.value` — the post-`interpret`, post-`transform`, post-validation value (engine.ts:1624) — so no response header, and therefore no ETag, can reach it. `revalidateOnHit` (cache.ts:441) is a false friend: it re-checks the stored value against the `output` SCHEMA (engine.ts:1604), never the network. And a hit short-circuits everything below the lookup — measured across 3 calls: hooks fired `[onRequest, onResponse]` ONCE, `interpret` ran 1 time, 1 request reached the server, and the event spine on a hit is `[start, cache:hit, result, done]` with no `request` phase at all, because engine.ts:1613-1617 yields `resultEvt` and returns without entering `runFrom`. The one workaround — folding the ETag into the VALUE via `transform`, so `{ etag, body }` is what gets stored — is self-defeating: an opaque `transform` is un-fingerprintable, so ADR 0004 fails closed and the stitch refuses to cache at all (measured 2 requests across 2 calls, `bypass: opaque transform without cache.transformVersion or trustTransform`). Two sharper edges: the cache cannot STORE a 304 either — forced into its own key via `vary`, three identical conditional calls measured `[undefined, undefined, undefined]` with statuses `[200,304,304,304]` and 4 network requests, because `op.set` writes `{ v: undefined }` and cache.ts:482 reads that as a permanent miss; and without an explicit `vary`, request headers are not in the key at all (cache.ts:129-147), so a hand-set `If-None-Match` never leaves the process (measured statuses `[200]`). Separately: `cache.ttl` does NOT honour the injected `clock` — `memoryStore` reads `Date.now()` (store.ts:16,45), measured as 1 request after advancing a `manualClock` by a virtual hour against a 1s TTL. What DOES work is composing them: `cache` outermost for the hot window, revalidation underneath for the cold one — measured 1 request inside the TTL, a free 304 after expiry, and version 2 picked up on the next cold poll, at 2 billed responses across the run', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c6-per-credential.ts b/docs/scenarios/proofs/conditional-requests-304/c6-per-credential.ts new file mode 100644 index 00000000..fdae43c0 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c6-per-credential.ts @@ -0,0 +1,349 @@ +// C6 — per-credential keying. GitHub mints ETags per token; replaying principal A's validator on +// principal B's request is a correctness bug, and on a server whose validators are content-derived +// it is a DATA LEAK: the 304 comes back and a client that then serves its stored body hands B what +// A fetched. +// +// Two stores are in play and they are governed by completely different things: +// +// • the built-in `cache` — `tenancy: 'principal'` is the DEFAULT and fail-closed (types.ts:1147), +// folding the seam-bound principal into the key (cache.ts:398, 451-457). Measured below in both +// directions, including what `tenancy: 'app'` costs. +// • the ETag store, which is user code — `tenancy` does not know it exists. Nothing keys it for +// you, and the seam that CAN key it is not obvious: `ResolvedStitchConfig` carries no +// `principal` (it lives on `AuthContext`, engine.ts:1032), so neither `Surface.buildRequest` nor +// `Surface.interpret` can read it. `buildRequest` cannot even see the credential HEADER, because +// it runs at engine.ts:253, before `cfg.auth.apply` at engine.ts:649. +// +// The seams that DO see the resolved credential are `hooks.onRequest` (engine.ts:652) and +// `Surface.execute` (engine.ts:666) — both downstream of auth. All four positions are measured. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c6-per-credential.ts +import { bearer } from '../../../../packages/core/src/auth'; +import { seam, stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Adapter, + AdapterRequest, +} from '../../../../packages/core/src/types'; +import { clockStore } from './clock-store'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +interface Entry { + etag: string; + body: unknown; +} + +/** The resolved `Authorization` header, or `''`. Case-insensitive: the engine never normalises. */ +const credOf = (headers: Record): string => { + for (const [k, v] of Object.entries(headers)) + if (k.toLowerCase() === 'authorization') return v; + return ''; +}; + +/** An `execute` revalidator over a SHARED store, keyed by `keyOf`. The whole claim is `keyOf`. */ +function revalidating( + transport: Adapter, + store: Map, + keyOf: (req: AdapterRequest) => string, +): Surface { + return { + id: 'http+revalidate', + execute: async (req) => { + const key = keyOf(req); + const entry = store.get(key); + if (entry) req.headers['If-None-Match'] = entry.etag; + const res = await transport(req); + if (res.status === 304 && entry) + return { ...res, body: entry.body }; + const etag = res.headers['etag']; + if (res.status === 200 && etag !== undefined) + store.set(key, { etag, body: res.body }); + return res; + }, + }; +} + +async function main(): Promise { + heading('C6 — one principal’s validator must never answer for another'); + + // ── (a) the built-in cache: `tenancy: 'principal'` (the default) isolates ───────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + const sm = seam({ + clock, + adapter: api.adapter(), + store: clockStore(clock), + }); + const forUser = (p: string) => + sm.as(p).stitch({ + url: api.url, + auth: bearer(`tok-${p}`), + cache: { ttl: '60s' }, // tenancy defaults to 'principal' + }); + const alice = forUser('alice'); + const bob = forUser('bob'); + const a = await alice.safe({}); + const b = await bob.safe({}); + check( + '(a) alice sees', + (a.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check( + '(a) bob sees', + (b.data as { viewer?: string }).viewer, + 'tok-bob', + ); + check('(a) requests', api.requests, 2); + check( + '(a) their cache keys differ', + (await alice.cache.keyOf({})) !== (await bob.cache.keyOf({})), + true, + ); + } + + // ── (b) …and `tenancy: 'app'` does not ──────────────────────────────────────────────────── + // Documented as "correct only for public, unauthenticated data" (types.ts:1140). Measured, on + // authenticated data, it serves alice's private body to bob from ONE request. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + const sm = seam({ + clock, + adapter: api.adapter(), + store: clockStore(clock), + }); + const forUser = (p: string) => + sm.as(p).stitch({ + url: api.url, + auth: bearer(`tok-${p}`), + cache: { ttl: '60s', tenancy: 'app' }, + }); + await forUser('alice').safe({}); + const b = await forUser('bob').safe({}); + check( + '(b) bob receives viewer', + (b.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check('(b) requests', api.requests, 1); + note( + '(b) → `tenancy: "app"` is a documented trade, not a bug', + 'recorded here because the same word does NOT protect the ETag store (case d)', + ); + } + + // ── (c) where the principal and the credential are actually visible ─────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + let cfgHasPrincipal = 'MISSING'; + let credInBuildRequest = 'MISSING'; + let credInExecute = 'MISSING'; + let credInOnRequest = 'MISSING'; + const transport = api.adapter(); + const probe: Surface = { + id: 'probe', + buildRequest: (cfg, _input, base) => { + cfgHasPrincipal = String( + 'principal' in cfg ? 'present' : 'absent', + ); + credInBuildRequest = credOf(base.headers) || 'absent'; + return base; + }, + execute: async (req) => { + credInExecute = credOf(req.headers) || 'absent'; + return transport(req); + }, + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.body }, + }; + const sm = seam({ clock }); + await sm + .as('alice') + .stitch({ + url: api.url, + kind: probe, + auth: bearer('tok-alice'), + hooks: { + onRequest: (ctx) => { + credInOnRequest = + credOf(ctx.req?.headers ?? {}) || 'absent'; + }, + }, + }) + .safe({}); + check( + '(c) `principal` on the surface’s cfg', + cfgHasPrincipal, + 'absent', + ); + check('(c) credential in `buildRequest`', credInBuildRequest, 'absent'); + check( + '(c) credential in `hooks.onRequest`', + credInOnRequest, + 'Bearer tok-alice', + ); + check( + '(c) credential in `Surface.execute`', + credInExecute, + 'Bearer tok-alice', + ); + note( + '(c) → only the two POST-AUTH seams can key an ETag store by credential', + '`buildRequest` runs at engine.ts:253, before `cfg.auth.apply` at engine.ts:649', + ); + } + + // ── (d) THE LEAK: a hand-rolled store keyed on the URL alone ────────────────────────────── + // One line of difference from case (e). The server's validators are content-derived here (the + // Apache/CDN default), so bob's request carrying alice's validator gets a 304 — and the client + // serves alice's body. Nothing errors, nothing warns, and the rate-limit numbers look GREAT. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + const shared = new Map(); + const kind = revalidating( + api.adapter(), + shared, + (req) => `${req.method} ${req.url}`, // ← no credential in the key + ); + const sm = seam({ clock }); + const forUser = (p: string) => + sm.as(p).stitch({ url: api.url, kind, auth: bearer(`tok-${p}`) }); + const a = await forUser('alice').safe({}); + const b = await forUser('bob').safe({}); + check( + '(d) alice sees', + (a.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check( + '(d) bob receives', + (b.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check('(d) ETag store entries', shared.size, 1); + checkSeq( + '(d) what the server saw', + api.hits.map((h) => `${h.token}|${h.inm}→${h.status}`), + ['tok-alice|(none)→200', 'tok-bob|"v1"→304'], + ); + check('(d) billed', api.billed, 1); + note( + '(d) → BOB WAS SERVED ALICE’S PRIVATE BODY, and the metrics improved', + 'a 304 rate of 50% is exactly what a working revalidator looks like', + ); + } + + // ── (e) the fix: fold the credential into the ETag store key ────────────────────────────── + // `Surface.execute` sees the resolved `Authorization` header (case c), so ONE key expression + // fixes it, with no per-principal plumbing at the call site. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + const shared = new Map(); + const kind = revalidating( + api.adapter(), + shared, + (req) => `${req.method} ${req.url} ${credOf(req.headers)}`, // ← credential IS the key + ); + const sm = seam({ clock }); + const forUser = (p: string) => + sm.as(p).stitch({ url: api.url, kind, auth: bearer(`tok-${p}`) }); + const alice = forUser('alice'); + const bob = forUser('bob'); + await alice.safe({}); + await bob.safe({}); + const a2 = await alice.safe({}); + const b2 = await bob.safe({}); + check('(e) ETag store entries', shared.size, 2); + check( + '(e) alice’s second poll', + (a2.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check( + '(e) bob’s second poll', + (b2.data as { viewer?: string }).viewer, + 'tok-bob', + ); + checkSeq( + '(e) what the server saw', + api.hits.map((h) => `${h.token}|${h.inm}→${h.status}`), + [ + 'tok-alice|(none)→200', + 'tok-bob|(none)→200', + 'tok-alice|"v1"→304', + 'tok-bob|"v1"→304', + ], + ); + check('(e) billed', api.billed, 2); + } + + // ── (f) the OTHER correlation bug: a closure variable under concurrency ─────────────────── + // The `interpret` + `hooks.onRequest` pairing (C3 seam 1) has no channel between the two, so the + // key has to live in a closure variable that `onRequest` writes and `interpret` reads. Two + // CONCURRENT calls to different resources on one stitch interleave, and the store silently ends + // up holding one entry instead of two — a revalidator that never revalidates. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const store = new Map(); + let pendingKey = ''; + const correlated: Surface = { + id: 'closure-correlated', + interpret: (res, cfg) => { + const key = pendingKey; // ← whatever the LAST onRequest happened to write + if (res.status === 304) + return { ok: true, data: store.get(key)?.body }; + const failure = verdictOf(res, cfg); + if (failure) return failure; + const etag = res.headers['etag']; + if (etag !== undefined) + store.set(key, { etag, body: res.body }); + return { ok: true, data: res.body }; + }, + }; + const issues = stitch({ + url: api.url, + kind: correlated, + adapter: api.adapter(), + clock, + hooks: { + onRequest: (ctx) => { + if (!ctx.req) return; + pendingKey = `${ctx.req.method} ${ctx.req.url}`; + const entry = store.get(pendingKey); + if (entry) ctx.req.headers['If-None-Match'] = entry.etag; + else delete ctx.req.headers['If-None-Match']; + }, + }, + }); + await Promise.all([ + issues.safe({ query: { page: 1 } }), + issues.safe({ query: { page: 2 } }), + ]); + check('(f) ETag store entries after 2 concurrent calls', store.size, 1); + await Promise.all([ + issues.safe({ query: { page: 1 } }), + issues.safe({ query: { page: 2 } }), + ]); + check('(f) requests after 4 calls', api.requests, 4); + check('(f) billed', api.billed, 3); + note( + '(f) → 3 of 4 polls paid full price, and nothing said why', + 'the `execute` seam has no such gap: the request and its response are one function call', + ); + } + + finish( + 'C6', + 'SPLIT, and the split is the finding. For the built-in `cache`, YES: `tenancy: "principal"` is the fail-closed DEFAULT and it holds — measured 2 requests and two different derived keys for two seam-bound principals, each seeing their own `viewer`; flipping to `tenancy: "app"` serves alice’s private body to bob off 1 request (a documented trade, types.ts:1140, not a bug). For a hand-rolled ETag store, NO — `tenancy`/`vary` do not know it exists, and the seams that could key it are not the obvious ones: `ResolvedStitchConfig` carries no `principal` at all (it lives on `AuthContext`, engine.ts:1032), and `Surface.buildRequest` cannot even see the credential HEADER because it runs at engine.ts:253, BEFORE `cfg.auth.apply` at engine.ts:649 — measured `absent` in both positions. Only `hooks.onRequest` (engine.ts:652) and `Surface.execute` (engine.ts:666) see the resolved credential, measured `Bearer tok-alice` in both. The consequence, against a server with content-derived validators: a store keyed on `METHOD URL` alone leaks — measured `[tok-alice|(none)→200, tok-bob|"v1"→304]`, ONE store entry, and bob receiving `viewer: tok-alice`, with the rate-limit metrics IMPROVING as it happens. Adding the credential to the key expression fixes it in one term: measured 2 store entries, 4 requests, `[…alice→200, …bob→200, …alice 304, …bob 304]`, each principal seeing their own data. A second correlation bug rides alongside: the `interpret` + `hooks.onRequest` pairing has no channel between the two seams, so the key must live in a closure variable — measured under 2 concurrent calls to different resources, the store ends up with 1 entry instead of 2 and 3 of the next 4 polls pay full price', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c7-weak-validators.ts b/docs/scenarios/proofs/conditional-requests-304/c7-weak-validators.ts new file mode 100644 index 00000000..6549b2f3 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c7-weak-validators.ts @@ -0,0 +1,205 @@ +// C7 — weak validators. `W/"v1"` and `"v1"` are different strings, `If-None-Match` is specified to +// compare WEAKLY (RFC 9110 §8.8.3.2), and a client that normalises or strips the `W/` prefix breaks +// revalidation against a compliant server — or, worse, matches where it should not. +// +// The measurement is byte-exactness end to end: what the server minted, what the client stored, and +// what came back on the wire. It is taken twice — once through the fake adapter that every other +// claim uses, and once through the REAL `fetchAdapter` with an injected `fetch`, because header +// mangling would live in the transport if it lived anywhere. +// +// StitchAPI has no ETag code at all (`grep -r 'If-None-Match\|304' packages/core/src` finds nothing; +// the sole `ETag` mention is fingerprint.ts:53, a comment about strong/weak SCHEMA fingerprints). +// Headers are a plain `Record` merged at engine.ts:232 and handed to the transport +// untouched. That absence is exactly why this passes. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c7-weak-validators.ts +import { fetchAdapter, stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** The `execute` revalidator, parameterised only by the transport. */ +function revalidating(transport: ReturnType): Surface { + const store = new Map(); + return { + id: 'http+revalidate', + execute: async (req) => { + const key = `${req.method} ${req.url}`; + const entry = store.get(key); + if (entry) req.headers['If-None-Match'] = entry.etag; + const res = await transport(req); + if (res.status === 304 && entry) + return { ...res, body: entry.body }; + const etag = res.headers['etag']; + if (res.status === 200 && etag !== undefined) + store.set(key, { etag, body: res.body }); + return res; + }, + }; +} + +async function main(): Promise { + heading('C7 — does `W/"v1"` survive the round trip byte-exact?'); + + // ── (a) a weak-validator server, end to end ─────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, weak: true }); + const issues = stitch({ + url: api.url, + kind: revalidating(api.adapter()), + clock, + }); + await issues.safe({}); + const second = await issues.safe({}); + api.mutate(); + const third = await issues.safe({}); + checkSeq('(a) `If-None-Match` on the wire', api.validators, [ + '(none)', + 'W/"v1.t1"', + 'W/"v1.t1"', + ]); + checkSeq('(a) statuses', api.statuses, [200, 304, 200]); + check( + '(a) 304 poll data.version', + (second.data as { version?: number }).version, + 1, + ); + check( + '(a) post-mutation data.version', + (third.data as { version?: number }).version, + 2, + ); + note( + '(a) → the `W/` prefix is neither stripped nor re-quoted', + 'headers are a plain Record merged at engine.ts:232 and handed to the transport untouched', + ); + } + + // ── (b) the strong server, for contrast — same code path, different string ──────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = stitch({ + url: api.url, + kind: revalidating(api.adapter()), + clock, + }); + await issues.safe({}); + await issues.safe({}); + checkSeq('(b) `If-None-Match` on the wire', api.validators, [ + '(none)', + '"v1.t1"', + ]); + note( + '(b) → the client carries whatever the server minted', + 'which is the only correct policy: the tag is OPAQUE to the client', + ); + } + + // ── (c) through the REAL `fetchAdapter`, with an injected `fetch` ───────────────────────── + // Header mangling, if it existed, would live here — `fetchAdapter` builds a `Headers` object + // from `req.headers` (http-adapter.ts:40, 54-61) before calling `fetch`. + { + const sent: string[] = []; + const record: typeof fetch = async (_input, init) => { + sent.push( + new Headers(init?.headers).get('if-none-match') ?? '(none)', + ); + return new Response(null, { + status: 304, + headers: { etag: 'W/"v1"', 'content-type': 'application/json' }, + }); + }; + const adapter = fetchAdapter({ fetch: record }); + const res = await adapter({ + url: 'https://api.github.example/repos/octo/hello/issues', + method: 'GET', + headers: { 'If-None-Match': 'W/"v1"' }, + }); + checkSeq('(c) header the transport actually sent', sent, ['W/"v1"']); + check( + '(c) weak ETag read back off the response', + res.headers['etag'], + 'W/"v1"', + ); + + // A multi-tag validator list — the other shape RFC 9110 allows — also survives whole. + const list = 'W/"v1", "v2", W/"v3"'; + await adapter({ + url: 'https://api.github.example/repos/octo/hello/issues', + method: 'GET', + headers: { 'If-None-Match': list }, + }); + check('(c) multi-tag list survives', sent[1], list); + } + + // ── (d) header NAME casing is preserved too, and never normalised ───────────────────────── + // Worth pinning: the engine merges `cfg.headers` and `input.headers` with a plain spread + // (engine.ts:232) and does no case folding, so `if-none-match` and `If-None-Match` are two + // DIFFERENT keys in the outgoing record. A server reads them the same way; a `delete` in a + // hook does not. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, weak: true }); + const seenKeys: string[][] = []; + const issues = stitch({ + url: api.url, + adapter: api.adapter(), + clock, + headers: { 'if-none-match': 'W/"v1"' }, + hooks: { + onRequest: (ctx) => { + // The obvious "clear it" line, written against the OTHER casing. + delete ctx.req?.headers['If-None-Match']; + seenKeys.push(Object.keys(ctx.req?.headers ?? {})); + }, + }, + }); + await issues.safe({}); + checkSeq( + '(d) outgoing header keys after the delete', + seenKeys[0] ?? [], + ['if-none-match'], + ); + checkSeq('(d) validator still on the wire', api.validators, ['W/"v1"']); + note( + '(d) → `delete headers["If-None-Match"]` does NOT remove `headers["if-none-match"]`', + 'the engine never case-folds request header names (engine.ts:232), so a hook must match the casing it set', + ); + } + + // ── (e) the client's OPACITY is what makes weak comparison work at all ──────────────────── + // Hand the server a STRONG tag for a representation it minted a WEAK validator for. RFC 9110 + // says `If-None-Match` compares weakly, so a compliant server matches — and it can only do that + // because the client shipped the tag it was given, unaltered, rather than "canonicalising" it. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, weak: true }); + const issues = stitch({ url: api.url, adapter: api.adapter(), clock }); + await issues.safe({}); + // `W/"v1.t1"` minted; send the strong form of the same tag. + await issues.safe({ headers: { 'If-None-Match': '"v1.t1"' } }); + checkSeq('(e) `If-None-Match` on the wire', api.validators, [ + '(none)', + '"v1.t1"', + ]); + checkSeq( + '(e) statuses (weak comparison matched)', + api.statuses, + [200, 304], + ); + note( + '(e) → the server decided the match, on the exact bytes the client sent', + 'any client-side normalisation would have made this comparison the client’s to get wrong', + ); + } + + finish( + 'C7', + 'YES — byte-exact, in both directions, through both transports. A weak-validator server round-trips as `["(none)", "W/\\"v1.t1\\"", "W/\\"v1.t1\\""]` on the wire with statuses `[200,304,200]`, the 304 poll yielding version 1 and the post-mutation poll version 2; the strong server on the identical code path sends `"v1.t1"`. Through the REAL `fetchAdapter` with an injected `fetch`, the header the transport handed to `fetch` was measured as `W/"v1"` and the response ETag read back as `W/"v1"`; a multi-tag list `W/"v1", "v2", W/"v3"` also survived whole. Because the client never touches the tag, the SERVER gets to apply RFC 9110’s weak comparison: sending the strong form `"v1.t1"` for a representation whose validator was minted `W/"v1.t1"` was measured as a 304. The reason all of this works is that there is nothing to survive: StitchAPI has NO ETag/`If-None-Match`/304 code anywhere in `packages/core/src` (the only `ETag` mention is fingerprint.ts:53, a comment about SCHEMA fingerprints), and request headers are a plain `Record` merged with a spread at engine.ts:232. That same absence has a sharp edge, measured separately: header names are never case-folded, so `delete ctx.req.headers["If-None-Match"]` does not remove a validator that was set as `"if-none-match"` — measured, the key survived the delete and `W/"v1"` still went out', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c8-the-payoff.ts b/docs/scenarios/proofs/conditional-requests-304/c8-the-payoff.ts new file mode 100644 index 00000000..06c54a0f --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c8-the-payoff.ts @@ -0,0 +1,210 @@ +// C8 — the payoff, measured rather than argued. Ten polls, three strategies, two worlds. +// +// The metric is `billed`: responses that would count against a rate limit. GitHub's rule is that a +// 304 answered from a correctly-authorized conditional request costs nothing against the primary +// limit, so `billed` counts every non-304 and nothing else. The second metric is STALENESS: how many +// of the ten polls handed the caller a version the server had already superseded — because a TTL +// cache buys its request savings with exactly that, and a table that reports only requests is +// flattering it. +// +// Two worlds, because one number hides the trade: +// • QUIET — nothing changes across the ten polls. This is what a polling loop does 99% of the time. +// • CHANGE — the resource changes once, just before poll 6. This is the 1% the loop exists for. +// +// `cache.ttl` needs `clockStore(clock)`: the default `memoryStore` reads `Date.now()` and ignores the +// injected clock entirely (C5 case e), so a virtual hour expires nothing. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c8-the-payoff.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { clockStore } from './clock-store'; +import { FakeEtagApi } from './fake-etag-api'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { revalidating } from './revalidate'; + +const POLLS = 10; +/** The poll the resource changes on, in the CHANGE world. Polls are 1-indexed. */ +const CHANGES_BEFORE = 6; +/** Virtual seconds between polls. */ +const INTERVAL_MS = 60_000; + +interface RunResult { + /** Responses that would count against a rate limit. */ + billed: number; + /** Requests that left the process. */ + requests: number; + /** The version each of the ten polls handed the caller. */ + versions: (number | null)[]; + /** Polls that served a version the server had already superseded. */ + stale: number; +} + +type Strategy = 'none' | 'ttl' | 'revalidate'; + +async function run(strategy: Strategy, changes: boolean): Promise { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const issues = + strategy === 'none' + ? stitch({ url: api.url, adapter: api.adapter(), clock }) + : strategy === 'ttl' + ? stitch({ + url: api.url, + adapter: api.adapter(), + clock, + store: clockStore(clock), + // Long enough to cover the whole run — the configuration a team reaches for + // when the goal is "stop hammering the API". + cache: { ttl: '30m', tenancy: 'app' }, + }) + : stitch({ + url: api.url, + kind: revalidating({ transport: api.adapter() }), + clock, + }); + + const versions: (number | null)[] = []; + let stale = 0; + for (let poll = 1; poll <= POLLS; poll++) { + if (changes && poll === CHANGES_BEFORE) api.mutate(); + const r = await issues.safe({}); + const got = + (r.data as { version?: number } | undefined)?.version ?? null; + versions.push(got); + // `api.body(...).version` is the server's CURRENT truth at this instant. + if (got !== api.body('(none)').version) stale++; + await clock.advance(INTERVAL_MS); + } + return { billed: api.billed, requests: api.requests, versions, stale }; +} + +const row = (label: string, r: RunResult): void => { + note( + ` ${label.padEnd(24)}`, + `billed ${String(r.billed).padStart(2)}/${String(POLLS)} requests ${String(r.requests).padStart(2)} stale ${String(r.stale)} versions ${JSON.stringify(r.versions)}`, + ); +}; + +async function main(): Promise { + heading('C8 — ten polls, three strategies, two worlds'); + + // ── the QUIET world: nothing changes ────────────────────────────────────────────────────── + heading(' QUIET — the resource never changes (the 99% case)'); + const quietNone = await run('none', false); + const quietTtl = await run('ttl', false); + const quietRev = await run('revalidate', false); + row('no caching', quietNone); + row('TTL cache (30m)', quietTtl); + row('revalidation', quietRev); + + check('QUIET no-caching billed', quietNone.billed, 10); + check('QUIET TTL billed', quietTtl.billed, 1); + check('QUIET revalidation billed', quietRev.billed, 1); + check('QUIET revalidation requests', quietRev.requests, 10); + check('QUIET revalidation stale polls', quietRev.stale, 0); + check('QUIET TTL stale polls', quietTtl.stale, 0); + note( + ' → in the quiet world TTL and revalidation cost the SAME', + 'both bill 1; the difference is that revalidation still asked, 9 times, for free', + ); + + // ── the CHANGE world: the resource moves once, before poll 6 ────────────────────────────── + heading( + ' CHANGE — the resource changes once, before poll 6 (the 1% the loop exists for)', + ); + const changeNone = await run('none', true); + const changeTtl = await run('ttl', true); + const changeRev = await run('revalidate', true); + row('no caching', changeNone); + row('TTL cache (30m)', changeTtl); + row('revalidation', changeRev); + + check('CHANGE no-caching billed', changeNone.billed, 10); + check('CHANGE no-caching stale polls', changeNone.stale, 0); + check('CHANGE TTL billed', changeTtl.billed, 1); + check('CHANGE TTL stale polls', changeTtl.stale, 5); + checkSeq( + 'CHANGE TTL versions', + changeTtl.versions, + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ); + check('CHANGE revalidation billed', changeRev.billed, 2); + check('CHANGE revalidation stale polls', changeRev.stale, 0); + checkSeq( + 'CHANGE revalidation versions', + changeRev.versions, + [1, 1, 1, 1, 1, 2, 2, 2, 2, 2], + ); + note( + ' → this is what a TTL cache costs', + 'the same 1 billed response, and it NEVER SAW the change: 5 of 10 polls served version 1', + ); + + // ── the SAVING, stated the way a rate-limit budget states it ────────────────────────────── + { + const saved = changeNone.billed - changeRev.billed; + check('rate-limit responses saved by revalidation', saved, 8); + note( + ' → 8 of 10 polls became free, with zero staleness', + 'at GitHub’s 5,000/hour primary limit, a 1-minute poll of 50 resources goes from 3,000/h to 600/h', + ); + } + + // ── the LOAD-BALANCER INODE CASE: revalidation that silently does nothing ───────────────── + // Apache's default `FileETag` embeds the inode, so two servers behind a balancer mint different + // validators for byte-identical content and `If-None-Match` never matches. The client is correct, + // the config is correct, and the feature is worth exactly nothing — with no error to notice. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, inodeEtags: true }); + const kind = revalidating({ transport: api.adapter() }); + const issues = stitch({ url: api.url, kind, clock }); + for (let i = 0; i < POLLS; i++) { + await issues.safe({}); + await clock.advance(INTERVAL_MS); + } + check('INODE billed', api.billed, 10); + check('INODE requests', api.requests, 10); + check('INODE 304s', api.notModified, 0); + check( + 'INODE revalidated (the surface’s own counter)', + kind.stats.revalidated, + 0, + ); + check('INODE stored', kind.stats.stored, 10); + note( + ' → 10 validators sent, 10 full responses back, no error anywhere', + '`stats.revalidated === 0` while `stats.stored === 10` is the shape of this failure — assert on it', + ); + } + + // ── the SERVER WITH NO ETAG AT ALL — the other silent nothing ───────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock }); + const noEtag = async ( + req: Parameters>[0], + ): ReturnType> => { + const res = await api.adapter()(req); + const { etag: _dropped, ...rest } = res.headers; + return { ...res, headers: rest }; + }; + const kind = revalidating({ transport: noEtag }); + const issues = stitch({ url: api.url, kind, clock }); + for (let i = 0; i < 3; i++) await issues.safe({}); + check('NO-ETAG billed', api.billed, 3); + check('NO-ETAG stored', kind.stats.stored, 0); + check('NO-ETAG unvalidatable', kind.stats.unvalidatable, 3); + note( + ' → distinguishable from the inode case, and from working', + '`stats.unvalidatable` counts 200s the server refused to give a validator for', + ); + } + + finish( + 'C8', + 'MEASURED, and the TTL row is the one worth reading twice. Ten polls at one-minute virtual intervals. QUIET world (nothing changes): no caching bills 10/10; a 30-minute TTL cache bills 1/10; revalidation bills 1/10 while still making all 10 requests — so in the case a poller spends 99% of its time in, TTL and revalidation cost the SAME. CHANGE world (the resource moves once, before poll 6): no caching bills 10/10 and is never stale; revalidation bills 2/10 and is never stale, versions `[1,1,1,1,1,2,2,2,2,2]` — the change picked up on the very poll it happened; the TTL cache bills 1/10 and NEVER SEES THE CHANGE — versions `[1,1,1,1,1,1,1,1,1,1]`, 5 of 10 polls serving a superseded version. That is what the extra billed response buys, and it is the whole argument: 8 of 10 polls became free with ZERO staleness. Two silent-failure modes are measured alongside, because both look exactly like success: the load-balancer INODE case (a server minting a fresh validator per response) polls 10 times, bills 10, gets 0 304s and raises nothing — detectable only as `stats.revalidated === 0` while `stats.stored === 10`; and a server that sends no `ETag` at all bills 3/3 with `stats.stored === 0` and `stats.unvalidatable === 3`. Both are worth an assertion in a real deployment, because neither will ever produce an error', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/c9-assembled-solution.ts b/docs/scenarios/proofs/conditional-requests-304/c9-assembled-solution.ts new file mode 100644 index 00000000..8586010e --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/c9-assembled-solution.ts @@ -0,0 +1,283 @@ +// C9 — the assembled answer, run against every shape in this scenario, and compared honestly with +// the hand-rolled equivalent. +// +// Both implementations drive the SAME fake server through the SAME `fetch`-shaped entry point — the +// StitchAPI side through the real `fetchAdapter({ fetch })`, the hand-rolled side calling it +// directly — so neither gets a shortcut on the transport, and the comparison asserts their +// observable results are IDENTICAL on every shape before the line counts are read off the files. +// +// pnpm exec tsx docs/scenarios/proofs/conditional-requests-304/c9-assembled-solution.ts +import { bearer } from '../../../../packages/core/src/auth'; +import { fetchAdapter, stitch } from '../../../../packages/core/src/index'; +import { seam } from '../../../../packages/core/src/index'; +import type { StandardSchemaV1 } from '../../../../packages/core/src/standard-schema'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeEtagApi } from './fake-etag-api'; +import { handRolledClient } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { revalidating } from './revalidate'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * Executable lines of a proof file — the comparable unit. Import statements (single- and + * multi-line), blank lines and comment-only lines are all removed, on BOTH sides, so the number is + * the code someone actually has to write and maintain. + */ +function executableLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .replace(/^import[\s\S]*?;$/gm, '') // whole import statements, however they wrap + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +/** The shapes this scenario is about, each as a server factory + a poll script. */ +const SHAPES = { + 'quiet (4 polls, no change)': { opts: {}, mutateBefore: 0, polls: 4 }, + 'one change at poll 3': { opts: {}, mutateBefore: 3, polls: 4 }, + 'weak validators': { opts: { weak: true }, mutateBefore: 3, polls: 4 }, + 'inode ETags (never matches)': { + opts: { inodeEtags: true }, + mutateBefore: 0, + polls: 4, + }, +} as const; + +/** What a run produced, in a form both implementations can be compared on. */ +interface Observed { + versions: (number | null)[]; + statuses: number[]; + billed: number; + requests: number; +} + +async function main(): Promise { + heading( + 'C9 — the assembled answer, on every shape, against the hand-rolled twin', + ); + + const stitched: Record = {}; + const rolled: Record = {}; + + for (const [label, shape] of Object.entries(SHAPES)) { + // ── the StitchAPI answer ────────────────────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, ...shape.opts }); + const issues = stitch({ + url: api.url, + kind: revalidating({ + transport: fetchAdapter({ fetch: api.fetchImpl() }), + }), + clock, + }); + const versions: (number | null)[] = []; + for (let poll = 1; poll <= shape.polls; poll++) { + if (poll === shape.mutateBefore) api.mutate(); + const r = await issues.safe({}); + versions.push( + (r.data as { version?: number } | undefined)?.version ?? + null, + ); + } + stitched[label] = { + versions, + statuses: api.statuses, + billed: api.billed, + requests: api.requests, + }; + } + // ── the hand-rolled twin ────────────────────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, ...shape.opts }); + const client = handRolledClient(api.fetchImpl()); + const versions: (number | null)[] = []; + for (let poll = 1; poll <= shape.polls; poll++) { + if (poll === shape.mutateBefore) api.mutate(); + const d = await client.get(api.url); + versions.push( + (d as { version?: number } | undefined)?.version ?? null, + ); + } + rolled[label] = { + versions, + statuses: api.statuses, + billed: api.billed, + requests: api.requests, + }; + } + } + + heading(' the two implementations, shape by shape'); + for (const label of Object.keys(SHAPES)) { + const a = stitched[label]; + const b = rolled[label]; + if (!a || !b) continue; + note( + ` ${label.padEnd(28)}`, + `versions ${JSON.stringify(a.versions)} statuses ${JSON.stringify(a.statuses)} billed ${String(a.billed)}/${String(a.requests)}`, + ); + checkSeq(` ${label} — versions identical`, a.versions, b.versions); + checkSeq(` ${label} — statuses identical`, a.statuses, b.statuses); + check(` ${label} — billed identical`, a.billed, b.billed); + } + + // ── the numbers each shape is supposed to produce ───────────────────────────────────────── + heading(' the numbers themselves'); + checkSeq( + 'quiet versions', + stitched['quiet (4 polls, no change)']?.versions ?? [], + [1, 1, 1, 1], + ); + check('quiet billed', stitched['quiet (4 polls, no change)']?.billed, 1); + checkSeq( + 'changed versions', + stitched['one change at poll 3']?.versions ?? [], + [1, 1, 2, 2], + ); + check('changed billed', stitched['one change at poll 3']?.billed, 2); + checkSeq( + 'weak versions', + stitched['weak validators']?.versions ?? [], + [1, 1, 2, 2], + ); + check('weak billed', stitched['weak validators']?.billed, 2); + check( + 'inode billed (the feature buys nothing)', + stitched['inode ETags (never matches)']?.billed, + 4, + ); + + // ── what the StitchAPI side keeps that the hand-rolled side gave up ─────────────────────── + // The behaviour is identical, so the extra lines have to be buying something else. They buy the + // things that stayed CONFIG: auth, the contract, the trace spine, the per-principal keying. + { + const clock = manualClock(); + const api = new FakeEtagApi({ clock, etagScope: 'content' }); + const issuesSchema: StandardSchemaV1 = { + '~standard': { + version: 1, + vendor: 'conditional-requests-304-proof', + validate: (value: unknown) => + (value as { repo?: unknown } | null)?.repo === undefined + ? { issues: [{ message: 'not the issues payload' }] } + : { value }, + }, + }; + const kind = revalidating({ + transport: fetchAdapter({ fetch: api.fetchImpl() }), + }); + const sm = seam({ clock }); + const forUser = (p: string) => + sm.as(p).stitch({ + url: api.url, + kind, + auth: bearer(`tok-${p}`), + output: issuesSchema, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + timeout: { perAttempt: '5s' }, + }); + const alice = forUser('alice'); + const bob = forUser('bob'); + await alice.safe({}); + await bob.safe({}); + const a2 = await alice.safe({}); + const b2 = await bob.safe({}); + check('per-credential store entries', kind.stats.size, 2); + check( + 'alice’s revalidated poll', + (a2.data as { viewer?: string }).viewer, + 'tok-alice', + ); + check( + 'bob’s revalidated poll', + (b2.data as { viewer?: string }).viewer, + 'tok-bob', + ); + check('revalidated', kind.stats.revalidated, 2); + check('billed', api.billed, 2); + + // The trace spine is intact: one `start`/`request`/`result`/`done` per poll, under one run. + const events: string[] = []; + for await (const e of alice.stream({})) + events.push(e.type === 'progress' ? `progress:${e.phase}` : e.type); + checkSeq('event spine on a revalidated poll', events, [ + 'start', + 'progress:request', + 'result', + 'done', + ]); + + // …and the contract still runs on the SUBSTITUTED body (C4). + const probe = await alice.inspect({}); + check('inspect().status on a revalidated poll', probe.status, 304); + check( + 'inspect().data.repo', + (probe.data as { repo?: string }).repo, + 'octo/hello', + ); + note( + ' → `auth`, `output`, `retry`, `timeout`, `seam.as()` and the trace all stayed CONFIG', + 'none of them appear in `revalidate.ts`, and all of them would have to be written into the hand-rolled twin', + ); + } + + // ── the line count, read off the files ──────────────────────────────────────────────────── + heading(' the line count'); + { + const mine = executableLines('revalidate.ts'); + const theirs = executableLines('hand-rolled.ts'); + note( + ' user code (`revalidate.ts`)', + `${String(mine)} executable lines`, + ); + note( + ' hand-rolled (`hand-rolled.ts`)', + `${String(theirs)} executable lines`, + ); + // The two files now implement the SAME five rules plus the same bounded store, so the + // difference is attributable rather than hand-waved — and it attributes to two helpers that + // exist only because the engine hands the surface a SHARED header record it never case-folds + // (engine.ts:232): `credentialOf` (dig the resolved credential back out of the headers) and + // `clearValidator` (remove `If-None-Match` in whatever casing someone else wrote it). The + // hand-rolled twin owns its own header object and needs neither. Asserted as a BAND rather + // than an exact figure, so a Prettier line-wrap cannot turn a formatting change into a + // failed claim; the exact delta is printed either way. + note( + ' the StitchAPI side is LONGER by', + `${String(mine - theirs)} lines`, + ); + check( + 'the two files are the same size (within 15 lines)', + mine - theirs < 15, + true, + ); + note( + ' → and those lines are the two case-folding helpers', + '`credentialOf` + `clearValidator` — both exist because engine.ts:232 never case-folds header names', + ); + note( + ' → the counts being close IS the result', + 'the hand-rolled twin has NO auth, NO schema, NO retry, NO timeout, NO trace, NO per-principal seam', + ); + } + + finish( + 'C9', + `ASSEMBLED AND RUN. ${String(executableLines('revalidate.ts'))} executable lines of user code (\`revalidate.ts\`) in ONE seam — \`Surface.execute\`, the only position that sees a request and its own response in one function call, and the only one downstream of \`cfg.auth.apply\` that can key an ETag store by credential. It needs no custom \`interpret\`: the substituted body rides back on a still-304 response and \`httpInterpret\` passes it through, so \`.inspect().status\` honestly reports 304 while \`.data\` is the resource. Measured against a FEATURE-MATCHED hand-rolled twin (${String(executableLines('hand-rolled.ts'))} executable lines — its own request assembly, JSON decoding, status check, auth header and bounded store all counted) across four shapes — quiet, one mid-run change, weak validators, and inode ETags that never match — the versions, the status spine and the billed counts are IDENTICAL on every one: quiet \`[1,1,1,1]\` at 1 billed of 4; changed \`[1,1,2,2]\` at 2 of 4; weak \`[1,1,2,2]\` at 2 of 4; inode 4 of 4 with the feature buying nothing. So the extra lines are not buying behaviour. They are buying what stayed CONFIG: \`auth\`/\`output\`/\`retry\`/\`timeout\`/\`seam.as()\` and the trace spine, measured on a stitch carrying all of them — 2 store entries for 2 principals, each seeing their own \`viewer\`, the contract running on the SUBSTITUTED body, and an event spine of \`[start, progress:request, result, done]\` per poll. Every one of those would have to be written INTO the hand-rolled file. The honest headline is that the StitchAPI side is LONGER, by ${String(executableLines('revalidate.ts') - executableLines('hand-rolled.ts'))} lines, and those lines attribute exactly: \`credentialOf\` and \`clearValidator\`, two helpers that exist only because the engine hands a surface a SHARED header record it never case-folds (engine.ts:232) — the hand-rolled twin owns its own header object and needs neither`, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/conditional-requests-304/clock-store.ts b/docs/scenarios/proofs/conditional-requests-304/clock-store.ts new file mode 100644 index 00000000..976f6ad1 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/clock-store.ts @@ -0,0 +1,51 @@ +// A `StitchStore` whose TTL is driven by an INJECTED clock, so `cache.ttl` expiry is virtual time. +// +// This exists because the default `memoryStore()` reads `now()` — `Date.now()` (store.ts:16,45 via +// util.ts:4) — and therefore ignores a stitch's `clock` entirely. C5 measures that directly: a +// `cache: { ttl: '1s' }` stitch on a `manualClock()` advanced by a virtual HOUR still serves the +// cached entry. Every claim here that needs an expiring cache injects this instead, so the +// staleness numbers in C8 are exact rather than timing-dependent. +// +// It is a faithful copy of `memoryStore`'s semantics with `now()` swapped for `clock.now()`: the +// `expires === 0` sentinel means "no TTL", `set(key, undefined)` deletes, and `increment` keeps the +// first window's expiry. The opportunistic sweep is omitted — a proof run stores a handful of keys. +import type { Clock, StitchStore } from '../../../../packages/core/src/types'; + +export function clockStore(clock: Clock): StitchStore { + const data = new Map(); + const live = (e?: { expires: number }): boolean => + !!e && (e.expires === 0 || e.expires > clock.now()); + return { + async get(key) { + const e = data.get(key); + if (!live(e)) { + data.delete(key); + return undefined; + } + return e?.value; + }, + async set(key, value, ttl) { + if (value === undefined) { + data.delete(key); + return; + } + data.set(key, { value, expires: ttl ? clock.now() + ttl : 0 }); + }, + async increment(key, ttl) { + const e = data.get(key); + const n = (live(e) ? (e?.value as number) : 0) + 1; + data.set(key, { + value: n, + expires: live(e) + ? (e?.expires ?? 0) + : ttl + ? clock.now() + ttl + : 0, + }); + return n; + }, + async close() { + data.clear(); + }, + }; +} diff --git a/docs/scenarios/proofs/conditional-requests-304/fake-etag-api.ts b/docs/scenarios/proofs/conditional-requests-304/fake-etag-api.ts new file mode 100644 index 00000000..36de4789 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/fake-etag-api.ts @@ -0,0 +1,274 @@ +// A fake, in-memory CONDITIONAL-REQUEST API, shaped the way GitHub's REST API is: +// +// GET /repos/o/r/issues → 200 + body + `ETag: "v1"` +// GET … with `If-None-Match: "v1"` (matching) → **304, NO body**, `ETag: "v1"` echoed +// GET … with `If-None-Match: "v0"` (stale) → 200 + the NEW body + `ETag: "v2"` +// +// Everything runs off an injected {@link Clock} and nothing touches the network. The provider +// records EVERY hit with the exact `If-None-Match` string it received, which makes four things +// measurements rather than arguments: +// +// - `billed` — responses that would count against a rate limit. GitHub's rule: a 304 answered +// from a correctly-authorized conditional request costs nothing, a 200 costs one. This counter +// IS the payoff C8 measures. +// - `validators` — the byte-exact `If-None-Match` value on every request (`'(none)'` when the +// header was absent). `W/"v1"` surviving as `W/"v1"` is C7's whole claim. +// - `notModified` / `full` — the response mix. +// - per-`token` state — the resource's ETag is minted per credential, the way GitHub's is, so +// replaying principal A's validator as principal B is visibly a MISS (C6). +// +// The 304 body is `undefined`, not `null` and not `''`. That is not a guess: `fetchAdapter` decodes +// a zero-byte JSON response as `text === '' ? undefined` (http-adapter.ts:135), and C1 pins that +// end-to-end through the real adapter with an injected `fetch` rather than trusting this fake. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +/** One recorded request, as the server saw it. */ +export interface RecordedHit { + method: string; + /** Path only (the fake has one host), e.g. `/repos/o/r/issues`. */ + path: string; + /** The RAW `If-None-Match` value received, byte-exact, or `'(none)'` when absent. */ + inm: string; + /** The bearer token the request carried, or `'(none)'` — the ETag namespace. */ + token: string; + status: number; + /** Would this response count against a rate limit? `false` only for a 304. */ + billed: boolean; + /** Virtual time (ms) the request arrived, read off the injected clock. */ + at: number; +} + +/** The resource body. `version` moves on every `mutate()`; `etag` tracks it. */ +export interface Issues { + repo: string; + version: number; + /** WHOSE view this is — the token that asked. Private per credential. */ + viewer: string; + issues: { id: number; title: string }[]; +} + +export interface EtagApiOptions { + clock: Clock; + /** + * How the server namespaces its validators. + * + * - `'token'` (default) — the ETag string embeds the credential, GitHub-style. One principal's + * validator can never match another's, so a cross-principal replay merely MISSES. + * - `'content'` — the ETag is derived from the representation alone, which is what a naive + * server (and Apache's non-inode default) does. The BODY still differs per credential + * (`viewer`), so replaying principal A's validator on principal B's request returns **304** + * and a client that then serves its stored body hands B **A's private data**. This is the + * footgun the per-credential keying rule exists to prevent, and it is only visible against + * a server shaped like this one. + */ + etagScope?: 'token' | 'content'; + /** + * Mint WEAK validators (`W/"v1"`) instead of strong ones (`"v1"`). RFC 9110 says + * `If-None-Match` compares WEAKLY, so a compliant server matches `W/"v1"` against `"v1"` — + * this fake does the same, and records what it was actually sent either way. + */ + weak?: boolean; + /** + * The **load-balancer inode case**: every response gets a fresh, unique ETag even when the + * body is byte-identical (Apache's default `FileETag` embeds the inode, so two servers behind + * a balancer never agree). Revalidation then NEVER succeeds and the feature silently does + * nothing — measurable as `billed === polls`. + */ + inodeEtags?: boolean; + /** Repo name in the body/path. Default `'octo/hello'`. */ + repo?: string; +} + +const HOST = 'https://api.github.example'; + +/** Case-insensitive header read — a real server does not care how the client cased the name. */ +const header = ( + headers: Record | undefined, + name: string, +): string | undefined => { + if (!headers) return undefined; + const lc = name.toLowerCase(); + for (const [k, v] of Object.entries(headers)) + if (k.toLowerCase() === lc) return v; + return undefined; +}; + +/** + * RFC 9110 §8.8.3.2 WEAK comparison, which is the one `If-None-Match` is specified to use: strip a + * leading `W/` from both sides and compare the opaque quoted tags. `W/"v1"` therefore matches + * `"v1"`. `*` matches any existing representation. + */ +const weakMatch = (candidate: string, current: string): boolean => { + const strip = (t: string): string => (t.startsWith('W/') ? t.slice(2) : t); + return candidate.split(',').some((raw) => { + const t = raw.trim(); + return t === '*' || strip(t) === strip(current); + }); +}; + +/** + * The conditional-request API. One instance is one server. State is per-`token` because ETags are + * per-credential on GitHub: `mutate()` moves the resource for everyone, but each token's minted + * validator is its own string, so one principal's validator is never valid for another. + */ +export class FakeEtagApi { + /** Every hit, in order. */ + readonly hits: RecordedHit[] = []; + private readonly clock: Clock; + private readonly weak: boolean; + private readonly inodeEtags: boolean; + private readonly etagScope: 'token' | 'content'; + private readonly repo: string; + /** Bumped by `mutate()`; the resource's content version. */ + private version = 1; + /** Per-token ETag suffix, so two credentials never mint the same validator string. */ + private readonly tokenTag = new Map(); + private nextTokenTag = 0; + /** Monotonic counter behind `inodeEtags` — a new "inode" on every single response. */ + private inode = 0; + + constructor(opts: EtagApiOptions) { + this.clock = opts.clock; + this.weak = opts.weak ?? false; + this.inodeEtags = opts.inodeEtags ?? false; + this.etagScope = opts.etagScope ?? 'token'; + this.repo = opts.repo ?? 'octo/hello'; + } + + /** The resource's URL. */ + get url(): string { + return `${HOST}/repos/${this.repo}/issues`; + } + + /** Responses that would COUNT against a rate limit — every non-304. The payoff metric. */ + get billed(): number { + return this.hits.filter((h) => h.billed).length; + } + + /** How many requests arrived at all (billed or not). */ + get requests(): number { + return this.hits.length; + } + + /** How many were answered `304 Not Modified`. */ + get notModified(): number { + return this.hits.filter((h) => h.status === 304).length; + } + + /** The byte-exact `If-None-Match` value on every request, in order. */ + get validators(): string[] { + return this.hits.map((h) => h.inm); + } + + /** The status of every response, in order. */ + get statuses(): number[] { + return this.hits.map((h) => h.status); + } + + /** Change the resource. Every stored validator is now stale. */ + mutate(): void { + this.version += 1; + } + + /** The validator this server would mint right now for `token`. */ + etagFor(token: string): string { + if (!this.tokenTag.has(token)) + this.tokenTag.set(token, `t${(this.nextTokenTag += 1)}`); + const scope = + this.etagScope === 'content' + ? '' + : `.${this.tokenTag.get(token) ?? 't0'}`; + const tag = this.inodeEtags + ? `i${(this.inode += 1)}` + : `v${this.version}${scope}`; + return this.weak ? `W/"${tag}"` : `"${tag}"`; + } + + /** The body served at the current version, as `token` sees it. */ + body(token: string): Issues { + return { + repo: this.repo, + version: this.version, + viewer: token, + issues: Array.from({ length: this.version }, (_, i) => ({ + id: i + 1, + title: `issue ${i + 1} for ${token}`, + })), + }; + } + + /** A StitchAPI {@link Adapter} bound to this server. */ + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const { status, etag, body } = this.handle( + req.method, + req.url, + req.headers, + ); + // A real 304 carries NO body. `fetchAdapter` decodes an empty JSON response as + // `undefined` (http-adapter.ts:135), so that is what the wire hands the engine. + return { + status, + headers: { etag, 'content-type': 'application/json' }, + body, + }; + }; + } + + /** + * A `fetch`-shaped entry point onto the same server. C9's hand-rolled twin drives this, and the + * StitchAPI side drives it through the REAL `fetchAdapter({ fetch })` — so the comparison runs + * both implementations over one transport contract rather than giving either a shortcut. + */ + fetchImpl(): typeof fetch { + return async (input, init) => { + const url = typeof input === 'string' ? input : String(input); + const headers: Record = {}; + new Headers(init?.headers).forEach((v, k) => { + headers[k] = v; + }); + const { status, etag, body } = this.handle( + init?.method ?? 'GET', + url, + headers, + ); + return new Response(status === 304 ? null : JSON.stringify(body), { + status, + headers: { etag, 'content-type': 'application/json' }, + }); + }; + } + + /** The one request handler both entry points share. */ + private handle( + method: string, + url: string, + headers: Record, + ): { status: number; etag: string; body: Issues | undefined } { + const inm = header(headers, 'if-none-match'); + const auth = header(headers, 'authorization'); + const token = auth ? auth.replace(/^Bearer\s+/i, '') : '(none)'; + const current = this.etagFor(token); + const matched = inm !== undefined && weakMatch(inm, current); + const status = matched ? 304 : 200; + this.hits.push({ + method, + path: new URL(url).pathname, + inm: inm ?? '(none)', + token, + status, + billed: status !== 304, + at: this.clock.now(), + }); + return { + status, + etag: current, + body: matched ? undefined : this.body(token), + }; + } +} diff --git a/docs/scenarios/proofs/conditional-requests-304/hand-rolled.ts b/docs/scenarios/proofs/conditional-requests-304/hand-rolled.ts new file mode 100644 index 00000000..98b5bfc4 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/hand-rolled.ts @@ -0,0 +1,104 @@ +// The same five rules with NO StitchAPI in them, so C9's line-count comparison is honest and its +// behaviour comparison is exact. It drives the same fake server through the same `fetch`-shaped +// entry point the StitchAPI side reaches through `fetchAdapter({ fetch })`, so neither side gets a +// shortcut on the transport. +// +// The rules, identical to `revalidate.ts`: +// 1. replay the stored validator as `If-None-Match` +// 2. a 304 is answered with the stored body +// 3. a 200 with an ETag stores `{ etag, body }`; a 200 without one FORGETS the key +// 4. the key folds in the credential +// 5. a 304 with no stored body (an orphan validator) refetches unconditionally +// +// …plus the bounded store, so the feature sets match line for line and the count difference is not +// this file quietly leaving something out. +// +// What is NOT free here, and is what the extra lines on the StitchAPI side buy: request assembly, +// JSON decoding, the non-2xx check, and the auth header — all of which a real hand-rolled client +// genuinely has to write, so all of which are counted. + +export interface HandRolledEntry { + etag: string; + body: unknown; +} + +export interface HandRolledStats { + revalidated: number; + stored: number; + orphans: number; + unvalidatable: number; + size: number; +} + +export interface HandRolledClient { + get(url: string): Promise; + readonly stats: HandRolledStats; +} + +export function handRolledClient( + fetchImpl: typeof fetch, + token?: string, + entries = 500, +): HandRolledClient { + const store = new Map(); + const stats: HandRolledStats = { + revalidated: 0, + stored: 0, + orphans: 0, + unvalidatable: 0, + size: 0, + }; + + const send = async (url: string, etag?: string): Promise => { + const headers: Record = { accept: 'application/json' }; + if (token !== undefined) headers['authorization'] = `Bearer ${token}`; + if (etag !== undefined) headers['if-none-match'] = etag; + return fetchImpl(url, { method: 'GET', headers }); + }; + + const learn = async (key: string, res: Response): Promise => { + const text = await res.text(); + const body: unknown = text === '' ? undefined : JSON.parse(text); + const etag = res.headers.get('etag'); + if (etag === null) { + stats.unvalidatable++; + store.delete(key); + } else { + if (store.has(key)) store.delete(key); + store.set(key, { etag, body }); + stats.stored++; + while (store.size > entries) { + const oldest = store.keys().next().value; + if (oldest === undefined) break; + store.delete(oldest); + } + } + stats.size = store.size; + return body; + }; + + return { + stats, + async get(url) { + const key = `GET ${url} ${token ?? ''}`; + const entry = store.get(key); + const res = await send(url, entry?.etag); + if (res.status === 304) { + if (entry) { + stats.revalidated++; + return entry.body; + } + stats.orphans++; + store.delete(key); + stats.size = store.size; + const fresh = await send(url); + if (fresh.status !== 200) + throw new Error(`HTTP ${String(fresh.status)}`); + return learn(key, fresh); + } + if (res.status !== 200) + throw new Error(`HTTP ${String(res.status)}`); + return learn(key, res); + }, + }; +} diff --git a/docs/scenarios/proofs/conditional-requests-304/harness.ts b/docs/scenarios/proofs/conditional-requests-304/harness.ts new file mode 100644 index 00000000..49c018a1 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/harness.ts @@ -0,0 +1,63 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is mostly COUNTS and HEADER STRINGS: how many responses would have +// counted against a rate limit, and what `If-None-Match` actually went out on the wire. So both +// assertions print the measured value whether they pass or fail — `["(none)","W/\"v1\""]` is the +// finding, and it has to be readable out of context. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the validator spine + * (`["(none)", "\"v1\"", "\"v1\""]`) and the status spine (`[200, 304, 304]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/conditional-requests-304/revalidate.ts b/docs/scenarios/proofs/conditional-requests-304/revalidate.ts new file mode 100644 index 00000000..6071e499 --- /dev/null +++ b/docs/scenarios/proofs/conditional-requests-304/revalidate.ts @@ -0,0 +1,157 @@ +// USER CODE — the assembled answer this scenario arrives at, and the subject of C9's line count. +// +// It is ONE seam: a `Surface` whose only hook is `execute` (ADR 0008). `execute` replaces the +// transport at engine.ts:666-674 — inside the resilience chain, so `retry`/`throttle`/`circuit`/ +// `timeout`/`trace` still wrap it — and it is the only position in the library that sees a request +// AND its own response in one function call. That is what the rest of the pieces need: +// +// • the REQUEST, after `cfg.auth.apply` (engine.ts:649), so the store can be keyed by the +// resolved credential. `Surface.buildRequest` runs before auth and cannot; `ResolvedStitchConfig` +// carries no `principal` at all (C6). +// • the RESPONSE, so a 304 can be answered with the stored body. The substituted body rides back +// on a response whose status is STILL 304, and `httpInterpret` hands it to the caller unchanged, +// because 304 was never a failure (C1). No custom `interpret` is needed, and the status stays +// honest: `.inspect().status` reports 304 while `.data` is the resource. +// • both together, so a response is correlated with ITS OWN request rather than with whatever a +// closure variable last held (C6 case f measured that pairing losing entries under concurrency). +// +// The cost is that a surface with `execute` ignores `StitchConfig.adapter` (surface.ts:116), so the +// transport is passed in here instead of configured on the stitch. +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +/** What is stored per key: the two halves that must never be separated. */ +export interface ValidatorEntry { + /** The server's opaque validator, byte-exact — `"v1"` or `W/"v1"` (C7). */ + etag: string; + /** The body that validator describes. This is what a 304 means "you already have". */ + body: unknown; +} + +export interface RevalidateOptions { + /** The real transport. Required: a surface with `execute` ignores `StitchConfig.adapter`. */ + transport: Adapter; + /** + * Key derivation. The default folds in the resolved `Authorization` header, because ETags are + * per-credential: replaying principal A's validator on principal B's request returns a 304 from + * a server whose validators are content-derived, and the client then serves A's body to B (C6). + */ + keyOf?: (req: AdapterRequest) => string; + /** Cap on stored entries, oldest-first. Default 500 — a poller runs for weeks. */ + entries?: number; +} + +/** Counters, so a caller can assert the thing is working rather than assume it. */ +export interface RevalidateStats { + /** 304s answered from the store — the polls that cost nothing. */ + revalidated: number; + /** 200s whose ETag+body were stored. */ + stored: number; + /** 304s that arrived with NO stored body (an orphan validator), refetched unconditionally. */ + orphans: number; + /** 200s carrying no `ETag` — the server is not minting validators, so nothing can be saved. */ + unvalidatable: number; + /** Live entries. */ + size: number; +} + +/** The resolved `Authorization` header, or `''`. The engine never case-folds header names. */ +const credentialOf = (headers: Record): string => { + for (const [k, v] of Object.entries(headers)) + if (k.toLowerCase() === 'authorization') return v; + return ''; +}; + +/** Remove any casing of `If-None-Match` — the engine keeps whatever spelling was written (C7). */ +const clearValidator = (headers: Record): void => { + for (const k of Object.keys(headers)) + if (k.toLowerCase() === 'if-none-match') delete headers[k]; +}; + +/** + * A conditional-request surface: replay the stored validator, and answer a 304 with the stored body. + * + * ```ts + * const issues = stitch({ + * url: 'https://api.github.com/repos/o/r/issues', + * kind: revalidating({ transport: fetchAdapter() }), + * auth: bearer(env('GITHUB_TOKEN')), + * }); + * ``` + */ +export function revalidating( + opts: RevalidateOptions, +): Surface & { readonly stats: RevalidateStats } { + const { transport } = opts; + const max = opts.entries ?? 500; + const keyOf = + opts.keyOf ?? + ((req: AdapterRequest): string => + `${req.method} ${req.url} ${credentialOf(req.headers)}`); + // Insertion-ordered, so the oldest key is `keys().next()` — the LRU shape `cache` uses too. + const store = new Map(); + const stats: RevalidateStats = { + revalidated: 0, + stored: 0, + orphans: 0, + unvalidatable: 0, + size: 0, + }; + + /** Record what a fresh 200 taught us — or forget the key when it taught us nothing. */ + const learn = (key: string, res: AdapterResponse): void => { + if (res.status !== 200) return; + const etag = res.headers['etag']; + if (etag === undefined) { + stats.unvalidatable++; + store.delete(key); + } else { + if (store.has(key)) store.delete(key); + store.set(key, { etag, body: res.body }); + stats.stored++; + while (store.size > max) { + const oldest = store.keys().next().value; + if (oldest === undefined) break; + store.delete(oldest); + } + } + stats.size = store.size; + }; + + return { + id: 'http+revalidate', + stats, + execute: async (req) => { + const key = keyOf(req); + const entry = store.get(key); + if (entry) req.headers['If-None-Match'] = entry.etag; + else clearValidator(req.headers); + + const res = await transport(req); + + if (res.status !== 304) { + learn(key, res); + return res; + } + if (entry) { + stats.revalidated++; + // The status stays 304 on purpose — `.inspect()` should not be lied to. + return { ...res, body: entry.body }; + } + // ORPHAN VALIDATOR: a stored ETag whose body we no longer have (a restarted process, + // an evicted entry, a validator persisted without its payload). Returning the 304 as-is + // would rebuild the C1 bug by hand, so refetch unconditionally. One extra request, once. + stats.orphans++; + store.delete(key); + stats.size = store.size; + clearValidator(req.headers); + const fresh = await transport(req); + learn(key, fresh); + return fresh; + }, + }; +} diff --git a/docs/scenarios/proofs/cost-based-rate-limits/README.md b/docs/scenarios/proofs/cost-based-rate-limits/README.md new file mode 100644 index 00000000..37e0e785 --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/README.md @@ -0,0 +1,65 @@ +# Proofs — cost-based rate limits reported in the response body + +Runnable evidence for the claims in +[`../../cost-based-rate-limits.md`](../../cost-based-rate-limits.md). + +Every script is standalone, offline, and deterministic: it injects a fake Shopify GraphQL Admin API +through StitchAPI's `adapter` seam and drives refill off an injected `manualClock()`, so every +number below is exact virtual time — no wall-clock sleeps, nothing flaky, no network. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c1-retry-on-200-throttled.ts + +# all of them +for f in docs/scenarios/proofs/cost-based-rate-limits/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, +so they test the working tree, not the published bundle. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------- | +| `c1-retry-on-200-throttled.ts` | can `retry` fire on a 200-with-THROTTLED? | **No.** `retry.on` is handed **1** argument, the number `200` | +| `c2-computed-wait.ts` | can the wait be computed from the body? | **Not via `backoff`** — but `SurfaceOutcome.after` is honoured exactly (6000ms) | +| `c3-extensions-reachability.ts` | is `extensions.cost` reachable? | **Only via `hooks.onResponse`.** Error, event stream and `.inspect().raw` all lose it | +| `c4-delegate-on-200.ts` | does `throttle.delegate` trip on a 200? | **No** (default `[429]`). `on: 200` trips on **successes** too | +| `c5-throttle-rate-cost.ts` | can `throttle.rate` express a cost budget? | **No.** 10 calls the bucket takes instantly are spread over **18s** | +| `c6-assembled-solution.ts` | can a user assemble correct behaviour? | **Yes** — a custom `Surface`, **73 lines**, 8/8 queries survive a hostile neighbour | + +## Files + +- `fake-shopify.ts` — the provider: a 1000-point leaky bucket refilling 50/s, per-operation cost, + 200-with-`THROTTLED` on over-spend, `extensions.cost` on **every** response, and `drain()` to + simulate a third-party app spending the shop's shared bucket. +- `harness.ts` — `check` / `checkNear` / `note` / `heading` / `finish`. No test framework. +- `shopify-cost-surface.ts` — **user code** for C6: the `CostLedger`, the cost-aware `Surface`, and + the `costGate` proactive hook. + +## Reading the numbers honestly + +- **C1(f) and C2(a2) are hacks, not seams.** Mutating `ctx.res.status` inside `onResponse` really + does drive the retry matcher (the hook fires at `engine.ts:705`, the matcher reads at `:743`) — + but it rewrites the status every later stage sees, and the final `StitchError.status` comes back + as the invented `429` rather than the wire's `200`. They are measured because "I couldn't find the + spelling" and "the built-in can't do it" are different claims, and both needed ruling out. +- **C2's `@ts-expect-error` blocks are the proof, not decoration.** A `@ts-expect-error` that is + _not_ an error fails `tsc`. These files typecheck clean under `packages/core`'s full strict set, + so every "this is a type error" claim is machine-checked. +- **C5's 18s is not a strawman.** `'1/2s'` is the _most generous_ spacing that never outruns a 50/s + refill at 100 points per query. The comparison is against the same 10 queries with no throttle at + all, which the 1000-point bucket absorbs at t=0. +- **C6 measures one process.** It proves the _logic_ is expressible on the public API. A deployment + with several workers on one shop needs the ledger in a shared store — the surface seam is + unchanged, but `CostLedger` would have to become async, and `interpret` is **synchronous** + (`surface.ts:61-64`), so a distributed ledger cannot live inside it. That is a real limit of this + design, not a detail. +- **C6(f)'s adapter alternative works but goes blind.** The engine reported `attempts: 1` and **0** + `retry` progress events for a call that really made two requests and slept 6s, because the loop + ran below the resilience chain. `timeout.total` does not bound it either. diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c1-retry-on-200-throttled.ts b/docs/scenarios/proofs/cost-based-rate-limits/c1-retry-on-200-throttled.ts new file mode 100644 index 00000000..9d4df4af --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c1-retry-on-200-throttled.ts @@ -0,0 +1,231 @@ +// C1 — can the BUILT-IN `retry` fire on an HTTP 200 carrying `errors[].extensions.code === +// 'THROTTLED'`? Every spelling on the public surface is tried here, and each one is measured by +// counting the requests the fake shop actually received. +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c1-retry-on-200-throttled.ts +import { graphql, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StatusMatch } from '../../../../packages/core/src/types'; +import { FakeShopify } from './fake-shopify'; +import { check, finish, heading, note } from './harness'; + +const DOC = 'query BigSync { products { id } }'; + +// A shop whose bucket is already empty, so the very first call is THROTTLED. +function emptyShop(): { + shop: FakeShopify; + clock: ReturnType; +} { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); // another app took the whole bucket + return { shop, clock }; +} + +async function main(): Promise { + heading('C1 — can `retry` fire on a 200-with-THROTTLED?'); + + // ── (a) the DEFAULT retry set: [429, 502, 503, 504] ──────────────────────────────────────── + { + const { shop, clock } = emptyShop(); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 5 }, + }); + const r = await call.safe(); + check('(a) default retry.on — requests made', shop.calls.length, 1); + check('(a) call failed', r.ok, false); + note('(a) error message', r.error?.message); + } + + // ── (b) a PREDICATE on `retry.on` — what does it actually receive? ───────────────────────── + // If the predicate were handed the response it could read `errors[]`. It is not: `acceptsStatus` + // (resilience.ts:25-32) returns the authored function unchanged and the engine calls it as + // `retryMatch(res.status)` (engine.ts:743) — one argument, a number. + { + const { shop, clock } = emptyShop(); + const received: unknown[][] = []; + const on = ((...args: unknown[]): boolean => { + received.push(args); + return false; + }) as StatusMatch; + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 5, on }, + }); + await call.safe(); + check('(b) predicate invoked', received.length > 0, true); + check('(b) arguments handed to retry.on', received[0]?.length, 1); + check('(b) argument type', typeof received[0]?.[0], 'number'); + check('(b) argument value', received[0]?.[0], 200); + note('(b) the body was NOT passed', JSON.stringify(received[0] ?? [])); + } + + // ── (c) `retry.on: 200` — it DOES retry, and that is the footgun ─────────────────────────── + // The status matcher runs BEFORE the surface interprets the body (engine.ts:743 vs :775), so + // `on: 200` cannot distinguish a THROTTLED 200 from a SUCCESSFUL one. It retries BOTH. + { + const { shop, clock } = emptyShop(); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 1000 }, + }, + }); + const p = call.safe(); + await clock.advance(10_000); + await p; + check('(c) retry.on:200 — requests made', shop.calls.length, 3); + note('(c) it retried, but only because EVERY 200 matches', 'see (d)'); + } + + // ── (d) the same config against a HEALTHY shop: successes are retried too ────────────────── + // This is the cost of (c): a call that succeeded on attempt 1 is fired 3 times and the shop is + // charged 3× the points. Nothing in the config can express "retry only the throttled 200". + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 1000 }, + }, + }); + const p = call.safe(); + await clock.advance(10_000); + const r = await p; + check('(d) healthy shop — requests made', shop.calls.length, 3); + check('(d) all three succeeded', shop.throttledCount, 0); + check('(d) call ok', r.ok, true); + check( + '(d) points the shop was charged for ONE logical call', + shop.pointsCharged, + 900, // 300 × 3 — the retry charged the shop three times for one result + ); + } + + // ── (e) `verdict.flag` — a BODY path, and it SILENTLY PASSES A THROTTLE THROUGH ──────────── + // `verdict.flag` is the one built-in that reads the body to decide a verdict, so it is the + // natural thing to reach for. On Shopify's throttled body it is INERT: the payload has no + // `data` key at all, and an ABSENT path is "no signal" (surface.ts:181-189), so the status + // verdict (200) stands. The caller is handed the THROTTLED envelope as a successful result. + { + const { shop, clock } = emptyShop(); + const call = stitch({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + method: 'POST', + adapter: shop.adapter(), + clock, + retry: { attempts: 5 }, + verdict: { flag: 'data.ok' }, + }); + const r = await call.safe(); + check('(e) verdict.flag — requests made', shop.calls.length, 1); + check('(e) the call REPORTED SUCCESS on a throttle', r.ok, true); + check( + '(e) …and handed the caller the THROTTLED envelope as data', + (r.data as { errors?: { message?: string }[] })?.errors?.[0] + ?.message, + 'Throttled', + ); + } + + // ── (e2) even when the flag IS present and falsy, it never retries ───────────────────────── + // A separate minimal body, because Shopify's throttled payload has no falsy flag to point at. + // `verdictOf` returns `{ ok: false, message, status }` — no `retry` arm (surface.ts:174-191) — + // so the 5-attempt budget is untouched. + { + const clock = manualClock(); + let hits = 0; + const call = stitch({ + url: 'https://api.example.com/a', + adapter: async () => { + hits++; + return { status: 200, headers: {}, body: { ok: false } }; + }, + clock, + retry: { attempts: 5 }, + verdict: { flag: 'ok' }, + }); + const r = await call.safe(); + check('(e2) flag present + falsy — call failed', r.ok, false); + check('(e2) requests made (retry.attempts was 5)', hits, 1); + note('(e2) message', r.error?.message); + } + + // ── (f) can an `onResponse` HOOK force a retry by mutating the status? ───────────────────── + // The hook fires at engine.ts:705, BEFORE the retry check at :743, and receives the live `res`. + // Mutating `res.status` there does reach the matcher — measured below — but it also rewrites + // what every later stage sees. This is a hack, not a supported seam; C6 measures the real one. + { + const { shop, clock } = emptyShop(); + let hookSawExtensions = false; + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 1000 } }, + hooks: { + onResponse: (ctx) => { + const body = ctx.res?.body as + { extensions?: unknown } | undefined; + if (body?.extensions !== undefined) + hookSawExtensions = true; + // Rewrite a THROTTLED 200 into a 429 so the DEFAULT retry set matches it. + const errs = ( + ctx.res?.body as + | { errors?: { extensions?: { code?: string } }[] } + | undefined + )?.errors; + if ( + ctx.res && + errs?.some((e) => e.extensions?.code === 'THROTTLED') + ) + ctx.res.status = 429; + }, + }, + }); + const p = call.safe(); + await clock.advance(10_000); + const r = await p; + check( + '(f) hook saw extensions on the response', + hookSawExtensions, + true, + ); + check( + '(f) status rewrite DID drive retry — requests', + shop.calls.length, + 3, + ); + check('(f) final call still failed', r.ok, false); + note( + '(f) final error status (rewritten, not the wire status)', + r.error?.status, + ); + } + + finish( + 'C1', + 'no built-in `retry` spelling fires on a 200-with-THROTTLED: `retry.on` is handed the STATUS ONLY (1 arg), `on: 200` cannot tell a throttled 200 from a good one, and `verdict.flag` fails without retrying', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c2-computed-wait.ts b/docs/scenarios/proofs/cost-based-rate-limits/c2-computed-wait.ts new file mode 100644 index 00000000..9b6746da --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c2-computed-wait.ts @@ -0,0 +1,264 @@ +// C2 — can the retry WAIT be computed from the response body, i.e. Shopify's own +// `(requestedQueryCost - currentlyAvailable) / restoreRate`, instead of a backoff curve? +// +// Arrival times are measured in VIRTUAL ms off `manualClock()`, so the gaps below are exact. +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c2-computed-wait.ts +import { graphql, stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { RetryOptions } from '../../../../packages/core/src/types'; +import { FakeShopify, costOfBody, deficitWaitMs } from './fake-shopify'; +import { check, checkNear, finish, heading, note } from './harness'; + +const DOC = 'query BigSync { products { id } }'; + +function emptyShop(cost: number): { + shop: FakeShopify; + clock: ReturnType; +} { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: cost } }); + shop.drain(1000); + return { shop, clock }; +} + +/** Gaps (ms) between successive requests, in virtual time. */ +const gaps = (shop: FakeShopify): number[] => + shop.calls.slice(1).map((c, i) => c.at - (shop.calls[i]?.at ?? 0)); + +async function main(): Promise { + heading('C2 — can the retry wait be COMPUTED from the body?'); + + // ── (a) is there a function form of `backoff` anywhere? ──────────────────────────────────── + // `backoff?: BackoffCurve | AtLeastOne` (types.ts:991) — `BackoffOptions` is + // `{ curve, base, max }` (types.ts:967-974). No arm of that union is callable. The + // `@ts-expect-error` below is the PROOF: this file typechecks only if the line IS an error. + { + // @ts-expect-error — a delay function is not assignable to `backoff`; there is no such form. + const rejected: RetryOptions = { attempts: 2, backoff: () => 6000 }; + void rejected; + check('(a) `backoff: () => ms` is a TYPE ERROR', true, true); + } + + // ── (a2) and if you cast past it, it is SILENTLY IGNORED ─────────────────────────────────── + // `backoffDelay` (resilience.ts:38-56) reads `.curve` / `.base` / `.max` off whatever it is + // given. A function has none of them, so it degrades to the default `expo-jitter`/100ms curve + // and is never called — no throw, no warning. + { + const { shop, clock } = emptyShop(300); + let invoked = 0; + const sneaky = (): number => { + invoked++; + return 6000; + }; + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + // The cast is the point: this is what "I got past the type error" looks like. + retry: { attempts: 3, on: 200, backoff: sneaky } as unknown as { + attempts: number; + on: number; + backoff: 'fixed'; + }, + }); + const p = call.safe(); + await clock.advance(60_000); + await p; + check('(a2) the delay function was invoked', invoked, 0); + check('(a2) requests still made', shop.calls.length, 3); + note( + '(a2) gaps (ms) — the default jittered curve, not 6000', + gaps(shop).join(', '), + ); + } + + // ── (b) what the curve waits vs. what Shopify said to wait ──────────────────────────────── + // Cost 300, bucket 0, restore 50/s → the server's own arithmetic says 6000ms. `fixed`/100ms + // waits 100ms, so every retry throttles again. + { + const { shop, clock } = emptyShop(300); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 100 }, + }, + }); + const p = call.safe(); + await clock.advance(60_000); + const r = await p; + check('(b) attempts made', shop.calls.length, 3); + check('(b) ALL of them throttled', shop.throttledCount, 3); + check('(b) curve gap (ms)', gaps(shop).join(','), '100,100'); + const cost = costOfBody((r.error?.body ?? null) as unknown); + note('(b) error body carried extensions.cost', cost !== undefined); + checkNear( + '(b) the wait Shopify prescribed (ms)', + deficitWaitMs({ + requestedQueryCost: 300, + actualQueryCost: null, + throttleStatus: { + maximumAvailable: 1000, + currentlyAvailable: 0, + restoreRate: 50, + }, + }), + 6000, + ); + } + + // ── (c) a HAND-COMPUTED constant works — for exactly one query cost ──────────────────────── + // `fixed`/6000 succeeds on attempt 2 when the cost is 300. Change the cost to 900 and the same + // config throttles again: a constant cannot track a per-query deficit. + { + const { shop, clock } = emptyShop(300); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 6000, max: 60_000 }, + }, + }); + const p = call.safe(); + await clock.advance(60_000); + await p; + check( + '(c) cost 300 + fixed 6000 — throttled once', + shop.throttledCount, + 1, + ); + // …and then it KEEPS GOING. `on: 200` matches the successful response too, so the loop + // runs the full 3 attempts and the shop is charged twice for one logical result. + check('(c) requests made (attempts: 3)', shop.calls.length, 3); + check('(c) points charged for ONE result', shop.pointsCharged, 600); + } + { + const { shop, clock } = emptyShop(900); // a pricier query, same config + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 3, + on: 200, + backoff: { curve: 'fixed', base: 6000, max: 60_000 }, + }, + }); + const p = call.safe(); + await clock.advance(60_000); + const r = await p; + check( + '(c2) cost 900 + the SAME fixed 6000 — all throttled', + shop.throttledCount, + 3, + ); + check('(c2) call failed', r.ok, false); + note('(c2) correct wait would have been (ms)', ((900 - 0) / 50) * 1000); + } + + // ── (d) `retry.respect` — Shopify sends no Retry-After header, so it is inert ────────────── + { + const { shop, clock } = emptyShop(300); + const call = graphql({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + document: DOC, + adapter: shop.adapter(), + clock, + retry: { + attempts: 2, + on: 200, + respect: true, + backoff: { curve: 'fixed', base: 100 }, + }, + }); + const p = call.safe(); + await clock.advance(60_000); + await p; + // Probe the throttled response's headers on a SEPARATE shop, so this measurement does not + // pollute the arrival-time recording above. + const probe = emptyShop(300); + const throttledRes = await probe.shop.adapter()({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + method: 'POST', + headers: {}, + body: { operationName: 'BigSync' }, + }); + check( + '(d) Retry-After present on the throttled response', + Object.keys(throttledRes.headers).includes('retry-after'), + false, + ); + check( + '(d) respect:true still waited the curve (ms)', + gaps(shop).join(','), + '100', + ); + } + + // ── (e) THE ONE DOOR: a surface's `SurfaceOutcome.after` IS a computed wait ──────────────── + // `interpret` runs inside the attempt loop (engine.ts:775) and its retry arm honours + // `after` via `parseDuration(outcome.after)` (engine.ts:798). A surface author can therefore + // return the number the body dictates. This is the seam C6 builds on. + { + const { shop, clock } = emptyShop(300); + const waits: number[] = []; + // The `id` must be a LITERAL type, not widened to `string`: the config guards key off + // `kind: { id: '…' }`, and a widened `string` matches every one of them (see C6 (e2)). + const costSurface: Surface & { readonly id: 'shopify-cost' } = { + id: 'shopify-cost', + interpret: (res) => { + const cost = costOfBody(res.body); + if (cost && cost.actualQueryCost === null) { + const after = deficitWaitMs(cost); + waits.push(after); + return { + ok: false, + retry: true, + message: 'THROTTLED', + after, + }; + } + return { ok: true, data: res.body }; + }, + }; + const call = stitch({ + url: 'https://shop.myshopify.com/admin/api/graphql.json', + method: 'POST', + kind: costSurface, + adapter: shop.adapter(), + clock, + retry: { attempts: 3 }, + }); + const p = call.safe({ body: { operationName: 'BigSync' } }); + await clock.advance(60_000); + const r = await p; + check('(e) surface asked for a computed wait', waits.length, 1); + checkNear('(e) the wait it asked for (ms)', waits[0] ?? -1, 6000); + check( + '(e) the engine HONOURED it — gap (ms)', + gaps(shop).join(','), + '6000', + ); + check('(e) succeeded on attempt 2', r.ok, true); + check('(e) requests made', shop.calls.length, 2); + } + + finish( + 'C2', + '`backoff` has NO function form (type error; silently ignored if cast) and `retry.respect` reads a HEADER Shopify never sends — but `SurfaceOutcome.after` on a custom surface IS honoured as a computed wait', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c3-extensions-reachability.ts b/docs/scenarios/proofs/cost-based-rate-limits/c3-extensions-reachability.ts new file mode 100644 index 00000000..1543a47f --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c3-extensions-reachability.ts @@ -0,0 +1,236 @@ +// C3 — is `extensions.cost.throttleStatus` reachable AT ALL on a `graphql()` stitch? +// +// Two paths matter, because a cost ledger needs BOTH: the budget must be re-read from every +// response, successes included (the shop's bucket is shared, so only the server's number is true). +// (a) on SUCCESS, where the graphql helper pins `pick: 'data'` (stitch.ts:1263) +// (b) on the THROTTLED failure, where the surface rejects a 200 +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c3-extensions-reachability.ts +import { graphql } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { FakeShopify, costOfBody } from './fake-shopify'; +import { check, finish, heading, note } from './harness'; + +const DOC = 'query BigSync { products { id } }'; +const URL = 'https://shop.myshopify.com/admin/api/graphql.json'; + +async function main(): Promise { + heading('C3 — is extensions.cost reachable on a graphql() stitch?'); + + // ── (a) SUCCESS: the caller gets `data`, and `extensions` is gone ────────────────────────── + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + let hookBody: unknown; + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + hooks: { + onResponse: (ctx) => { + hookBody = ctx.res?.body; + }, + }, + }); + const r = await call.safe(); + check('(a) call succeeded', r.ok, true); + check( + '(a) caller sees `extensions`', + costOfBody(r.data) !== undefined, + false, + ); + note('(a) what the caller got', JSON.stringify(r.data)); + // The HOOK, however, sees the WHOLE body — `onResponse` is handed the live AdapterResponse + // (types.ts:1282, engine.ts:705), before `pick` runs (engine.ts:968). + const hookCost = costOfBody(hookBody); + check( + '(a) onResponse hook sees extensions.cost', + hookCost !== undefined, + true, + ); + check( + '(a) …currentlyAvailable it read', + hookCost?.throttleStatus.currentlyAvailable, + 700, + ); + check('(a) …actualQueryCost it read', hookCost?.actualQueryCost, 300); + } + + // ── (a2) `.inspect().raw` does NOT reach it — `raw` is pre-VALIDATION, not pre-PICK ──────── + // The natural second guess after the hook. `raw` is the body the drift findings are diffed + // against, which is taken AFTER `pick` has already unwrapped `data`. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + }); + const ins = await call.inspect(); + check( + '(a2) inspect().raw carries extensions.cost', + costOfBody(ins.raw) !== undefined, + false, + ); + note('(a2) what inspect().raw actually is', JSON.stringify(ins.raw)); + note('(a2) inspect().source', ins.source); + } + + // ── (b) THROTTLED: the thrown StitchError does NOT carry the body ────────────────────────── + // graphql's `interpret` rejects the 200 (surface.ts:310-323). A body verdict is returned, not + // thrown, so the engine builds the terminal event with `surfaceErrEvt` (engine.ts:1143-1158) — + // which sets only `message` / `status` / `attempts` and attaches NO `ERROR_SOURCE`. The + // `StitchError` rebuilt in stitch.ts:509-515 therefore has no `body`. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + }); + const r = await call.safe(); + check('(b) call failed', r.ok, false); + check('(b) error message', r.error?.message, 'GraphQL: Throttled'); + check( + '(b) StitchError.body is present', + r.error?.body !== undefined, + false, + ); + check( + '(b) → extensions.cost off the error', + costOfBody(r.error?.body), + undefined, + ); + // The status DOES survive (graphql's outcome carries `status: res.status`, + // surface.ts:321) — and it is `200`, which is the whole trap in one number. + check('(b) StitchError.status', r.error?.status, 200); + } + + // ── (b2) the EVENT STREAM on a throttle: no body either ──────────────────────────────────── + // The `error` event's own type (types.ts:1343-1354) has `message`/`status`/`retryAfter`/ + // `attempts` — there is no field a body could ride on. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + }); + const events: StitchEvent[] = []; + for await (const ev of call.stream()) events.push(ev); + const errEv = events.find((e) => e.type === 'error'); + check('(b2) an error event was emitted', errEv !== undefined, true); + note( + '(b2) error event keys', + Object.keys(errEv ?? {}) + .sort() + .join(','), + ); + check( + '(b2) any event carrying extensions.cost', + events.some( + (e) => costOfBody((e as { body?: unknown }).body) !== undefined, + ), + false, + ); + } + + // ── (b3) but a HOOK still sees it on the throttled response ──────────────────────────────── + // This is the seam that survives: `onResponse` fires for every attempt, throttled or not. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + let hookBody: unknown; + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + hooks: { + onResponse: (ctx) => { + hookBody = ctx.res?.body; + }, + }, + }); + await call.safe(); + const cost = costOfBody(hookBody); + check( + '(b3) hook saw extensions.cost on the THROTTLE', + cost !== undefined, + true, + ); + check('(b3) …requestedQueryCost', cost?.requestedQueryCost, 300); + check( + '(b3) …currentlyAvailable', + cost?.throttleStatus.currentlyAvailable, + 0, + ); + check('(b3) …restoreRate', cost?.throttleStatus.restoreRate, 50); + } + + // ── (b4) `.inspect()` on the throttle: `raw` is NULL ─────────────────────────────────────── + // `source` is `'live'` (a real request ran and a real body came back), yet `raw` is null: the + // surface rejected the response before the raw body was retained. So `.inspect()` is not a + // route to the throttled payload either. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + }); + const ins = await call.inspect(); + check('(b4) inspect().error set', ins.error !== null, true); + check('(b4) inspect().source', ins.source, 'live'); + check('(b4) inspect().raw', ins.raw, null); + } + + // ── (c) can `pick` be re-aimed at extensions? Yes — and it costs you `data` ──────────────── + // `graphql()` pins `pick: config.pick ?? 'data'` (stitch.ts:1263), so `pick` IS overridable. + // But `pick` is one path: aim it at the cost and the payload is gone. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + pick: 'extensions.cost', + }); + const r = await call.safe(); + check('(c) pick:"extensions.cost" — call ok', r.ok, true); + check( + '(c) caller now sees currentlyAvailable', + (r.data as { throttleStatus?: { currentlyAvailable?: number } }) + ?.throttleStatus?.currentlyAvailable, + 700, + ); + check( + '(c) …but the GraphQL `data` payload is gone', + (r.data as { ok?: boolean }).ok, + undefined, + ); + } + + finish( + 'C3', + 'extensions.cost is reachable on both paths through EXACTLY ONE seam — hooks.onResponse. The success RESULT is pick-stripped to data, .inspect().raw is post-pick (success) or null (throttle), and the StitchError and error EVENT carry no body at all', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c4-delegate-on-200.ts b/docs/scenarios/proofs/cost-based-rate-limits/c4-delegate-on-200.ts new file mode 100644 index 00000000..592ac57e --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c4-delegate-on-200.ts @@ -0,0 +1,208 @@ +// C4 — does `throttle: { delegate: true }` trip on a 200-with-THROTTLED, or is it status-keyed +// too? `throttle.rate`'s own doc comment (types.ts:1018-1019) points here — *"Where a real quota +// needs spending the way the vendor accounts for it, hand the backoff to an outer gate with +// `delegate`"* — so this is the DOCUMENTED escape hatch for exactly this scenario. +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c4-delegate-on-200.ts +import { RateLimitError, graphql } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeShopify, costOfBody } from './fake-shopify'; +import { check, finish, heading, note } from './harness'; + +const DOC = 'query BigSync { products { id } }'; +const URL = 'https://shop.myshopify.com/admin/api/graphql.json'; + +async function main(): Promise { + heading('C4 — does throttle.delegate trip on a 200-with-THROTTLED?'); + + // ── (a) the DEFAULT `on: [429]` — a 200 does not trip it ─────────────────────────────────── + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + throttle: { delegate: true }, + }); + const r = await call.safe(); + check('(a) call failed', r.ok, false); + check( + '(a) a RateLimitError surfaced', + r.error instanceof RateLimitError, + false, + ); + check( + '(a) what surfaced instead', + r.error?.message, + 'GraphQL: Throttled', + ); + note( + '(a) → the documented escape hatch does not reach a body-reported quota', + 'delegate is keyed on `rlMatch(res.status)` (engine.ts:731)', + ); + } + + // ── (b) `on: 200` DOES trip it — and the THROWING path carries the body ──────────────────── + // `RateLimitError` lifts `body`/`url` off the response at construction (resilience.ts:297-298) + // and keeps the whole `response`. So this IS a route to `extensions.cost` on the failure path. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + throttle: { delegate: true, on: 200 }, + }); + let thrown: unknown; + try { + await call(); + } catch (e) { + thrown = e; + } + check( + '(b) await → a real RateLimitError', + thrown instanceof RateLimitError, + true, + ); + const rle = thrown as RateLimitError; + check('(b) RateLimitError.status', rle.status, 200); + const cost = costOfBody(rle.body); + check( + '(b) RateLimitError.body carries extensions.cost', + cost !== undefined, + true, + ); + check('(b) …requestedQueryCost', cost?.requestedQueryCost, 300); + check( + '(b) …currentlyAvailable', + cost?.throttleStatus.currentlyAvailable, + 0, + ); + // No `Retry-After` header exists, so the structured hint the outer gate is meant to use + // is empty — the number it needs is in the BODY, which the gate must parse itself. + check('(b) RateLimitError.retryAfter', rle.retryAfter, undefined); + } + + // ── (b2) `.safe()` gives back the SAME error — no downgrade ─────────────────────────────── + // FIXED by #662 (issue #651, finding 2). This block used to measure the opposite: `.safe()` + // coerced every terminal through `asStitchError`, which copied `message` + `status` + `cause` + // and NOT `body`, so a RateLimitError arrived as a bare StitchError with `body: undefined` and + // the real instance one undocumented hop away on `.cause`. `RateLimitError` now extends + // `StitchError`, so `SafeResult.error` (typed `StitchError`) holds it without coercion and the + // pacing payload is on the error itself. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + throttle: { delegate: true, on: 200 }, + }); + const r = await call.safe(); + check( + '(b2) safe() → instanceof RateLimitError', + r.error instanceof RateLimitError, + true, + ); + check('(b2) safe() error class', r.error?.name, 'RateLimitError'); + check('(b2) safe() error.status survives', r.error?.status, 200); + // The pacing payload is on the error itself now — no `.cause` hop. + check( + '(b2) safe() error.body has extensions.cost', + costOfBody(r.error?.body)?.requestedQueryCost, + 300, + ); + check( + '(b2) …and it is the same error the throwing path raises', + r.error instanceof RateLimitError && r.error.response.status, + 200, + ); + note( + '(b2) → .safe() and try/catch now agree', + 'RateLimitError extends StitchError (#662), so no coercion happens', + ); + } + + // ── (c) …but `on: 200` fires on SUCCESSFUL responses too ─────────────────────────────────── + // `on` is a `StatusMatch` (types.ts:1048), so it cannot see `errors[]`. A perfectly good + // query is reported to the host as a rate limit, and its `data` is thrown away. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); // FULL bucket + const call = graphql({ + url: URL, + document: DOC, + adapter: shop.adapter(), + clock, + throttle: { delegate: true, on: 200 }, + }); + let thrown: unknown; + try { + await call(); + } catch (e) { + thrown = e; + } + check('(c) the shop accepted the query', shop.throttledCount, 0); + check( + '(c) …and the caller still got a RateLimitError', + thrown instanceof RateLimitError, + true, + ); + check( + '(c) the successful data was discarded', + (thrown as RateLimitError).body !== undefined && + costOfBody((thrown as RateLimitError).body)?.actualQueryCost, + 300, // a perfectly good, fully-paid-for response, reported as a rate limit + ); + note( + '(c) the "rate limit" it reported', + JSON.stringify((thrown as RateLimitError | undefined)?.body), + ); + } + + // ── (d) delegate also DISABLES self-pacing, as documented ────────────────────────────────── + // `delegate` skips the acquire entirely (engine.ts:626-628), so a `rate` set alongside it is + // inert. You cannot keep in-process pacing AND delegate the backoff. + { + const clock = manualClock(); + const shop = new FakeShopify({ + clock, + costs: { Cheap: 1 }, + defaultCost: 1, + }); + const call = graphql({ + url: URL, + document: 'query Cheap { shop { id } }', + adapter: shop.adapter(), + clock, + throttle: { delegate: true, rate: '1/s' }, + }); + const p = Promise.all([call.safe(), call.safe(), call.safe()]); + await clock.advance(0); + await p; + const times = shop.calls.map((c) => c.at); + check('(d) 3 calls at rate 1/s — requests made', shop.calls.length, 3); + check( + '(d) all dispatched at the same instant', + times.join(','), + '0,0,0', + ); + note('(d) → `rate` is inert under `delegate`', 'engine.ts:626-628'); + } + + finish( + 'C4', + 'delegate is status-keyed (`on?: StatusMatch`, default [429]) so a 200-THROTTLED never trips it; forcing `on: 200` DOES surface a RateLimitError carrying extensions.cost, but it fires on every successful 200 too and discards the data', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c5-throttle-rate-cost.ts b/docs/scenarios/proofs/cost-based-rate-limits/c5-throttle-rate-cost.ts new file mode 100644 index 00000000..f718cd25 --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c5-throttle-rate-cost.ts @@ -0,0 +1,171 @@ +// C5 — can `throttle.rate` express a COST budget (1000 points, refilling 50/s) rather than a +// request rate? Its own doc comment says it is *"a minimum spacing between successive calls … +// not a token bucket"* (types.ts:1006-1019). This measures what that costs you in throughput +// when you try to approximate a bucket with a spacing. +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c5-throttle-rate-cost.ts +import { graphql } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { ThrottleOptions } from '../../../../packages/core/src/types'; +import { FakeShopify } from './fake-shopify'; +import { check, finish, heading, note } from './harness'; + +const URL = 'https://shop.myshopify.com/admin/api/graphql.json'; + +/** Fire `n` calls concurrently, advance virtual time, and return their arrival times (ms). */ +async function fire( + shop: FakeShopify, + clock: ReturnType, + n: number, + throttle: string | ThrottleOptions, + document: string, +): Promise { + const call = graphql({ + url: URL, + document, + adapter: shop.adapter(), + clock, + throttle: throttle as string, + }); + const p = Promise.all(Array.from({ length: n }, () => call.safe())); + await clock.advance(600_000); + await p; + return shop.calls.map((c) => c.at); +} + +async function main(): Promise { + heading('C5 — can throttle.rate express a cost budget?'); + + // ── (a) the unit is REQUESTS, and it is a string grammar with no room for points ─────────── + { + // @ts-expect-error — a bare number is rejected outright (types.ts:1021-1025). + const asNumber: ThrottleOptions = { rate: 1000 }; + void asNumber; + // `rate` is a plain `string`, so a points-denominated token TYPECHECKS — and then throws + // at construction, because `parseRate` fails loud rather than falling back to "no limit" + // (types.ts:1026-1030). There is no cost form; the grammar is `/`. + const asPoints: ThrottleOptions = { rate: '1000points/50s' }; + let threw: string | undefined; + try { + graphql({ + url: URL, + document: 'query Q { a }', + adapter: new FakeShopify({ clock: manualClock() }).adapter(), + throttle: asPoints as { rate: string }, + }); + } catch (e) { + threw = (e as Error).message; + } + check( + '(a) a points-denominated rate THROWS at construction', + threw !== undefined, + true, + ); + note('(a) the error', threw); + } + + // ── (b) it reads only the RATIO — window length is not a burst allowance ─────────────────── + // A bucket's defining feature is that it can spend 1000 points AT ONCE. `'10/10s'` looks like + // it should permit 10 immediately; it does not — it is a 1000ms spacing, same as `'1/s'`. + { + const clockA = manualClock(); + const shopA = new FakeShopify({ clock: clockA, defaultCost: 100 }); + const a = await fire(shopA, clockA, 4, '10/10s', 'query Q { a }'); + + const clockB = manualClock(); + const shopB = new FakeShopify({ clock: clockB, defaultCost: 100 }); + const b = await fire(shopB, clockB, 4, '1/s', 'query Q { a }'); + + check('(b) "10/10s" arrival times', a.join(','), '0,1000,2000,3000'); + check('(b) "1/s" arrival times', b.join(','), a.join(',')); + note( + '(b) → a 10-call burst the bucket would have allowed is spread over 3s', + '', + ); + } + + // ── (c) the throughput a spacing costs you ──────────────────────────────────────────────── + // Cost 100/query, bucket 1000, restore 50/s. The BUCKET permits 10 queries at t=0. The + // safest spacing that never outruns the refill is 100 points / 50 per sec = one call per 2s. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, defaultCost: 100 }); + const times = await fire(shop, clock, 10, '1/2s', 'query Q { a }'); + check('(c) 10 calls, none throttled', shop.throttledCount, 0); + check( + '(c) last arrival under `1/2s` (ms)', + times[times.length - 1], + 18_000, + ); + + // The same 10 calls with NO throttle: the bucket absorbs them instantly. + const clock2 = manualClock(); + const shop2 = new FakeShopify({ clock: clock2, defaultCost: 100 }); + const call2 = graphql({ + url: URL, + document: 'query Q { a }', + adapter: shop2.adapter(), + clock: clock2, + }); + const p2 = Promise.all(Array.from({ length: 10 }, () => call2.safe())); + await clock2.advance(0); + await p2; + check( + '(c) …the bucket alone would have taken all 10 at once', + shop2.throttledCount, + 0, + ); + check('(c) last arrival with NO throttle (ms)', shop2.calls[9]?.at, 0); + note( + '(c) cost of approximating a bucket with a spacing', + '18s vs 0s for the same work', + ); + } + + // ── (d) with MIXED costs there is no single correct spacing ─────────────────────────────── + // Size it for the expensive query (900 pts → one per 18s) and cheap queries crawl. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, defaultCost: 11 }); // cheap queries + const times = await fire(shop, clock, 5, '1/18s', 'query Cheap { a }'); + check( + '(d) 5 cheap (11-pt) calls sized for the 900-pt query — last arrival (ms)', + times[4], + 72_000, + ); + check('(d) points actually spent', shop.pointsCharged, 55); + note('(d) → 55 points spread over 72s; the bucket holds 1000', ''); + } + + // ── (d2) size it for the AVERAGE cost instead, and the expensive queries throttle ────────── + // The mixed workload averages (11 + 900) / 2 ≈ 455 points, so the "fair" spacing is + // 455 / 50 ≈ one call per 9s. A run of 900-point queries at that spacing spends 100 pts/s + // against a 50 pts/s refill: the bucket drains and the throttle the pacing existed to + // prevent happens anyway. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, defaultCost: 900 }); + const times = await fire(shop, clock, 4, '1/9s', 'query Big { a }'); + // t=0 accept (1000→100) · t=9s refill 450 → 550 < 900 THROTTLE · t=18s refill to the + // 1000 cap → accept · t=27s → 550 THROTTLE. Half the calls fail, on a limiter that was + // paced precisely to prevent that. + check( + '(d2) 4 expensive calls paced at the AVERAGE — throttled', + shop.throttledCount, + 2, + ); + check('(d2) …got through', shop.calls.length - shop.throttledCount, 2); + note('(d2) arrival times (ms)', times.join(',')); + note( + '(d2) → pacing for the average under-waits; pacing for the worst case is (d)', + 'no single spacing is correct for both', + ); + } + + finish( + 'C5', + '`throttle.rate` cannot express a cost budget: the unit is requests, the grammar is a string with no points form, and it is a minimum SPACING with no burst — approximating the 1000-point bucket costs 18s for work the bucket takes instantly, and with mixed costs no single spacing is correct', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/c6-assembled-solution.ts b/docs/scenarios/proofs/cost-based-rate-limits/c6-assembled-solution.ts new file mode 100644 index 00000000..3a0404a1 --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/c6-assembled-solution.ts @@ -0,0 +1,365 @@ +// C6 — THE DECIDING CLAIM. If C1–C5 fall short, can a user assemble CORRECT behaviour from the +// PUBLIC surface? This runs the assembled solution (`shopify-cost-surface.ts`) against the fake +// shop and checks all four requirements: +// (a) detect THROTTLED on a 200 +// (b) wait the COMPUTED deficit, not a curve +// (c) keep a running cost budget read from EVERY response, successes included +// (d) survive the shared bucket — a third-party app draining points mid-run +// +// pnpm exec tsx docs/scenarios/proofs/cost-based-rate-limits/c6-assembled-solution.ts +import { graphql, stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Adapter, StitchEvent } from '../../../../packages/core/src/types'; +import { + FakeShopify, + costOfBody, + deficitWaitMs, + isThrottled, +} from './fake-shopify'; +import { check, checkNear, finish, heading, note } from './harness'; +import { + CostLedger, + costGate, + shopifyCostSurface, +} from './shopify-cost-surface'; + +const DOC = 'query BigSync { products { id } }'; +const URL = 'https://shop.myshopify.com/admin/api/graphql.json'; + +async function main(): Promise { + heading('C6 — can correct behaviour be assembled from the public surface?'); + + // ── (a)+(b) detect on a 200, wait the computed deficit ──────────────────────────────────── + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); // empty bucket → the first attempt throttles + const ledger = new CostLedger(); + const call = stitch({ + url: URL, + kind: shopifyCostSurface(ledger), + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 4 }, + pick: 'data', + }); + const p = call.safe(); + await clock.advance(60_000); + const r = await p; + check('(a) the throttle was detected on a 200', shop.throttledCount, 1); + check('(a) the call ultimately SUCCEEDED', r.ok, true); + check( + '(a) caller got the graphql `data`', + (r.data as { ok?: boolean }).ok, + true, + ); + check('(b) waits requested', ledger.waits.length, 1); + checkNear( + '(b) the wait it asked for (ms)', + ledger.waits[0] ?? -1, + 6000, + ); + const gap = (shop.calls[1]?.at ?? 0) - (shop.calls[0]?.at ?? 0); + check('(b) the wait the engine actually took (ms)', gap, 6000); + note( + '(b) a curve would have waited', + '100ms (expo base), then 200ms — see C2', + ); + } + + // ── (c) the budget is read off EVERY response, successes included ───────────────────────── + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const ledger = new CostLedger(); + const call = stitch({ + url: URL, + kind: shopifyCostSurface(ledger), + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 4 }, + pick: 'data', + }); + for (let i = 0; i < 3; i++) { + const p = call.safe(); + await clock.advance(0); + await p; + } + check('(c) responses observed by the ledger', ledger.observed, 3); + check('(c) none of them were throttles', shop.throttledCount, 0); + check( + '(c) points spent, summed from actualQueryCost', + ledger.spent, + 900, + ); + check('(c) available, as the SHOP reported it', ledger.available, 100); + check('(c) …and that matches the shop', shop.currentlyAvailable(), 100); + check('(c) restoreRate learned from the wire', ledger.restoreRate, 50); + } + + // ── (d) the SHARED bucket: a third-party app drains points between our calls ────────────── + // The ledger's own arithmetic says there is headroom; the shop disagrees. Only the server's + // number is true, which is why the reactive half is not optional. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const ledger = new CostLedger(); + const call = stitch({ + url: URL, + kind: shopifyCostSurface(ledger), + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 6 }, + pick: 'data', + }); + // One clean call, so the ledger believes it has 700 points. + let p = call.safe(); + await clock.advance(0); + await p; + check('(d) ledger believes it has', ledger.available, 700); + + // A third-party inventory app now takes the shop's whole remaining bucket. + shop.drain(700); + check('(d) …but the shop actually has', shop.currentlyAvailable(), 0); + + p = call.safe(); + await clock.advance(60_000); + const r = await p; + check('(d) the call still SUCCEEDED', r.ok, true); + check('(d) it took one throttle to learn that', shop.throttledCount, 1); + checkNear( + '(d) and waited the deficit it was told (ms)', + ledger.waits[0] ?? -1, + 6000, + ); + // The ledger holds the SERVER's number, not its own arithmetic. Local bookkeeping would + // have said 700 − 300 = 400; the shop reported 0, and 0 is what the ledger carries. + check( + "(d) ledger holds the shop's number, not its own", + ledger.available, + 0, + ); + note('(d) what naive local bookkeeping would have believed', 700 - 300); + } + + // ── (d2) a sustained run with a hostile neighbour: everything completes ─────────────────── + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const ledger = new CostLedger(); + const call = stitch({ + url: URL, + kind: shopifyCostSurface(ledger), + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 8 }, + pick: 'data', + }); + let ok = 0; + for (let i = 0; i < 8; i++) { + shop.drain(1000); // a hostile neighbour empties the shop before every call + const pending = call.safe(); + await clock.advance(60_000); + if ((await pending).ok) ok++; + } + check( + '(d2) 8 queries against a shop emptied before each — succeeded', + ok, + 8, + ); + check( + '(d2) every one of them hit a throttle first', + shop.throttledCount, + 8, + ); + check('(d2) …and none reached the caller as an error', ok, 8); + check('(d2) points spent', ledger.spent, 2400); + check( + '(d2) every wait was the computed deficit', + ledger.waits.length, + 8, + ); + checkNear('(d2) each wait (ms)', ledger.waits[7] ?? -1, 6000); + } + + // ── (e) why the surface's `id` is 'graphql' ────────────────────────────────────────────── + // `document` is gated on `kind: { id: 'graphql' }` at the TYPE level (types.ts:581-585). A + // surface with its own id cannot use the config key its `buildRequest` needs. + { + const named: Surface & { readonly id: 'shopify' } = { + id: 'shopify', + interpret: (res) => ({ ok: true, data: res.body }), + }; + const clock = manualClock(); + const shop = new FakeShopify({ clock }); + stitch({ + url: URL, + kind: named, + adapter: shop.adapter(), + clock, + // @ts-expect-error — `document` requires the graphql surface; this id is not it. + document: DOC, + }); + check('(e) a custom `id` makes `document` a TYPE ERROR', true, true); + } + + // ── (e2) a surface typed as bare `Surface` cannot use `document` either ─────────────────── + // `Surface['id']` is `string`, and `string` does not satisfy the literal `'graphql'` the guard + // keys off. So the surface must be typed `Surface & { readonly id: 'graphql' }` — declaring + // `id: 'graphql'` in the object literal is not enough if the annotation widens it. + { + const widened: Surface = { + id: 'graphql', + interpret: (res) => ({ ok: true, data: res.body }), + }; + const clock = manualClock(); + const shop = new FakeShopify({ clock }); + stitch({ + url: URL, + kind: widened, + adapter: shop.adapter(), + clock, + // @ts-expect-error — the annotation widened `id` to `string`; the guard needs the literal. + document: DOC, + }); + check( + '(e2) a WIDENED `Surface` id also rejects `document`', + true, + true, + ); + } + + // ── (f) SEAM COMPARISON: the same behaviour written as a wrapping ADAPTER ───────────────── + // The other obvious seam. It works, but the retry loop it runs is its OWN: `retry.attempts`, + // `timeout.total`, the trace's attempt counter and the `progress` event stream all sit + // OUTSIDE it, so from the engine's point of view one very slow request happened. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + shop.drain(1000); + const ledger = new CostLedger(); + const inner = shop.adapter(); + const costAdapter: Adapter = async (req) => { + for (;;) { + const res = await inner(req); + const cost = costOfBody(res.body); + if (cost) ledger.record(cost); + if (!isThrottled(res.body) || !cost) return res; + await clock.sleep(deficitWaitMs(cost)); + } + }; + const call = graphql({ + url: URL, + document: DOC, + adapter: costAdapter, + clock, + retry: { attempts: 3 }, + }); + const p = call.safe(); + await clock.advance(60_000); + const r = await p; + check('(f) adapter seam — call succeeded', r.ok, true); + check('(f) the shop saw the retry', shop.calls.length, 2); + // The engine counted ONE attempt: the whole wait happened below its retry loop. + const events: StitchEvent[] = []; + const clock2 = manualClock(); + const shop2 = new FakeShopify({ + clock: clock2, + costs: { BigSync: 300 }, + }); + shop2.drain(1000); + const inner2 = shop2.adapter(); + const call2 = graphql({ + url: URL, + document: DOC, + clock: clock2, + retry: { attempts: 3 }, + adapter: async (req) => { + for (;;) { + const res = await inner2(req); + const cost = costOfBody(res.body); + if (!isThrottled(res.body) || !cost) return res; + await clock2.sleep(deficitWaitMs(cost)); + } + }, + }); + const gen = call2.stream(); + const drainP = (async () => { + for await (const ev of gen) events.push(ev); + })(); + await clock2.advance(60_000); + await drainP; + const retryEvents = events.filter( + (e) => e.type === 'progress' && e.phase === 'retry', + ); + check( + '(f) `retry` progress events the engine emitted', + retryEvents.length, + 0, + ); + check( + '(f) attempts the engine reported', + events.find((e) => e.type === 'done')?.attempts, + 1, + ); + note( + '(f) → the wait is invisible to trace, timeout.total and the event spine', + '', + ); + } + + // ── (g) the PROACTIVE half: `hooks.onRequest` may await, so it can gate the request ─────── + // The reactive path is correct but pays one wasted round-trip per throttle (8 of them in d2). + // `Hooks.onRequest` returns `void | Promise` (types.ts:1286), so awaiting inside it + // holds the request until the ledger says the query is affordable. + { + const clock = manualClock(); + const shop = new FakeShopify({ clock, costs: { BigSync: 300 } }); + const ledger = new CostLedger(); + const call = stitch({ + url: URL, + kind: shopifyCostSurface(ledger), + document: DOC, + adapter: shop.adapter(), + clock, + retry: { attempts: 6 }, + pick: 'data', + hooks: { + onRequest: costGate(ledger, 300, (ms) => clock.sleep(ms)), + }, + }); + // Prime the ledger, then let the neighbour empty the shop. + let p = call.safe(); + await clock.advance(60_000); + await p; + shop.drain(1000); + ledger.available = 0; // what the next response would have told us anyway + + p = call.safe(); + await clock.advance(60_000); + const r = await p; + check('(g) proactive gate — call succeeded', r.ok, true); + check( + '(g) …with NO wasted throttled round-trip', + shop.throttledCount, + 0, + ); + check('(g) requests sent in total', shop.calls.length, 2); + note( + '(g) → the reactive half still covers what the gate cannot predict (d)', + '', + ); + } + + finish( + 'C6', + "YES — a custom `Surface` closes all four requirements against the public API: `interpret` sees every body (detect + ledger), and `SurfaceOutcome.after` carries the computed deficit into the engine's own retry loop", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/cost-based-rate-limits/fake-shopify.ts b/docs/scenarios/proofs/cost-based-rate-limits/fake-shopify.ts new file mode 100644 index 00000000..f5b7f311 --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/fake-shopify.ts @@ -0,0 +1,237 @@ +// A fake, in-memory Shopify GraphQL Admin API that models the COST BUCKET honestly: +// +// - a leaky bucket of `maximumAvailable` points (1000) refilling at `restoreRate` points/sec, +// - a per-operation cost, so "requests per second" is not a meaningful unit, +// - over-spending answers **HTTP 200** with `errors: [{ extensions: { code: 'THROTTLED' } }]`, +// - `extensions.cost` rides EVERY response — success and throttle alike, +// - `drain()` simulates a third-party app spending the SHOP's shared bucket. +// +// Refill is driven by an INJECTED {@link Clock}, so every proof runs on `manualClock()` virtual +// time: the numbers below are exact, not timing-dependent. Nothing touches the network. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +/** Shopify's `extensions.cost.throttleStatus`, spelled exactly as the vendor spells it. */ +export interface ThrottleStatus { + maximumAvailable: number; + currentlyAvailable: number; + restoreRate: number; +} + +/** Shopify's `extensions.cost` envelope. `actualQueryCost` is null on a throttled response. */ +export interface CostExtension { + requestedQueryCost: number; + actualQueryCost: number | null; + throttleStatus: ThrottleStatus; +} + +/** One recorded hit, as the provider saw it. */ +export interface RecordedCall { + operationName: string; + requestedQueryCost: number; + /** Points in the shop's bucket at the moment the request was priced (after refill). */ + availableBefore: number; + throttled: boolean; + /** Virtual time (ms) the request arrived. */ + at: number; +} + +export interface ShopifyOptions { + clock: Clock; + /** Bucket size. Shopify's standard plan: 1000. */ + maximumAvailable?: number; + /** Points restored per second. Shopify's standard plan: 50. */ + restoreRate?: number; + /** Cost per operation name. Unlisted operations cost `defaultCost`. */ + costs?: Record; + /** Cost for an operation not named in `costs`. Default 10. */ + defaultCost?: number; + /** + * Shopify REFUNDS the difference between the requested and the actual cost. When set, the + * response reports `actualQueryCost = requestedQueryCost * actualRatio` and only that is debited. + */ + actualRatio?: number; +} + +export class FakeShopify { + /** Every hit, in order. `calls.length` IS the request count. */ + readonly calls: RecordedCall[] = []; + /** How many responses were 200-with-THROTTLED. */ + throttledCount = 0; + + readonly maximumAvailable: number; + readonly restoreRate: number; + + private available: number; + private lastRefillAt: number; + private readonly clock: Clock; + private readonly costs: Record; + private readonly defaultCost: number; + private readonly actualRatio: number; + + constructor(opts: ShopifyOptions) { + this.clock = opts.clock; + this.maximumAvailable = opts.maximumAvailable ?? 1000; + this.restoreRate = opts.restoreRate ?? 50; + this.costs = opts.costs ?? {}; + this.defaultCost = opts.defaultCost ?? 10; + this.actualRatio = opts.actualRatio ?? 1; + this.available = this.maximumAvailable; + this.lastRefillAt = opts.clock.now(); + } + + /** Points currently in the shop's bucket, refilled to *now*. */ + currentlyAvailable(): number { + this.refill(); + return this.available; + } + + /** + * Total points the shop was charged across every accepted call. Unlike `currentlyAvailable()` + * this does not move with refill, so it is the honest measure of what a retry policy SPENT. + */ + get pointsCharged(): number { + return this.calls + .filter((c) => !c.throttled) + .reduce( + (sum, c) => sum + c.requestedQueryCost * this.actualRatio, + 0, + ); + } + + /** + * Another app on the same shop spends `points`. This is the shared-bucket case: our client's + * own bookkeeping cannot predict it, so the budget must be re-read from every response. + */ + drain(points: number): void { + this.refill(); + this.available = Math.max(0, this.available - points); + } + + /** The cost the provider will price an operation at. */ + costOf(operationName: string): number { + return this.costs[operationName] ?? this.defaultCost; + } + + private refill(): void { + const nowMs = this.clock.now(); + const elapsedSec = (nowMs - this.lastRefillAt) / 1000; + this.lastRefillAt = nowMs; + if (elapsedSec <= 0) return; + this.available = Math.min( + this.maximumAvailable, + this.available + elapsedSec * this.restoreRate, + ); + } + + private throttleStatus(): ThrottleStatus { + return { + maximumAvailable: this.maximumAvailable, + currentlyAvailable: this.available, + restoreRate: this.restoreRate, + }; + } + + /** + * The GraphQL Admin endpoint. The graphql surface sends `{ query, variables, operationName }`, + * so the operation name is what prices the call. + */ + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const body = (req.body ?? {}) as { + operationName?: string; + variables?: Record; + }; + const operationName = body.operationName ?? 'anonymous'; + const requestedQueryCost = this.costOf(operationName); + + this.refill(); + const availableBefore = this.available; + const at = this.clock.now(); + + // THE TRAP: over-spending is a 200, not a 429. Nothing is debited, and `extensions.cost` + // still rides the response — it is what tells a client how long to wait. + if (requestedQueryCost > this.available) { + this.throttledCount++; + this.calls.push({ + operationName, + requestedQueryCost, + availableBefore, + throttled: true, + at, + }); + return { + status: 200, + headers: {}, + body: { + errors: [ + { + message: 'Throttled', + extensions: { code: 'THROTTLED' }, + }, + ], + extensions: { + cost: { + requestedQueryCost, + actualQueryCost: null, + throttleStatus: this.throttleStatus(), + } satisfies CostExtension, + }, + }, + }; + } + + const actualQueryCost = requestedQueryCost * this.actualRatio; + this.available -= actualQueryCost; + this.calls.push({ + operationName, + requestedQueryCost, + availableBefore, + throttled: false, + at, + }); + return { + status: 200, + headers: {}, + body: { + data: { ok: true, operationName }, + extensions: { + cost: { + requestedQueryCost, + actualQueryCost, + throttleStatus: this.throttleStatus(), + } satisfies CostExtension, + }, + }, + }; + }; + } +} + +/** The wait Shopify's own arithmetic prescribes, in ms: `(requested - available) / restoreRate`. */ +export function deficitWaitMs(cost: CostExtension): number { + const { requestedQueryCost, throttleStatus } = cost; + const deficit = requestedQueryCost - throttleStatus.currentlyAvailable; + if (deficit <= 0) return 0; + return (deficit / throttleStatus.restoreRate) * 1000; +} + +/** Read `extensions.cost` off any response body, or `undefined` when it is not there. */ +export function costOfBody(body: unknown): CostExtension | undefined { + return ( + body as { extensions?: { cost?: CostExtension } } | null | undefined + )?.extensions?.cost; +} + +/** Is this body a 200-with-THROTTLED? */ +export function isThrottled(body: unknown): boolean { + const errs = ( + body as + { errors?: { extensions?: { code?: string } }[] } | null | undefined + )?.errors; + return !!errs?.some((e) => e.extensions?.code === 'THROTTLED'); +} diff --git a/docs/scenarios/proofs/cost-based-rate-limits/harness.ts b/docs/scenarios/proofs/cost-based-rate-limits/harness.ts new file mode 100644 index 00000000..8415a1b1 --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/harness.ts @@ -0,0 +1,57 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED number either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured number is within `tolerance` of `expected`. The cost arithmetic here is + * floating point (points refill continuously), so an exact `Object.is` on a computed wait would + * assert the float, not the claim. + */ +export function checkNear( + label: string, + actual: number, + expected: number, + tolerance = 1e-6, +): void { + checks++; + const ok = Math.abs(actual - expected) <= tolerance; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${actual}${ok ? '' : ` (expected ${expected} ±${tolerance})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/cost-based-rate-limits/shopify-cost-surface.ts b/docs/scenarios/proofs/cost-based-rate-limits/shopify-cost-surface.ts new file mode 100644 index 00000000..89a2fe1d --- /dev/null +++ b/docs/scenarios/proofs/cost-based-rate-limits/shopify-cost-surface.ts @@ -0,0 +1,117 @@ +// USER CODE for C6 — the cost-aware Shopify surface, written entirely against StitchAPI's public +// API (`Surface`, `SurfaceOutcome`, `verdictOf`, `graphqlSurface`, all exported from the barrel). +// +// It closes all four requirements the scenario sets: +// (a) detects THROTTLED on an HTTP 200 → `interpret` reads `errors[].extensions.code` +// (b) waits the COMPUTED deficit, not a curve → `SurfaceOutcome.after` (engine.ts:798) +// (c) reads the budget off EVERY response → `interpret` runs on successes too +// (d) survives the SHARED bucket → the ledger is overwritten by the server's +// number on every response, never inferred +// +// The `id` is deliberately `'graphql'`: the `document` / `operationName` config keys are gated on +// `kind: { id: 'graphql' }` at the type level (types.ts:581-585), so a surface with any other id +// cannot use them. See `c6-assembled-solution.ts` (e) for the measurement of that. +import { graphqlSurface, verdictOf } from '../../../../packages/core/src/index'; +import type { + Surface, + SurfaceOutcome, +} from '../../../../packages/core/src/index'; +import type { Hooks } from '../../../../packages/core/src/types'; +import type { CostExtension } from './fake-shopify'; +import { costOfBody, deficitWaitMs, isThrottled } from './fake-shopify'; + +/** + * The running cost budget. Every value here comes from the SERVER — the shop's bucket is shared + * with other apps, so a locally-inferred balance is always a guess. + */ +export class CostLedger { + /** Last `currentlyAvailable` the shop reported, or `undefined` before the first response. */ + available: number | undefined; + restoreRate = 50; + maximumAvailable = 1000; + /** Points spent, summed from `actualQueryCost` on successful responses. */ + spent = 0; + /** Every computed wait the surface asked the engine for (ms) — measured by the proofs. */ + readonly waits: number[] = []; + /** Responses seen, throttled or not. */ + observed = 0; + + record(cost: CostExtension): void { + this.observed++; + this.available = cost.throttleStatus.currentlyAvailable; + this.restoreRate = cost.throttleStatus.restoreRate; + this.maximumAvailable = cost.throttleStatus.maximumAvailable; + if (cost.actualQueryCost !== null) this.spent += cost.actualQueryCost; + } + + /** How long to wait before a query costing `cost` points is affordable (ms). */ + waitFor(cost: number): number { + if (this.available === undefined) return 0; + const deficit = cost - this.available; + return deficit <= 0 ? 0 : (deficit / this.restoreRate) * 1000; + } +} + +/** A cost-aware GraphQL surface. Reactive half of the solution: detect, compute, re-attempt. */ +export function shopifyCostSurface( + ledger: CostLedger, +): Surface & { readonly id: 'graphql' } { + // `Surface.buildRequest` is optional, so under `exactOptionalPropertyTypes` it cannot be + // assigned straight across — pin the graphql one, which is always defined. + const buildRequest: NonNullable = ( + cfg, + input, + base, + ) => graphqlSurface.buildRequest?.(cfg, input, base) ?? base; + return { + id: 'graphql', + buildRequest, + interpret: (res, cfg): SurfaceOutcome => { + // Keep the declarative verdict in front of the body rules (surface.ts:166-167). + const failure = verdictOf(res, cfg); + if (failure) return failure; + + // (c) EVERY response updates the budget — successes carry `throttleStatus` too. + const cost = costOfBody(res.body); + if (cost) ledger.record(cost); + + // (a) the 200-with-THROTTLED, and (b) the wait the server's own arithmetic dictates. + if (isThrottled(res.body) && cost) { + const after = deficitWaitMs(cost); + ledger.waits.push(after); + return { + ok: false, + retry: true, + message: `THROTTLED — need ${cost.requestedQueryCost}, have ${cost.throttleStatus.currentlyAvailable}`, + after, + }; + } + + // Any OTHER GraphQL error stays a plain failure — defer to the built-in surface. + // (`interpret` is optional on `Surface`; the graphql surface always defines it.) + return ( + graphqlSurface.interpret?.(res, cfg) ?? { + ok: true, + data: res.body, + } + ); + }, + }; +} + +/** + * Proactive half: an `onRequest` hook that pauses until the ledger says the next query is + * affordable. `Hooks.onRequest` may return a promise (types.ts:1286), so awaiting inside it gates + * the request. This is what stops the client from spending into a deficit it can already predict — + * the reactive half above still handles the part it cannot (another app draining the shop). + */ +export function costGate( + ledger: CostLedger, + queryCost: number, + sleep: (ms: number) => Promise, +): NonNullable { + return async (): Promise => { + const wait = ledger.waitFor(queryCost); + if (wait > 0) await sleep(wait); + }; +} diff --git a/docs/scenarios/proofs/deprecation-headers/README.md b/docs/scenarios/proofs/deprecation-headers/README.md new file mode 100644 index 00000000..a4414f92 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/README.md @@ -0,0 +1,177 @@ +# Proofs — the vendor told you for six months, in a header + +Runnable evidence for the claims in [`../../deprecation-headers.md`](../../deprecation-headers.md). + +**C1 was the deciding claim and it refutes three earlier findings.** Response headers **are** +reachable on a successful awaited call. They are reachable in exactly three places — the `adapter`, +`hooks.onResponse` (`ctx.res.headers`), and a `Surface`'s `interpret(res, cfg)` (`res.headers`) — +and only the last of those can also decide what the call returns. Everything a caller normally +reaches for carries nothing: `await`, `.unwrap()`, `.safe()`, `.inspect()`, `.report()`, +`StitchError`, `transform`, and the entire event spine (4 events, **15 distinct keys between them, +not one a header**), which is why a `TraceSink` inherits the same hole. + +[Scenario 6](../../conditional-requests-304.md), [scenario 7](../../multipart-upload.md) and +[scenario 15](../../unconfirmed-write.md) each concluded "no headers here" from an accessor that +genuinely has none, and each generalised one step too far. The `ETag` was never going to be in +`Inspection`; it was always in `interpret`. **The definitive table is +[below](#the-accessor-table--does-this-carry-a-response-header).** + +The rest goes the library's way more often than not, and none of it is configuration: + +- **C4 (aggregation) works and the scenario-12 shape transfers.** One `TraceSink` at the seam, 500 + calls over 5 endpoints, reported `3 endpoints deprecated (users, search, orders), earliest sunset +in 12 days: users` — one row per endpoint, and **identical when traffic was skewed 200:1 toward + the healthy endpoints**, which is exactly what a per-call log line cannot do. +- **C5 (the tripwire) is exact.** `[sunset-1ms, sunset, sunset+1ms]` measured `["ok", "FAILED", +"FAILED"]` on an injected clock. It does **not** burn retry attempts (5 configured, 1 request + made) and does **not** open the circuit breaker (5 consecutive trips past `failures: 2`). +- **C3 (findings) works through one narrow door.** A folded notice becomes + `info|undeclared|_deprecation` on the ordinary drift channel, non-fatal, re-levellable to `warn`. + +And three things go against it: + +- **The header does not reach the sink on its own** (C4 c). No event carries headers, so a + `TraceSink` can only aggregate what a `Surface` already folded into the value — and an ordinary + `output` contract that does not declare `_deprecation` **deletes it again, silently** (C4 d). +- **`hooks.onResponse` is a write channel the docs say does not exist** (C2). Mutating `ctx.res` + changes the result: `res.body` added a key to the caller's value, `res.status` turned a vendor + `200` into a thrown `HTTP 503`, `res.headers` made the surface read a lie. +- **The line count goes against the library** (C8): **132 executable lines to a hand-rolled + control's 81**, for identical output. The first scenario in this pass where that happens. + +Every script is standalone and offline. Each prints one `PASS`/`FAIL` line and exits non-zero on +failure. **178 checks across 8 scripts.** + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c1-accessors.ts + +# all of them +for f in docs/scenarios/proofs/deprecation-headers/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. The suite takes about four seconds: every +sunset crossing is on a `manualClock`, and nothing here does real I/O. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/deprecation-headers/*.ts +``` + +## The accessor table — does this carry a response header? + +**This is the consolidation deliverable.** Measured on a **successful** (`200`) awaited call whose +response carried `Deprecation: @1735689600` and `Sunset: Thu, 01 Jan 2026 00:00:00 GMT`. Reproduce +the whole table with `pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c1-accessors.ts`. + +| Accessor | Headers? | What it carries instead | Can it act? | +| ----------------------- | -------- | ----------------------------------------------------------------------- | -------------------------------------------- | +| `await` / `.unwrap()` | **no** | the body, and only the body | — | +| `.safe()` | **no** | `{ ok, data, error }` — 3 keys | — | +| `.inspect()` | **no** | `{ data, raw, findings, status, error, source }` — `status`, no headers | — | +| `.inspect().raw` | **no** | the pre-validation **body** | — | +| `.report()` | **no** | the above + `{ attempts, timing, config, cache }` — 9 keys | — | +| `.report().config` | **no** | the **request** config (its `headers` are the ones you _sent_) | — | +| `.stream()` | **no** | 4 events, **15 distinct keys**, none a header | — | +| `TraceSink.handle` | **no** | the same events; `ctx` is `{ name, spanId, traceId }` | — | +| `StitchError` | **no** | `{ attempts, body, name, status, url }` — 5 keys (scenario 15) | — | +| `transform(body)` | **no** | one argument, and it is the body | — | +| `pick` / `output` | **no** | operate on the value, downstream of the body | — | +| **`adapter`** | **YES** | it _built_ the response — but knows no stitch `name` | no | +| **`hooks.onResponse`** | **YES** | `ctx.res.headers` (full `AdapterResponse`) + `ctx.name` | **observe** (mutation works — see footgun 1) | +| **`Surface.interpret`** | **YES** | `res.headers` + `cfg.name` + `cfg.clock` | **decides the value, and can fail the call** | + +Read the two positive rows together: `onResponse` is the **observation** seat and `interpret` is the +**decision** seat. `interpret` is the only place in the library where a response header and the +value the caller receives are in scope at the same time — every answer in this directory is built on +that one fact. + +## What each script establishes + +| Script | Question | Measured | +| ------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `c1-accessors.ts` | **DECIDING** — is a response header reachable, and where? | **YES, in 3 places.** 11 accessors carry nothing; `adapter`/`onResponse`/`interpret` do. Event spine: 15 keys, 0 headers | +| `c2-hooks.ts` | does `onResponse` see them, and can it act? | **Sees them. And MUTATES the result** — `res.body`, `res.status` (200→thrown 503), `res.headers`. Docs say it cannot | +| `c3-findings.ts` | can a header become a levelled, non-fatal finding? | **Yes** — `info\|undeclared\|_deprecation`, re-levellable to `warn`. No value, no endpoint, and re-levelling is per-KIND | +| `c4-aggregation.ts` | **DECIDING** — can a `TraceSink` produce the fleet view? | **Yes: 500 calls → `3 endpoints deprecated …, earliest sunset in 12 days: users`.** But the header never reaches it alone | +| `c5-tripwire.ts` | fail after a chosen sunset, not before? | **Exact to the ms.** `["ok","FAILED","FAILED"]`. No retries burned, no circuit opened, `status: 200` preserved | +| `c6-formats.ts` | any parsing help for sf-date / HTTP-date? | **None reachable.** `parseRetryAfter` does the job exactly and is behind none of the 17 subpaths. **29 lines hand-rolled** | +| `c7-noise.ts` | one line per call? de-dupable without hand-rolled state? | **360 lines / 3 facts.** `loggerSink` default is 2400. `levelOf` drops to 600 and **cannot** reach 3. A `Set` can | +| `c8-assembled.ts` | the whole job, against a hand-rolled control | **132 lines vs 81** — the library LOSES. Identical report, identical tripwire, 3 seams, **0 config keys** | + +## Files + +- `fake-vendor.ts` — five endpoints, three retiring, **every response a `200`**. The retirement + notices exist only in headers, and three spellings are served on purpose: `Deprecation: +@1735689600` (RFC 9745 sf-date), `Deprecation: Sat, 01 Mar 2025 …` (the pre-RFC draft spelling, + still emitted by real vendors), and `Sunset:` as an HTTP-date (RFC 8594, always). `NOW` is frozen + at 2025-12-20, which puts `users` **exactly 12 days** from its sunset. +- `harness.ts` — `check` / `checkSeq` / **`checkReach`** / `checkAtMost` / `note` / `heading` / + `finish`. `checkReach` is the assertion this scenario exists for: it prints the accessor, whether + the header was REACHED, and the value — the C1 table is literally its output. +- `deprecation.ts` — the user-code module. Both parsers (bracketed by `BEGIN`/`END PARSERS`, which + is what C6 counts), the `deprecationSurface` factory (`fold` / `onNotice` / `failAfterSunset`), + and the `DeprecationWatch` `TraceSink`. +- `assembled.ts` / `hand-rolled.ts` — the answer and the control, both over the same `Adapter` and + the same `Clock`, both delimited by `BEGIN`/`END USER CODE` so the line count is of code someone + maintains. The control imports the same parsers, so the 29 parser lines cancel. + +## Reading the numbers honestly + +- **C1 is the headline and it should be read as a correction, not a win.** The library does not + _surface_ headers — it exposes two seams that happen to have the response in scope. `Inspection` + and `StitchError` still carry no headers, exactly as scenarios 6, 7 and 15 measured. What changes + is the conclusion drawn from that: an `ETag` or a replay marker **is** recoverable, from + `interpret`, at the cost of writing a `Surface`. +- **C4's success has a coupling attached and it is not visible in the type system.** The sink reads + the notice off the `result` event's `data`; the data only has it because the surface folded it; + and an `output` contract that does not declare `_deprecation` strips it back out. Three + independently reasonable decisions, and the fleet report goes silently empty when they disagree. + The `onNotice` side channel avoids all of it and gives up the trace channel in exchange. +- **The C8 line count is a genuine loss, and the reason is structural.** The control reads the + header inline in the method that already had the response. StitchAPI needs a `Surface` object to + reach it and a `TraceSink` object to remember it — two indirections for a job that is, at heart, + four lines. What the extra lines buy is that the same seam already carries `retry`, `throttle`, + `cache`, `circuit`, `auth`, `timeout` and the trace tree as config keys. +- **Zero config keys know about this problem**, and that is the correct reading of C8 — not a + complaint. RFC 9745 says deprecation is a hint, so a default behaviour would be wrong; what a + library owes here is a seam, and the seam exists. + +## Footguns + +1. **`hooks.onResponse` can rewrite the call, and the documentation says it cannot.** The hooks + guide states hooks "never change what a stitch returns — a call's only result is its response". + Measured, that is true only of the hook's **return value**. `ctx.res` is the engine's live + response object (engine.ts:705) and the same object reaches `interpret` seventy lines later + (engine.ts:775), so mutating it works: `res.body` changed the caller's value, `res.status = 503` + turned a vendor `200` into a thrown error, and `res.headers` made a surface read `"REWRITTEN BY +HOOK"`. A hook and a surface reading the same header will disagree, and the hook wins. +2. **Seam-level `kind` is a compile error that works perfectly at runtime.** + `seam({ baseUrl, kind })` fails with `TS2353: 'kind' does not exist in type 'SeamOptions'`, yet + the engine composes it into every member — measured, members inherited `http+deprecation` and + folded correctly. A typed codebase therefore writes the surface on all 40 members for a + capability that already works from one. +3. **`ctx.name` defaults to the literal `"stitch"`.** `name` falls back to `path` or `'stitch'`, and + a `url`-configured stitch has no `path` — so **two different unnamed endpoints both arrive at the + sink as `stitch`** and a `Map` keyed on `ctx.name` silently merges them into one row. The URL is + only on the `start` event; recovering it costs a second `Map` keyed by `ctx.spanId` and a join. +4. **A cache hit re-serves a header captured once.** 10 calls, 1 wire request, and the sink counted + 10 — nine of them replaying a `Sunset` read once. With a long TTL a sunset that has already + passed keeps reporting as "in 12 days" until the entry expires. +5. **`Date.parse('@1735689600')` is `NaN`.** The obvious one-liner reads `Sunset` correctly and + reports **no deprecation at all** for the format RFC 9745 mandates — silently, because `NaN` is + falsy and every naive guard treats it as absent. +6. **`.report()` and `.inspect()` are fresh runs** that each add a request _and_ a tick to anything + the sink is counting (the same trap scenario 12 measured for drift rates). +7. **Re-levelling a header-derived finding is per-KIND, not per-path.** `severity: { undeclared: +'warn' }` raised the deprecation notice to `warn` and raised an unrelated new vendor field with + it, in the same run. diff --git a/docs/scenarios/proofs/deprecation-headers/assembled.ts b/docs/scenarios/proofs/deprecation-headers/assembled.ts new file mode 100644 index 00000000..6a2ee75e --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/assembled.ts @@ -0,0 +1,64 @@ +// The best available StitchAPI answer, as a caller would write it — the thing C8 counts and runs. +// +// Three decisions, and NONE of them is configuration. That is the finding: every other scenario in +// this pass had at least one lever in the config object, and this one has zero. The library's +// contribution here is that the seams EXIST and compose, not that any of them knows what a +// `Deprecation` header is. +// +// • READING IT — `kind: deprecationSurface(...)`. `interpret(res, cfg)` is the only hook that +// sees `res.headers` AND decides the result (C1). Written once, and — because seam-level `kind` +// is a compile error the runtime would otherwise honour (C4 f) — repeated on every member. +// • REPORTING IT — `trace: watch`, a `TraceSink` at the SEAM, so one sink covers every member +// and `ctx.name` says which endpoint (C4). This half genuinely is configured once. +// • ENFORCING IT — `failAfterSunset` on the surface, reading the stitch's injected `clock`. It +// does not burn retries and does not open the circuit (C5). +// +// The coupling to watch: the sink reads the notice off the `result` event's `data`, which only +// carries it because the surface folded it there — so `fold: true` and the sink are one unit, and +// an `output` contract that does not declare `_deprecation` silently breaks it (C4 d). +import { seam } from '../../../../packages/core/src/index'; +import type { + Adapter, + Clock, + Stitch, +} from '../../../../packages/core/src/types'; +import { DeprecationWatch, deprecationSurface } from './deprecation'; + +export interface WatchedApiOptions { + baseUrl: string; + adapter: Adapter; + clock: Clock; + /** Endpoints to mint, `name` → `path`. */ + endpoints: readonly { name: string; path: string }[]; + /** Turn an announced sunset into a hard failure once it has passed. */ + failAfterSunset: boolean; +} + +export interface WatchedApi { + members: Map; + watch: DeprecationWatch; +} + +// >>> BEGIN USER CODE +/** A seam whose every member reads its own retirement notice, and one sink that reports the fleet. */ +export function watchedApi(opts: WatchedApiOptions): WatchedApi { + const watch = new DeprecationWatch(opts.clock); + const api = seam({ + baseUrl: opts.baseUrl, + adapter: opts.adapter, + clock: opts.clock, + trace: watch, + }); + const surface = deprecationSurface({ + fold: true, + failAfterSunset: opts.failAfterSunset, + }); + const members = new Map(); + for (const e of opts.endpoints) + members.set( + e.name, + api.stitch({ name: e.name, path: e.path, kind: surface }), + ); + return { members, watch }; +} +// <<< END USER CODE diff --git a/docs/scenarios/proofs/deprecation-headers/c1-accessors.ts b/docs/scenarios/proofs/deprecation-headers/c1-accessors.ts new file mode 100644 index 00000000..73229598 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c1-accessors.ts @@ -0,0 +1,350 @@ +// C1 — THE CONSOLIDATION. Enumerate every accessor on a SUCCESSFUL awaited call and say precisely +// which carry a response header. +// +// Three earlier scenarios hit this wall from three directions — scenario 6 (an ETag off +// `Inspection`), scenario 7 (a part's ETag), scenario 15 (a replay marker off `StitchError`) — and +// each filed it as "no headers here". This script asks the question once, everywhere, so the answer +// stops being three separate absences and becomes one table with a positive result in it. +// +// The result, measured below: response headers are reachable in exactly THREE places, and only one +// of them can put what it found into the value the caller receives. +// +// RESULT ACCESSORS +// await / .unwrap() ABSENT the body, and only the body +// .safe() ABSENT { ok, data, error } — same body, plus a null error +// .inspect() ABSENT { data, raw, findings, status, error, source } — `status`, no headers +// .report() ABSENT the above + { attempts, timing, config, cache } +// StitchError ABSENT (scenario 15's finding, re-measured on the failure path) +// THE EVENT SPINE +// .stream() ABSENT 4 events on a clean run, 20 keys between them, none a header +// TraceSink ABSENT same events; `ctx` is { name, spanId, traceId, parentSpanId } +// PIPELINE HOOKS +// transform(body) ABSENT `(body: unknown) => unknown` — the signature has nowhere to put them +// pick / output ABSENT both operate on the value, downstream of the body +// WHERE THEY ACTUALLY ARE +// adapter REACHED it MADE the response — but it cannot name the endpoint or the run +// hooks.onResponse REACHED ctx.res.headers, every attempt <- observe +// Surface.interpret REACHED res.headers, and it decides the value <- act +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c1-accessors.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import type { + Adapter, + AdapterResponse, + HookContext, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { endpoint, headersFor, serving } from './fake-vendor'; +import { check, checkReach, checkSeq, finish, heading, note } from './harness'; + +const USERS = endpoint('users'); +/** The header value every accessor below is hunted for: RFC 8594, an HTTP-date, on a 200. */ +const SUNSET = 'Thu, 01 Jan 2026 00:00:00 GMT'; + +/** Pull `sunset` out of whatever an accessor handed back, without assuming a shape. */ +function sunsetIn(v: unknown): unknown { + if (typeof v !== 'object' || v === null) return undefined; + const rec = v as Record; + if (typeof rec['sunset'] === 'string') return rec['sunset']; + const headers = rec['headers']; + if (typeof headers === 'object' && headers !== null) + return (headers as Record)['sunset']; + return undefined; +} + +async function main(): Promise { + heading( + 'C1 — which accessor carries a response header on a SUCCESSFUL call', + ); + + // Sanity: the vendor really does put it on the wire. Everything below measures REACHABILITY, + // which is only a finding if the header was there to be reached. + check('(0) the vendor sends `Sunset`', headersFor(USERS)['sunset'], SUNSET); + check( + '(0) …and `Deprecation`, as an RFC 9745 sf-date', + headersFor(USERS)['deprecation'], + '@1735689600', + ); + check('(0) …on a', 200, 200); + + // ── RESULT ACCESSORS ───────────────────────────────────────────────────────────────────── + heading(' result accessors'); + { + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + }); + + const awaited = await call(); + checkReach('await', sunsetIn(awaited), false); + checkSeq( + '(a) …what it did carry', + Object.keys(awaited as object).sort(), + ['users'], + ); + + const unwrapped = await call.unwrap(); + checkReach('.unwrap()', sunsetIn(unwrapped), false); + + const safe = await call.safe(); + checkReach('.safe()', sunsetIn(safe.data), false); + checkSeq('(b) `SafeResult` keys', Object.keys(safe).sort(), [ + 'data', + 'error', + 'ok', + ]); + check('(b) ok', safe.ok, true); + check('(b) error', safe.error, null); + + const ins = await call.inspect(); + checkReach('.inspect()', sunsetIn(ins), false); + checkReach('.inspect().raw', sunsetIn(ins.raw), false); + checkSeq('(c) `Inspection` keys', Object.keys(ins).sort(), [ + 'data', + 'error', + 'findings', + 'source', + 'status', + ]); + check('(c) is `headers` on it?', 'headers' in ins, false); + check('(c) the one wire fact it DOES carry', ins.status, 200); + + const rep = await call.report(); + checkReach('.report()', sunsetIn(rep), false); + checkSeq('(d) `RunReport` keys', Object.keys(rep).sort(), [ + 'attempts', + 'cache', + 'config', + 'data', + 'error', + 'findings', + 'source', + 'status', + 'timing', + ]); + check('(d) is `headers` on it?', 'headers' in rep, false); + checkReach('.report().config', sunsetIn(rep.config), false); + note( + '(d) → `config` is the REQUEST config (`headers` there would be the ones you SENT). Nothing on a report describes the response beyond its status', + ); + } + + // ── THE EVENT SPINE ────────────────────────────────────────────────────────────────────── + heading(' the event spine'); + { + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + }); + const spine: string[] = []; + const everyKey = new Set(); + let reached: unknown; + for await (const e of call.stream()) { + spine.push(e.type); + for (const k of Object.keys(e)) everyKey.add(k); + const hit = sunsetIn(e); + if (hit !== undefined) reached = hit; + if (e.type === 'result' && sunsetIn(e.data) !== undefined) + reached = sunsetIn(e.data); + } + checkReach('.stream()', reached, false); + checkSeq('(e) event spine on a clean 200', spine, [ + 'start', + 'progress', + 'result', + 'done', + ]); + checkSeq('(e) EVERY key across EVERY event', [...everyKey].sort(), [ + 'at', + 'attempt', + 'attempts', + 'data', + 'elapsed', + 'input', + 'method', + 'name', + 'ok', + 'phase', + 'spanId', + 'status', + 'traceId', + 'type', + 'url', + ]); + note( + '(e) → 15 distinct keys across the whole spine and not one of them is a header. `status` is on `result`, `url` on `start`', + ); + } + { + const seen: unknown[] = []; + const ctxKeys = new Set(); + const sink: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + for (const k of Object.keys(ctx)) ctxKeys.add(k); + const hit = sunsetIn(e); + if (hit !== undefined) seen.push(hit); + }, + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + trace: sink, + })(); + checkReach('TraceSink', seen[0], false); + checkSeq('(f) `TraceContext` keys', [...ctxKeys].sort(), [ + 'name', + 'spanId', + 'traceId', + ]); + note( + '(f) → the sink sees the same events, so it inherits the same hole. `ctx` identifies the RUN, never the response', + ); + } + + // ── PIPELINE HOOKS ─────────────────────────────────────────────────────────────────────── + heading(' pipeline hooks'); + { + let transformSaw: unknown; + // Measured off the function the engine actually calls: how many arguments does it receive? + let transformArgc = -1; + const transform = function (...args: unknown[]): unknown { + transformArgc = args.length; + transformSaw = sunsetIn(args[0]); + return args[0]; + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + transform, + })(); + checkReach('transform(body)', transformSaw, false); + check('(g) arguments the engine hands `transform`', transformArgc, 1); + note( + '(g) → `transform?: (body: unknown) => unknown` (types.ts:1505). One parameter, and it is the body. There is no seat for a response', + ); + } + + // ── WHERE THE HEADERS ACTUALLY ARE ─────────────────────────────────────────────────────── + heading(' where they actually are'); + { + // The adapter MADE the response, so of course it holds the headers. Listed because it is a + // real seam a caller can wrap — and because what it CANNOT do is the point. + let adapterSaw: unknown; + let adapterKnewName = 'no'; + const base = serving(USERS); + const wrapped: Adapter = async (req): Promise => { + const res = await base(req); + adapterSaw = res.headers['sunset']; + adapterKnewName = 'name' in req ? 'yes' : 'no'; + return res; + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: wrapped, + })(); + checkReach('adapter', adapterSaw, true); + check( + '(h) does the adapter know the stitch NAME?', + adapterKnewName, + 'no', + ); + note( + '(h) → it has the response and the URL, and no idea which stitch it is serving or which run it belongs to. Fine for a global log, useless for a fleet report keyed by endpoint', + ); + } + { + let hookSaw: unknown; + let hookCtxKeys: string[] = []; + let hookName = ''; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + hooks: { + onResponse: (c: HookContext) => { + hookCtxKeys = Object.keys(c).sort(); + hookName = c.name; + hookSaw = c.res?.headers['sunset']; + }, + }, + })(); + checkReach('hooks.onResponse', hookSaw, true); + checkSeq('(i) `HookContext` keys on a 200', hookCtxKeys, [ + 'attempt', + 'name', + 'res', + ]); + check('(i) and it knows the endpoint', hookName, 'users'); + note( + '(i) → THE POSITIVE RESULT. `ctx.res` is the full `AdapterResponse` — `{ status, headers, body, url? }` — and `ctx.name` says which stitch. Scenarios 6, 7 and 15 each concluded "no headers" from an accessor that genuinely has none; this seam was never asked', + ); + } + { + let surfaceSaw: unknown; + let surfaceName = ''; + const probe: Surface = { + id: 'probe', + interpret: (res, cfg) => { + surfaceSaw = res.headers['sunset']; + surfaceName = cfg.name ?? ''; + return { ok: true, data: res.body }; + }, + }; + const value = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: probe, + })(); + checkReach('Surface.interpret', surfaceSaw, true); + check('(j) and it knows the endpoint', surfaceName, 'users'); + check( + '(j) …and it CHOSE that value', + JSON.stringify(value), + JSON.stringify(USERS.body), + ); + note( + '(j) → the other positive result, and the only one that is also a WRITE. `interpret(res, cfg)` gets the whole response AND returns what the call resolves to, so it is the one place a header can be turned into part of the answer', + ); + } + + // ── the same question on the FAILURE path, for the scenario-15 cross-reference ──────────── + heading(' the failure path (scenario 15 re-measured)'); + { + const r = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: async (): Promise => ({ + status: 503, + headers: headersFor(USERS), + body: { error: 'unavailable' }, + }), + retry: { attempts: 1 }, + }).safe(); + check('(k) `ok` on the 503', r.ok, false); + checkReach('StitchError', sunsetIn(r.error), false); + checkSeq( + '(k) `StitchError` own keys', + Object.keys(r.error ?? {}).sort(), + ['attempts', 'body', 'name', 'status', 'url'], + ); + note( + '(k) → confirms scenario 15 exactly: `StitchError` has `status`, `body` and `url`, and no `headers`. The vendor sent the notice on this response too and it died with the error', + ); + } + + finish( + 'C1', + "DEFINITIVE, AND IT IS NOT THE ABSENCE THREE EARLIER SCENARIOS RECORDED. Response headers ARE reachable on a successful call, in exactly THREE places: the `adapter` (it built the response, but knows no stitch name and cannot change the result), `hooks.onResponse` (`ctx.res.headers` — the whole `AdapterResponse`, plus `ctx.name`), and a Surface's `interpret(res, cfg)` (`res.headers`, plus it RETURNS the value the call resolves to). Everything a caller normally reaches for carries nothing: `await`, `.unwrap()`, `.safe()`, `.inspect()` (5 keys, `status` but no headers), `.report()` (9 keys, same), `StitchError` (5 keys — scenario 15 re-confirmed), `transform` (one parameter, and it is the body), and the ENTIRE event spine — 4 events, 15 distinct keys between them, not one a header, which is why a `TraceSink` inherits the same hole. So the three earlier findings were each correct about their own accessor and each generalised one step too far: the header was never in `Inspection` or `StitchError`, and it was always in `interpret` and `onResponse`", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c2-hooks.ts b/docs/scenarios/proofs/deprecation-headers/c2-hooks.ts new file mode 100644 index 00000000..7ffd1388 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c2-hooks.ts @@ -0,0 +1,303 @@ +// C2 — `hooks.onResponse` on a 200 carrying the headers: does it see them, and can it do anything +// beyond observe? +// +// It sees them (C1 established that). The interesting half is the second question, and the answer +// contradicts the documentation. The hooks guide says hooks "never change what a stitch returns — +// a call's only result is its response", and the RETURN VALUE of the hook is indeed ignored. But +// `ctx.res` is the engine's live response object, handed over by reference at engine.ts:705, and +// the same object is passed to the surface's `interpret` seventy lines later (engine.ts:775). So an +// `onResponse` that MUTATES `res` rewrites the call: +// +// mutate res.body -> the caller receives the mutated body +// mutate res.status -> a 200 becomes a 503 and the call THROWS +// mutate res.headers -> the surface reads the rewritten header +// +// That is a write channel with no type-level warning on it, and it is the wrong one to build this +// scenario on: it works by accident of ordering rather than by contract. C3 and C4 use the surface. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c2-hooks.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import type { + AdapterResponse, + HookContext, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { parseSunset, readNotice } from './deprecation'; +import { endpoint, headersFor, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const USERS = endpoint('users'); +const CLEAN = endpoint('payments'); + +async function main(): Promise { + heading('C2 — `hooks.onResponse` on a 200: observe, or act?'); + + // ── (a) it sees BOTH headers, in both formats, on a plain 200 ──────────────────────────── + { + let notice: ReturnType = null; + let status = -1; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + hooks: { + onResponse: (c: HookContext) => { + status = c.res?.status ?? -1; + notice = readNotice(c.res?.headers ?? {}); + }, + }, + })(); + check('(a) status the hook saw', status, 200); + check( + '(a) `Deprecation` parsed (RFC 9745 sf-date)', + notice === null + ? null + : (notice as { deprecatedAt: number | null }).deprecatedAt, + Date.parse('2025-01-01T00:00:00Z'), + ); + check( + '(a) `Sunset` parsed (RFC 8594 HTTP-date)', + notice === null + ? null + : (notice as { sunsetAt: number | null }).sunsetAt, + Date.parse('2026-01-01T00:00:00Z'), + ); + note( + '(a) → everything this scenario needs is in scope here, on the success path, with the endpoint name alongside it', + ); + } + + // ── (b) a clean endpoint gives the hook nothing, which is the correct silence ──────────── + { + let notice: unknown = 'unset'; + await stitch({ + name: 'payments', + url: 'https://api.vendor.test/v1/payments', + adapter: serving(CLEAN), + hooks: { + onResponse: (c: HookContext) => { + notice = readNotice(c.res?.headers ?? {}); + }, + }, + })(); + check('(b) notice on a clean endpoint', notice, null); + } + + // ── (c) the RETURN VALUE is ignored — the documented read-only half, confirmed ─────────── + { + const value = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + hooks: { + // Returning a replacement result: the type is `void | Promise`, so this is + // already a compile error in real code. Cast through so the RUNTIME behaviour is + // measured rather than assumed. + onResponse: (() => ({ replaced: true })) as unknown as ( + ctx: HookContext, + ) => void, + }, + })(); + checkSeq( + '(c) keys of the value the caller got', + Object.keys(value as object).sort(), + ['users'], + ); + note( + '(c) → the returned object is dropped on the floor. This is the half the docs describe, and it is accurate', + ); + } + + // ── (d) MUTATING `ctx.res.body` DOES change the result. Undocumented ───────────────────── + { + const value = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + hooks: { + onResponse: (c: HookContext) => { + const body = c.res?.body; + if (typeof body === 'object' && body !== null) + (body as Record)['sunsetAt'] = + parseSunset(c.res?.headers['sunset']); + }, + }, + })(); + checkSeq( + '(d) keys of the value the caller got', + Object.keys(value as object).sort(), + ['sunsetAt', 'users'], + ); + check( + '(d) the injected value', + (value as Record)['sunsetAt'], + Date.parse('2026-01-01T00:00:00Z'), + ); + note( + '(d) → the hooks guide says hooks "never change what a stitch returns". Measured, they can: `ctx.res` is the live object (engine.ts:705) and `interpret` reads the SAME object afterwards (engine.ts:775). Only the RETURN is ignored', + ); + } + + // ── (e) mutating `ctx.res.status` turns a 200 into a thrown error ──────────────────────── + { + const r = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + retry: { attempts: 1 }, + hooks: { + onResponse: (c: HookContext) => { + if (c.res) c.res.status = 503; + }, + }, + }).safe(); + check('(e) ok', r.ok, false); + check('(e) error message', r.error?.message, 'HTTP 503'); + check('(e) status on the error', r.error?.status, 503); + note( + '(e) → the vendor sent a 200. A hook rewrote it and the call failed. That is a tripwire, and it is a tripwire built on an undocumented alias', + ); + } + + // ── (f) a header mutated in the hook is what the SURFACE reads ─────────────────────────── + { + let surfaceSaw: string | undefined; + const probe: Surface = { + id: 'probe', + interpret: (res) => { + surfaceSaw = res.headers['sunset']; + return { ok: true, data: res.body }; + }, + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: probe, + hooks: { + onResponse: (c: HookContext) => { + if (c.res) c.res.headers['sunset'] = 'REWRITTEN BY HOOK'; + }, + }, + })(); + check('(f) what the surface read', surfaceSaw, 'REWRITTEN BY HOOK'); + check( + '(f) what the vendor actually sent', + headersFor(USERS)['sunset'], + 'Thu, 01 Jan 2026 00:00:00 GMT', + ); + note( + '(f) → hook-before-surface is the fixed order, so a hook can lie to the surface. Worth knowing before putting policy in one and diagnosis in the other', + ); + } + + // ── (g) it fires once PER ATTEMPT, not once per call ───────────────────────────────────── + { + let fired = 0; + let n = 0; + const r = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: async (): Promise => { + n += 1; + return { + status: n < 3 ? 503 : 200, + headers: headersFor(USERS), + body: n < 3 ? { error: 'flaky' } : USERS.body, + }; + }, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + hooks: { onResponse: () => void (fired += 1) }, + }).safe(); + check('(g) call succeeded', r.ok, true); + check('(g) requests made', n, 3); + check('(g) `onResponse` firings', fired, 3); + note( + '(g) → 3 firings for 1 call. A naive counter in this hook counts ATTEMPTS, and every retried response carried the notice too', + ); + } + + // ── (h) it cannot reach the event stream or produce a finding ──────────────────────────── + { + const events: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + events.push(e.type); + }, + }; + // The announcement channels a hook would need, hunted for on the REAL context object the + // engine passes — not on a hand-written stand-in. + let announceable: string[] = []; + let ctxKeys: string[] = []; + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + trace: sink, + hooks: { + onResponse: (c: HookContext) => { + ctxKeys = Object.keys(c).sort(); + announceable = [ + 'emit', + 'run', + 'findings', + 'spanId', + 'trace', + ].filter((k) => k in c); + }, + }, + }); + await call(); + checkSeq('(h) `HookContext` keys, measured', ctxKeys, [ + 'attempt', + 'name', + 'res', + ]); + checkSeq('(h) …of which, channels to announce on', announceable, []); + checkSeq('(h) event spine, unchanged by the hook', events, [ + 'start', + 'progress', + 'result', + 'done', + ]); + note( + '(h) → `AuthContext` has an `emit` for exactly this (types.ts:1232); `HookContext` has `{ name, attempt, req?, res?, error? }` and no way to say anything. What the hook learns stays in the hook unless the hook closes over something', + ); + } + + // ── (i) so the only honest way out of a hook is a closure ──────────────────────────────── + { + const collected: string[] = []; + const api = (name: string, path: string, ep: typeof USERS) => + stitch({ + name, + url: `https://api.vendor.test${path}`, + adapter: serving(ep), + hooks: { + onResponse: (c: HookContext) => { + const notice = readNotice(c.res?.headers ?? {}); + if (notice !== null) collected.push(c.name); + }, + }, + }); + await api('users', '/v1/users', USERS)(); + await api('payments', '/v1/payments', CLEAN)(); + await api('users', '/v1/users', USERS)(); + checkSeq('(i) endpoints a closure collected', collected, [ + 'users', + 'users', + ]); + note( + '(i) → it works, and it is per-call with no de-duplication and no ordering. Everything C4 wants has to be built on top of this by hand', + ); + } + + finish( + 'C2', + 'IT SEES THEM, AND IT CAN DO FAR MORE THAN OBSERVE — which is the problem. `ctx.res` on a 200 is the full `AdapterResponse`, so both headers parse out of `ctx.res.headers` with `ctx.name` alongside. The hook\'s RETURN value is ignored, exactly as the docs say. But `ctx.res` is the engine\'s live object, handed over at engine.ts:705 and read again by `interpret` at engine.ts:775, so MUTATION is a real write channel: mutating `res.body` added a key to the value the caller received, mutating `res.status` turned the vendor\'s 200 into a thrown `HTTP 503`, and mutating `res.headers` made the surface read `"REWRITTEN BY HOOK"` instead of the real `Sunset`. The hooks guide says hooks "never change what a stitch returns"; measured, that is true only of the return value. Three further limits make it the wrong seat for this scenario anyway: it fires once PER ATTEMPT (3 firings for 1 retried call, each carrying the notice), `HookContext` has no `emit`/`run`/`findings` so nothing it learns can reach the event stream or the drift report, and the only way out is a closure with no de-duplication of its own', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c3-findings.ts b/docs/scenarios/proofs/deprecation-headers/c3-findings.ts new file mode 100644 index 00000000..9c04c80b --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c3-findings.ts @@ -0,0 +1,333 @@ +// C3 — can a header-derived warning become a FINDING in the same channel as drift: levelled, +// non-fatal, naming the endpoint? Or does it need a parallel mechanism? +// +// The answer is YES, through one narrow door, and the door costs something. A surface can fold the +// notice into the value; an `output` contract that does not declare it then reports it as an +// `undeclared` soft-drift finding, on the ordinary drift channel, non-fatal, at `info` — and +// `severity: { undeclared: 'warn' }` re-levels it. So it reports where everything else reports. +// +// Three things it is NOT: +// • the finding does not carry the header VALUE — `detail` is `undeclared field (object)`, so it +// says "there is a notice here", never "the sunset is 1 Jan" +// • the finding does not name the ENDPOINT — `DriftFinding` is `{ level, path, change, detail, +// sample }`; the endpoint is `ctx.name`, and only a sink has that +// • re-levelling is per-KIND, not per-path, so raising the notice to `warn` raises every other +// undeclared field with it +// +// And there is no API at all for minting a finding directly: `drift()` takes a schema, a `Validator` +// returns `{ ok, value }` or `{ ok, issues }`, and issues are HARD (fatal) findings. A levelled +// non-fatal finding of your own design is not expressible. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c3-findings.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { Validator } from '../../../../packages/core/src/index'; +import type { + DriftFinding, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { deprecationSurface, readNotice } from './deprecation'; +import { endpoint, headersFor, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const USERS = endpoint('users'); +const CLEAN = endpoint('payments'); + +/** One finding, flattened to a line — level, kind, path, detail. */ +const fmt = (f: DriftFinding): string => + `${f.level}|${f.change}|${f.path}|${f.detail ?? ''}`; + +/** A permissive `output` that declares only `users` — everything else is undeclared drift. */ +const declaresUsersOnly: Validator<{ users: unknown }> = { + validate: async (v: unknown) => ({ + ok: true as const, + value: { users: (v as Record)['users'] }, + }), +}; + +async function main(): Promise { + heading('C3 — can a header-derived warning join the drift channel?'); + + // ── (a) it can. Fold in the surface, and the contract reports it as drift ──────────────── + { + const sinkFindings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type === 'drift') + sinkFindings.push(`${ctx.name} ${fmt(e.finding)}`); + }, + }; + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: deprecationSurface({ fold: true }), + output: drift(declaresUsersOnly), + trace: sink, + }); + const r = await call.safe(); + check('(a) the call still succeeded', r.ok, true); + check('(a) …and did not throw', r.error, null); + checkSeq('(a) the finding, at the sink', sinkFindings, [ + 'users info|undeclared|_deprecation|undeclared field (object)', + ]); + note( + '(a) → a real drift finding, on the ordinary channel, from a RESPONSE HEADER. Non-fatal: `ok: true`, `error: null`, and the value flowed', + ); + } + + // ── (b) a clean endpoint produces no finding — the silence has to be real ──────────────── + { + const found: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') found.push(fmt(e.finding)); + }, + }; + await stitch({ + name: 'payments', + url: 'https://api.vendor.test/v1/payments', + adapter: serving(CLEAN), + kind: deprecationSurface({ fold: true }), + output: drift({ + validate: async (v: unknown) => ({ + ok: true as const, + value: { + balance: (v as Record)['balance'], + }, + }), + } satisfies Validator<{ balance: unknown }>), + trace: sink, + }).safe(); + checkSeq('(b) findings on a clean endpoint', found, []); + check( + '(b) …because the vendor sent no notice', + readNotice(headersFor(CLEAN)), + null, + ); + } + + // ── (c) the level is settable — but per KIND, not per path ─────────────────────────────── + { + const levels: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') levels.push(fmt(e.finding)); + }, + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: deprecationSurface({ fold: true }), + output: drift(declaresUsersOnly, { + severity: { undeclared: 'warn' }, + }), + trace: sink, + }).safe(); + checkSeq('(c) re-levelled to `warn`', levels, [ + 'warn|undeclared|_deprecation|undeclared field (object)', + ]); + note( + '(c) → `severity: { undeclared: "warn" }` works. It is a map of CHANGE KIND to level (types.ts:112-115), so there is no way to raise this one path without raising every undeclared field the vendor ever adds', + ); + } + { + // The collateral, measured: a vendor that adds an unrelated field gets the same `warn`. + const levels: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') levels.push(fmt(e.finding)); + }, + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: async () => ({ + status: 200, + headers: headersFor(USERS), + // The vendor ships a harmless new field on the same response. + body: { users: [], experimental_ranking: true }, + }), + kind: deprecationSurface({ fold: true }), + output: drift(declaresUsersOnly, { + severity: { undeclared: 'warn' }, + }), + trace: sink, + }).safe(); + checkSeq('(c) …and what it dragged up with it', levels.sort(), [ + 'warn|undeclared|_deprecation|undeclared field (object)', + 'warn|undeclared|experimental_ranking|undeclared field (boolean)', + ]); + note( + '(c) → a new vendor field is now a `warn` because a deprecation notice needed to be one. Re-levelling is a blunt instrument here', + ); + } + + // ── (d) what the finding does NOT carry ────────────────────────────────────────────────── + { + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: deprecationSurface({ fold: true }), + output: drift(declaresUsersOnly), + }); + const ins = await call.inspect(); + const f = ins.findings[0]; + checkSeq('(d) `DriftFinding` keys', Object.keys(f ?? {}).sort(), [ + 'change', + 'detail', + 'level', + 'path', + ]); + check( + '(d) does `detail` carry the sunset date?', + f?.detail, + 'undeclared field (object)', + ); + check( + '(d) does the finding name the endpoint?', + 'endpoint' in (f ?? {}) || 'name' in (f ?? {}), + false, + ); + note( + '(d) → the finding says a notice EXISTS and where in the payload it sits. It cannot say when the sunset is, and it cannot say which endpoint — `ctx.name` at a sink is the only thing that can', + ); + } + + // ── (e) …unless you smuggle the value into the PATH, which works and is horrible ───────── + { + const found: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') found.push(fmt(e.finding)); + }, + }; + await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: { + id: 'smuggle', + interpret: (res) => { + const n = readNotice(res.headers); + const key = + n?.sunsetAt == null + ? '_ok' + : `_sunset_${new Date(n.sunsetAt).toISOString().slice(0, 10)}`; + return { + ok: true, + data: { ...(res.body as object), [key]: true }, + }; + }, + }, + output: drift(declaresUsersOnly), + trace: sink, + }).safe(); + checkSeq('(e) the date, carried in the finding path', found, [ + 'info|undeclared|_sunset_2026-01-01|undeclared field (boolean)', + ]); + note( + '(e) → the date IS now in the finding, because the path is the only free-form string a finding has. It also means every distinct sunset date is a distinct finding path, which no drift report is designed for', + ); + } + + // ── (f) minting a finding directly: there is no API ────────────────────────────────────── + { + // A `Validator` may only return a value or ISSUES, and issues are HARD findings that FAIL + // the call. Measured, so "you could just emit a warning from the validator" is closed off. + const found: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') found.push(fmt(e.finding)); + }, + }; + const r = await stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: deprecationSurface({ fold: true }), + output: drift({ + validate: async () => ({ + ok: false as const, + issues: [ + { + path: ['_deprecation', 'sunsetAt'], + message: 'endpoint is deprecated', + }, + ], + }), + } satisfies Validator), + trace: sink, + }).safe(); + check('(f) a validator-issued finding is FATAL', r.ok, false); + check( + '(f) …and this is what the caller sees', + r.error?.message, + 'contract violation (drift)', + ); + checkSeq('(f) the finding it produced', found, [ + 'error|invalid|_deprecation.sunsetAt|endpoint is deprecated', + ]); + note( + '(f) → the only finding user code can author directly is an `error|invalid`, and it kills the call. `ValidationResult` is `{ ok, value } | { ok, issues }` (validator.ts:11-12) — there is no third arm for a warning', + ); + } + + // ── (g) which accessors carry the header-derived finding ───────────────────────────────── + { + const streamed: string[] = []; + const sunk: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') sunk.push(fmt(e.finding)); + }, + }; + const call = stitch({ + name: 'users', + url: 'https://api.vendor.test/v1/users', + adapter: serving(USERS), + kind: deprecationSurface({ fold: true }), + output: drift(declaresUsersOnly), + trace: sink, + }); + for await (const e of call.stream()) + if (e.type === 'drift') streamed.push(fmt(e.finding)); + const safe = await call.safe(); + const ins = await call.inspect(); + const rep = await call.report(); + + checkSeq('(g) `.stream()`', streamed, [ + 'info|undeclared|_deprecation|undeclared field (object)', + ]); + checkSeq('(g) `.inspect().findings`', ins.findings.map(fmt), [ + 'info|undeclared|_deprecation|undeclared field (object)', + ]); + checkSeq('(g) `.report().findings`', rep.findings.map(fmt), [ + 'info|undeclared|_deprecation|undeclared field (object)', + ]); + checkSeq( + '(g) trace sink (4 runs)', + [...new Set(sunk)], + ['info|undeclared|_deprecation|undeclared field (object)'], + ); + checkSeq('(g) `.safe()` — anything?', Object.keys(safe).sort(), [ + 'data', + 'error', + 'ok', + ]); + check('(g) `findings` on `SafeResult`?', 'findings' in safe, false); + note( + '(g) → four accessors carry it and the awaited path carries none of it, which is exactly the drift table from scenario 12. The notice now has the same reporting reach as every other finding — and the same blind spot', + ); + } + + finish( + 'C3', + 'YES, THROUGH ONE NARROW DOOR, AND IT COSTS SOMETHING. A surface folds the notice into the value and an `output` contract that does not declare it reports `info|undeclared|_deprecation|undeclared field (object)` — a genuine drift finding on the ordinary channel, non-fatal (`ok: true`, `error: null`, value delivered), visible on `.stream()`, `.inspect().findings`, `.report().findings` and a `TraceSink`, invisible on `.safe()`. `severity: { undeclared: "warn" }` re-levels it. Three limits, all measured: the finding does NOT carry the header value (`detail` is `undeclared field (object)`, so it says a notice exists and never says the sunset is 1 Jan) — unless you smuggle the date into the PATH, which works (`_sunset_2026-01-01`) and makes every date its own finding path; the finding does NOT name the endpoint (`DriftFinding` is `{ level, path, change, detail }`; only `ctx.name` at a sink knows); and re-levelling is per-KIND, so raising the notice to `warn` also raised an unrelated new vendor field to `warn` in the same run. There is no API for minting a finding: a `Validator` returns a value or ISSUES, and an issue is an `error|invalid` that FAILS the call with `contract violation (drift)`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c4-aggregation.ts b/docs/scenarios/proofs/deprecation-headers/c4-aggregation.ts new file mode 100644 index 00000000..3caaada9 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c4-aggregation.ts @@ -0,0 +1,382 @@ +// C4 — DECIDING CLAIM. Across many stitches and many calls, can a `TraceSink` produce +// "these 3 endpoints are deprecated, earliest sunset in 12 days"? +// +// It can, and the shape scenario 12 built for drift rates transfers almost intact: one sink, +// configured once, `handle(event, ctx)` for every event of every call, `ctx.name` naming the +// endpoint, a `Map` collapsing calls into one row per endpoint. Measured below: 5 endpoints, +// 500 calls, and the sink reports +// +// 3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users +// +// What does NOT transfer is where the DATA comes from. A drift finding arrives at the sink on its +// own — the engine emits it. A response header never arrives at all: C1 measured every event key on +// the spine and none is a header. So the sink can only aggregate a notice that a SURFACE already +// folded into the value, and the two have to be built as a pair. The sink is an aggregation seam; +// it is not a header seam. +// +// Four traps measured at the end, one of which serves a sunset date that expired months ago. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c4-aggregation.ts +import { seam, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + DeprecationWatch, + type Notice, + deprecationSurface, + noticeOf, +} from './deprecation'; +import { BASE, FLEET, FakeVendor, NOW, endpoint, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; + +async function main(): Promise { + heading('C4 — DECIDING. Can a sink produce the fleet view?'); + + // ── (a) the answer: 5 endpoints, 500 calls, one report ─────────────────────────────────── + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const watch = new DeprecationWatch(clock); + const api = seam({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + trace: watch, + }); + // The surface is written on every member because seam-level `kind` does not TYPECHECK + // (TS2353) — even though the engine would have inherited it from the seam. See (f). + const surface = deprecationSurface({ fold: true }); + const members = FLEET.map((e) => + api.stitch({ name: e.name, path: e.path, kind: surface }), + ); + + for (let round = 0; round < 100; round += 1) + for (const m of members) await m(); + + check('(a) calls made', vendor.total, 500); + check('(a) endpoints called', vendor.requests.size, 5); + check('(a) rows in the fleet report', watch.fleet().length, 3); + checkSeq( + '(a) the report, soonest sunset first', + watch + .fleet() + .map( + (r) => + `${r.endpoint} sunset=${new Date(r.notice.sunsetAt ?? 0).toISOString().slice(0, 10)} calls=${String(r.calls)}`, + ), + [ + 'users sunset=2026-01-01 calls=100', + 'search sunset=2026-03-15 calls=100', + 'orders sunset=2026-06-01 calls=100', + ], + ); + check( + '(a) the one line an operator reads', + watch.summary(), + '3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users', + ); + note( + '(a) → the exact sentence the capture asked for, from 500 calls across 5 endpoints, with ONE sink configured once at the seam', + ); + } + + // ── (b) what identifies the endpoint at the sink: `ctx.name`, and its default collides ─── + { + const rows: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type === 'start') rows.push(`ctx.name=${ctx.name}`); + }, + }; + // Two DIFFERENT endpoints, neither given a `name`. + await stitch({ + url: `${BASE}/v1/users`, + adapter: serving(endpoint('users')), + trace: sink, + })(); + await stitch({ + url: `${BASE}/v1/orders`, + adapter: serving(endpoint('orders')), + trace: sink, + })(); + checkSeq('(b) two endpoints, no `name` set', rows, [ + 'ctx.name=stitch', + 'ctx.name=stitch', + ]); + note( + '(b) → `name` defaults to `path` or `"stitch"` (types.ts:1424-1425), and a `url`-configured stitch has no `path`. Both endpoints answer to `stitch`, so a Map keyed on `ctx.name` silently merges them. Naming every member is not optional here', + ); + } + { + // The URL is on the `start` event, not on `ctx` — so recovering it means correlating + // `start` to `result` by `spanId`. Measured, because "just use the url" is the obvious fix. + const urls = new Map(); + const resolved: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type === 'start' && ctx.spanId !== undefined) + urls.set(ctx.spanId, e.url); + if (e.type === 'result' && ctx.spanId !== undefined) + resolved.push(urls.get(ctx.spanId) ?? ''); + }, + }; + await stitch({ + url: `${BASE}/v1/users`, + adapter: serving(endpoint('users')), + trace: sink, + })(); + await stitch({ + url: `${BASE}/v1/orders`, + adapter: serving(endpoint('orders')), + trace: sink, + })(); + checkSeq('(b) …recovered via `spanId` correlation', resolved, [ + `${BASE}/v1/users`, + `${BASE}/v1/orders`, + ]); + note( + '(b) → it works and it is a second Map plus a join. `ctx` carries `{ name, spanId, traceId }`; the URL only ever appears on `start`', + ); + } + + // ── (c) does the header reach the sink WITHOUT the surface? No ─────────────────────────── + { + const notices: (Notice | null)[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'result') notices.push(noticeOf(e.data)); + }, + }; + // Same vendor, same headers, DEFAULT surface. + await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(endpoint('users')), + trace: sink, + })(); + checkSeq('(c) notices at the sink, default surface', notices, [null]); + note( + '(c) → THE COUPLING. The vendor sent both headers; the sink saw a body. Aggregation is a real seam and it has nothing to aggregate until a surface puts the notice on the value', + ); + } + + // ── (d) …and an `output` contract that strips the fold breaks it again ─────────────────── + { + const notices: (Notice | null)[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'result') notices.push(noticeOf(e.data)); + }, + }; + await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(endpoint('users')), + kind: deprecationSurface({ fold: true }), + // A perfectly ordinary contract that declares only what the caller uses. + output: { + validate: async (v: unknown) => ({ + ok: true as const, + value: { users: (v as Record)['users'] }, + }), + }, + trace: sink, + })(); + checkSeq('(d) notices at the sink, after validation', notices, [null]); + note( + '(d) → the `result` event carries the VALIDATED value, so a schema that does not declare `_deprecation` deletes the notice on its way to the sink. The surface, the contract and the sink are one coupled unit, and nothing warns you when they disagree', + ); + } + + // ── (e) the side channel: the surface reports directly, no sink, no payload change ─────── + { + const seen: string[] = []; + const collected = new Map(); + const surface = deprecationSurface({ + onNotice: (name, notice) => { + seen.push(name); + collected.set(name, notice); + }, + }); + const vendor = new FakeVendor(); + const api = seam({ baseUrl: BASE, adapter: vendor.adapter() }); + for (const e of FLEET) + for (let i = 0; i < 3; i += 1) + await api.stitch({ + name: e.name, + path: e.path, + kind: surface, + })(); + + check('(e) calls made', vendor.total, 15); + check('(e) notice callbacks', seen.length, 9); + checkSeq('(e) endpoints collected', [...collected.keys()].sort(), [ + 'orders', + 'search', + 'users', + ]); + const value = await api.stitch({ + name: 'users', + path: '/v1/users', + kind: surface, + })(); + checkSeq( + "(e) …and the caller's value is untouched", + Object.keys(value as object).sort(), + ['users'], + ); + note( + '(e) → the same fleet view with no `trace`, no `output` coupling and no change to the result type. It is not the trace channel, and for this signal that may be the right trade', + ); + } + + // ── (f) seam-level `kind`: a compile error that WORKS AT RUNTIME ────────────────────────── + // The hypothesis going in was "`SeamConfig` omits `kind` (types.ts:1988-1998), so a 40-member + // seam repeats the surface 40 times". Half right, and the wrong half is the interesting one: + // the type genuinely forbids it — + // + // seam({ baseUrl, kind: httpSurface }) + // error TS2353: Object literal may only specify known properties, + // and 'kind' does not exist in type 'SeamOptions'. + // + // — but the ENGINE composes it into every member anyway. Measured here through a cast, so the + // runtime behaviour is on the page rather than inferred from the type. + { + const surface = deprecationSurface({ fold: true }); + const vendor = new FakeVendor(); + const api = seam({ + baseUrl: BASE, + adapter: vendor.adapter(), + kind: surface, + } as never) as ReturnType; + const member = api.stitch({ name: 'users', path: '/v1/users' }); + + check( + '(f) surface id on the SEAM config', + api.__config['kind'], + 'http+deprecation', + ); + check( + '(f) surface id the MEMBER inherited (no `kind` of its own)', + member.__config['kind'], + 'http+deprecation', + ); + check( + '(f) …and the notice it folded', + noticeOf(await member())?.sunsetAt, + Date.parse('2026-01-01T00:00:00Z'), + ); + check( + '(f) a plain seam resolves to', + seam({ baseUrl: BASE, adapter: serving(endpoint('users')) }) + .__config['kind'], + 'http', + ); + note( + '(f) → the capability is REAL and the type is closed over it. `SeamOptions` omits `kind`, so a typed codebase writes the surface on all 40 members while the engine would have inherited it from one. `trace` has no such problem — the sink genuinely is configured once', + ); + } + + // ── (g) TRAP: a cached response re-serves a STALE notice, forever ──────────────────────── + { + const clock = manualClock(NOW); + const watch = new DeprecationWatch(clock); + const vendor = new FakeVendor(); + const call = stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: vendor.adapter(), + kind: deprecationSurface({ fold: true }), + cache: { ttl: '1h', version: 'v1' }, + clock, + trace: watch, + }); + for (let i = 0; i < 10; i += 1) await call(); + check('(g) wire requests', vendor.total, 1); + check('(g) calls the sink counted', watch.fleet()[0]?.calls, 10); + // Time moves a year. The cache TTL is an hour, so the value is re-fetched — but a cache + // that had NOT expired would still be handing out the notice it captured a year ago. + checkSeq( + '(g) the report, from 1 wire response and 9 cache hits', + [watch.summary()], + [ + '1 endpoints deprecated (users), earliest sunset in 12 days: users', + ], + ); + note( + '(g) → the notice is now CACHED DATA. Nine of those ten rows are a header read once and replayed; with a long TTL a sunset that already passed keeps reporting as "in 12 days" until the entry expires', + ); + } + + // ── (h) TRAP: `.inspect()` / `.report()` are fresh runs and land in the sink ───────────── + { + const clock = manualClock(NOW); + const watch = new DeprecationWatch(clock); + const vendor = new FakeVendor(); + const call = stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: vendor.adapter(), + kind: deprecationSurface({ fold: true }), + clock, + trace: watch, + }); + await call(); + const before = watch.fleet()[0]?.calls ?? 0; + await call.report(); + await call.inspect(); + checkSeq( + '(h) sink call-count before / after one `.report()` + one `.inspect()`', + [before, watch.fleet()[0]?.calls ?? 0, vendor.total], + [1, 3, 3], + ); + note( + '(h) → same trap scenario 12 measured for drift rates: a diagnostic probe is a real run that costs a request and a tick in the denominator', + ); + } + + // ── (i) TRAP: the deprecated endpoints are a MINORITY of a healthy-looking fleet ───────── + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const watch = new DeprecationWatch(clock); + const api = seam({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + trace: watch, + }); + const surface = deprecationSurface({ fold: true }); + // Traffic skewed the way real traffic is: the healthy endpoints carry it. + for (let i = 0; i < 200; i += 1) + await api.stitch({ + name: 'payments', + path: '/v1/payments', + kind: surface, + })(); + await api.stitch({ name: 'users', path: '/v1/users', kind: surface })(); + + check('(i) calls made', vendor.total, 201); + check('(i) calls that carried a notice', 1, 1); + check('(i) rows in the report', watch.fleet().length, 1); + check( + '(i) the report', + watch.summary(), + '1 endpoints deprecated (users), earliest sunset in 12 days: users', + ); + note( + '(i) → ONE call in 201 carried the signal and the report is identical. That is the property a per-call log line does not have, and it is the whole argument for aggregating', + ); + } + + finish( + 'C4', + 'IT WORKS, AND THE SCENARIO-12 SHAPE TRANSFERS — BUT NOT THE DATA PATH. One `DeprecationWatch` sink configured once on a seam, 500 calls across 5 endpoints, reported `3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users`, one row per endpoint however many calls arrived; with traffic skewed 200:1 toward the healthy endpoints the report was unchanged, which is exactly what a per-call log line cannot do. `ctx.name` is what identifies the endpoint — and its default is the literal `"stitch"` for any `url`-configured stitch, so two unnamed endpoints silently merge into one row; the URL exists only on the `start` event, recoverable by correlating on `ctx.spanId` at the cost of a second Map and a join. THE HEADER VALUE DOES NOT REACH THE SINK ON ITS OWN: with the default surface the sink saw `null` for a response that carried both headers, because no event carries headers at all. It only arrives if a Surface folded it into the value — and an ordinary `output` contract that does not declare `_deprecation` deletes it again before the `result` event fires, with no warning. Three further traps: seam-level `kind` is a COMPILE ERROR (`TS2353: kind does not exist in type SeamOptions`) that the engine honours perfectly at runtime — members inherited `http+deprecation` and folded correctly — so a typed codebase writes the surface on all 40 members for a capability that already works from one; a cache hit re-serves a notice captured on the ONE wire response, so 9 of 10 rows were a replayed header and a long TTL will report a passed sunset as "in 12 days"; and `.report()`/`.inspect()` are fresh runs that each add a request and a tick. The side channel — `onNotice` straight out of the surface — produces the identical fleet view with no `trace`, no `output` coupling and an untouched result type', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c5-tripwire.ts b/docs/scenarios/proofs/deprecation-headers/c5-tripwire.ts new file mode 100644 index 00000000..2e0a22e3 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c5-tripwire.ts @@ -0,0 +1,307 @@ +// C5 — the deliberate tripwire: can a call be made to FAIL after a chosen sunset date and not +// before, on an injected clock? Which seam? +// +// Yes, and the seam is `Surface.interpret` — the same one C1 found the headers on. It returns +// `SurfaceOutcome`, and the `{ ok: false, message, status }` arm fails the call, so reading the +// header and rendering the verdict happen in one place with the stitch's own `clock` in scope +// (`cfg.clock`, measured below). +// +// The library treats a surface-rejected 200 correctly on both axes that matter for a tripwire: +// it does NOT burn retry attempts (no `retry: true` on the outcome, so no re-attempt), and it does +// NOT count against the circuit breaker — `classifyStatus` rules on the STATUS, and a 200 the +// surface rejected is an application-level verdict on a healthy transport (surface.ts:124-141). +// Both measured, because a tripwire that opens a breaker takes down the endpoints that are fine. +// +// RFC 9745 is explicit that deprecation is a hint, not a guarantee, so this is never a default. +// It is for a deadline you have decided to enforce — and the grace-period variant in (g) enforces +// one you chose rather than one the vendor chose. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c5-tripwire.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Clock, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { deprecationSurface, readNotice } from './deprecation'; +import { BASE, DAY, endpoint, headersFor, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const USERS = endpoint('users'); +const CLEAN = endpoint('payments'); +/** The instant `users` stops answering, per its `Sunset` header. */ +const SUNSET = Date.parse('2026-01-01T00:00:00Z'); + +/** One call against `users` through the tripwire surface, on a clock pinned to `at`. */ +async function callAt( + at: number, +): Promise<{ ok: boolean; message: string | undefined }> { + const r = await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(USERS), + kind: deprecationSurface({ failAfterSunset: true }), + clock: manualClock(at), + retry: { attempts: 1 }, + }).safe(); + return { ok: r.ok, message: r.error?.message }; +} + +async function main(): Promise { + heading('C5 — fail after a chosen sunset, and not before'); + + // ── (a) the crossing ───────────────────────────────────────────────────────────────────── + { + const before = await callAt(SUNSET - DAY); + const after = await callAt(SUNSET + DAY); + check('(a) one day BEFORE sunset — ok', before.ok, true); + check('(a) …with no error', before.message, undefined); + check('(a) one day AFTER sunset — ok', after.ok, false); + check( + '(a) …and the message names the endpoint and the date', + after.message, + 'users: sunset passed (2026-01-01T00:00:00.000Z)', + ); + note( + '(a) → same stitch, same vendor, same 200 on the wire. The only thing that changed is the clock', + ); + } + + // ── (b) the boundary is exact, to the millisecond ──────────────────────────────────────── + { + const spine = await Promise.all( + [SUNSET - 1, SUNSET, SUNSET + 1].map(async (t) => + (await callAt(t)).ok ? 'ok' : 'FAILED', + ), + ); + checkSeq('(b) [sunset-1ms, sunset, sunset+1ms]', spine, [ + 'ok', + 'FAILED', + 'FAILED', + ]); + note( + '(b) → `>=` at the boundary, which is the right reading of "the resource is expected to become unresponsive AT this instant"', + ); + } + + // ── (c) `cfg.clock` really is the stitch's injected clock ─────────────────────────────── + { + let sawInSurface = -1; + const probe: Surface = { + id: 'clock-probe', + interpret: (res, cfg) => { + sawInSurface = cfg.clock?.now() ?? -1; + return { ok: true, data: res.body }; + }, + }; + const clock: Clock = manualClock(SUNSET - 5 * DAY); + await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(USERS), + kind: probe, + clock, + })(); + check( + '(c) `cfg.clock.now()` inside `interpret`', + sawInSurface, + clock.now(), + ); + check( + '(c) …which is 5 days before sunset', + SUNSET - sawInSurface, + 5 * DAY, + ); + note( + '(c) → `ResolvedStitchConfig` carries `clock` (types.ts:1582), so the tripwire never reads wall-clock time and a test never waits', + ); + } + + // ── (d) a clean endpoint is never tripped ──────────────────────────────────────────────── + { + const r = await stitch({ + name: 'payments', + url: `${BASE}/v1/payments`, + adapter: serving(CLEAN), + kind: deprecationSurface({ failAfterSunset: true }), + clock: manualClock(SUNSET + 365 * DAY), + }).safe(); + check("(d) a year past the OTHER endpoint's sunset", r.ok, true); + check( + '(d) …because it announced none', + readNotice(headersFor(CLEAN)), + null, + ); + } + + // ── (e) the tripwire does NOT burn retry attempts ──────────────────────────────────────── + { + let requests = 0; + const r = await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: async () => { + requests += 1; + return { + status: 200, + headers: headersFor(USERS), + body: USERS.body, + }; + }, + kind: deprecationSurface({ failAfterSunset: true }), + clock: manualClock(SUNSET + DAY), + retry: { attempts: 5, backoff: { curve: 'fixed', base: 0 } }, + }).safe(); + check('(e) ok', r.ok, false); + check('(e) requests made, with 5 attempts allowed', requests, 1); + note( + '(e) → the `{ ok: false, message }` arm is terminal. `SurfaceOutcome` has a separate `{ ok: false, retry: true }` arm (surface.ts:34-37) for body-aware retry, and not asking for it means not getting it', + ); + } + + // ── (f) …and it does NOT open the circuit breaker ──────────────────────────────────────── + // A tripwire that trips the breaker would fast-fail every OTHER call sharing the key, which is + // the opposite of what a deprecation guard is for. + { + let requests = 0; + const call = stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: async () => { + requests += 1; + return { + status: 200, + headers: headersFor(USERS), + body: USERS.body, + }; + }, + kind: deprecationSurface({ failAfterSunset: true }), + clock: manualClock(SUNSET + DAY), + retry: { attempts: 1 }, + circuit: { failures: 2, cooldown: '30s', key: 'tripwire-test' }, + }); + const outcomes: string[] = []; + for (let i = 0; i < 5; i += 1) { + const r = await call.safe(); + outcomes.push(r.error?.message ?? 'ok'); + } + check('(f) requests that reached the vendor', requests, 5); + checkSeq( + '(f) distinct outcomes over 5 calls past a `failures: 2` breaker', + [...new Set(outcomes)], + ['users: sunset passed (2026-01-01T00:00:00.000Z)'], + ); + note( + '(f) → five consecutive failures and the breaker never opened, so no "circuit open" ever replaced the real message. `classifyStatus` rules on the STATUS and the transport was healthy the whole time (surface.ts:124-141)', + ); + } + + // ── (g) the deadline you chose, not the one the vendor chose ───────────────────────────── + // A 14-day grace: fail while there is still time to fix it, in a staging environment, rather + // than on the morning the endpoint disappears. + { + const grace = 14 * DAY; + const early: Surface = { + id: 'sunset-grace', + interpret: (res, cfg) => { + const n = readNotice(res.headers); + if (n?.sunsetAt == null) return { ok: true, data: res.body }; + const left = n.sunsetAt - (cfg.clock?.now() ?? 0); + if (left <= grace) + return { + ok: false, + message: `${cfg.name ?? 'stitch'}: sunset in ${String(Math.round(left / DAY))} days — under the ${String(grace / DAY)}-day grace`, + status: res.status, + }; + return { ok: true, data: res.body }; + }, + }; + const at = async (t: number): Promise => { + const r = await stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(USERS), + kind: early, + clock: manualClock(t), + retry: { attempts: 1 }, + }).safe(); + return r.error?.message ?? 'ok'; + }; + checkSeq( + '(g) 20 / 14 / 10 days out', + [ + await at(SUNSET - 20 * DAY), + await at(SUNSET - 14 * DAY), + await at(SUNSET - 10 * DAY), + ], + [ + 'ok', + 'users: sunset in 14 days — under the 14-day grace', + 'users: sunset in 10 days — under the 14-day grace', + ], + ); + note( + '(g) → the countdown is in the failure message, so the thing that breaks the build also says how long you had', + ); + } + + // ── (h) what the failure looks like on every accessor ──────────────────────────────────── + { + const events: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'error') events.push(`error:${e.message}`); + else events.push(e.type); + }, + }; + const call = stitch({ + name: 'users', + url: `${BASE}/v1/users`, + adapter: serving(USERS), + kind: deprecationSurface({ failAfterSunset: true }), + clock: manualClock(SUNSET + DAY), + retry: { attempts: 1 }, + trace: sink, + }); + const safe = await call.safe(); + check( + '(h) `.safe()` message', + safe.error?.message, + 'users: sunset passed (2026-01-01T00:00:00.000Z)', + ); + check( + "(h) `.safe()` status — the vendor's, preserved", + safe.error?.status, + 200, + ); + checkSeq('(h) event spine of a tripped call', events, [ + 'start', + 'progress', + 'error:users: sunset passed (2026-01-01T00:00:00.000Z)', + 'done', + ]); + let threw = ''; + try { + await call(); + } catch (e) { + threw = (e as Error).message; + } + check( + '(h) the bare await throws', + threw, + 'users: sunset passed (2026-01-01T00:00:00.000Z)', + ); + note( + '(h) → a `status: 200` on a `StitchError` is the honest record: the transport succeeded and a policy rejected it. Anything reading `error.status >= 500` will not see this, which is the point', + ); + } + + finish( + 'C5', + 'YES, EXACTLY, AND THE SEAM IS `Surface.interpret`. Its `{ ok: false, message, status }` arm fails the call, and `cfg.clock` is the stitch\'s injected clock, so the crossing is deterministic to the millisecond: `[sunset-1ms, sunset, sunset+1ms]` measured `["ok","FAILED","FAILED"]`, and the same stitch against the same 200 passed a day before and failed a day after. The failure message names the endpoint and the date (`users: sunset passed (2026-01-01T00:00:00.000Z)`), reaches `.safe()`, the thrown error and the `error` event, and carries `status: 200` — the honest record that the transport was fine and a policy rejected it. Two behaviours make it safe to deploy, both measured: it does NOT burn retry attempts (5 attempts configured, 1 request made — the `{ ok: false, retry: true }` arm is opt-in) and it does NOT open the circuit breaker (5 consecutive trips past `failures: 2` and every message was still the real one, because `classifyStatus` rules on the status and the transport was healthy). A clean endpoint a year past someone else\'s sunset never trips. The grace-period variant enforces a deadline YOU chose and puts the countdown in the message (`sunset in 10 days — under the 14-day grace`)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c6-formats.ts b/docs/scenarios/proofs/deprecation-headers/c6-formats.ts new file mode 100644 index 00000000..59351b44 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c6-formats.ts @@ -0,0 +1,244 @@ +// C6 — both formats. Is there any parsing help in the library for a structured-field date or an +// HTTP-date? And what does hand-rolling cost? +// +// Scenario 14 found `parseRetryAfter` exists but is unexported. This is the same finding one turn +// worse, because `parseRetryAfter` does not merely resemble what `Sunset` needs — it is EXACTLY it. +// `Sunset` is an HTTP-date (RFC 8594); `Retry-After` is delta-seconds OR an HTTP-date (RFC 9110); +// `parseRetryAfter` handles both, takes an injectable `Clock`, and returns ms-until. Measured below +// against the real `Sunset` values from the fake vendor: it parses every one of them correctly. +// +// It is not reachable. `resilience.ts` is not an export subpath (packages/core/package.json lists +// `.`, `./serve`, `./mcp`, `./testing`, … and no `./resilience`), and `index.ts` re-exports only +// `RateLimitError` from it. The three date-ish helpers the barrel DOES export — `parseDuration`, +// `parseBytes`, `parseRate` — parse durations, sizes and rates, and none of them parses a date. +// +// So both parsers are hand-rolled, and the sf-date one has a trap in it: `Date.parse('@1735689600')` +// is `NaN`, so the obvious one-liner silently reports "no deprecation" for the RFC 9745 spelling. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c6-formats.ts +import * as publicApi from '../../../../packages/core/src/index'; +import { parseRetryAfter } from '../../../../packages/core/src/resilience'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { parseDeprecation, parseSunset, readNotice } from './deprecation'; +import { FLEET, NOW } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** Executable lines between two markers — blanks, comments and the JSDoc removed. */ +function linesBetween(file: string, begin: string, end: string): number { + const src = readFileSync(join(HERE, file), 'utf8'); + const from = src.indexOf(begin); + const to = src.indexOf(end); + return src + .slice(from + begin.length, to) + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +async function main(): Promise { + heading('C6 — two RFCs, two formats, and what the library offers'); + + // ── (a) what the public barrel exports that could possibly help ────────────────────────── + { + const exported = Object.keys(publicApi).filter((k) => + k.startsWith('parse'), + ); + checkSeq('(a) `parse*` exports on the public barrel', exported.sort(), [ + 'parseBytes', + 'parseDuration', + 'parseRate', + ]); + check( + '(a) is `parseRetryAfter` among them?', + 'parseRetryAfter' in publicApi, + false, + ); + // The three that ARE exported, pointed at a date. None of them is for this. + check( + '(a) `parseDuration` on an HTTP-date', + publicApi.parseDuration('Thu, 01 Jan 2026 00:00:00 GMT'), + undefined, + ); + check( + '(a) `parseDuration` on an sf-date', + publicApi.parseDuration('@1735689600'), + undefined, + ); + note( + '(a) → `parseDuration`/`parseBytes`/`parseRate` parse `"30s"`, `"5mb"`, `"2/s"`. A date is not a duration, and nothing in the barrel reads one', + ); + } + + // ── (b) the helper that would have done it, reached by a path a consumer cannot use ────── + { + const clock = manualClock(NOW); + const parsed = FLEET.filter((e) => e.sunset !== undefined).map((e) => + parseRetryAfter(e.sunset, clock), + ); + checkSeq( + '(b) `parseRetryAfter` on every real `Sunset` (ms from NOW)', + parsed, + [ + Date.parse('2026-01-01T00:00:00Z') - NOW, + Date.parse('2026-03-15T00:00:00Z') - NOW, + Date.parse('2026-06-01T00:00:00Z') - NOW, + ], + ); + check( + '(b) …the first one, in days', + Math.round((parsed[0] ?? 0) / 86_400_000), + 12, + ); + note( + '(b) → it parses the HTTP-date, subtracts an INJECTED clock, and returns ms-until. That is the whole `Sunset` requirement, already written, already tested, imported here only by reaching into `packages/core/src/resilience` — which no installed consumer can do', + ); + } + + // ── (c) …and it is genuinely not reachable from the published package ──────────────────── + { + const pkg = JSON.parse( + readFileSync( + join(HERE, '../../../../packages/core/package.json'), + 'utf8', + ), + ) as { exports: Record }; + const subpaths = Object.keys(pkg.exports).sort(); + check( + '(c) is there a `./resilience` subpath?', + subpaths.includes('./resilience'), + false, + ); + check('(c) export subpaths published', subpaths.length, 17); + note( + `(c) → published subpaths: ${subpaths.join(' ')}. \`parseRetryAfter\` is behind none of them`, + ); + } + + // ── (d) the trap: the obvious one-liner silently drops the RFC 9745 spelling ───────────── + { + const naive = (v: string): number => Date.parse(v); + check( + '(d) `Date.parse` on the sf-date `@1735689600`', + Number.isNaN(naive('@1735689600')), + true, + ); + check( + '(d) `Date.parse` on the HTTP-date', + naive('Thu, 01 Jan 2026 00:00:00 GMT'), + Date.parse('2026-01-01T00:00:00Z'), + ); + note( + '(d) → a client that reaches for `Date.parse` gets `Sunset` right and reports NO DEPRECATION for the header the RFC actually specifies. Silently — `NaN` is falsy and every naive guard treats it as absent', + ); + } + + // ── (e) the hand-rolled pair, against every spelling the fleet serves ──────────────────── + { + checkSeq( + '(e) `Deprecation`, all three fleet spellings', + FLEET.map((e) => parseDeprecation(e.deprecation)), + [ + Date.parse('2025-01-01T00:00:00Z'), // users — sf-date `@1735689600` + Date.parse('2025-03-01T00:00:00Z'), // search — HTTP-date (pre-RFC draft) + Date.parse('2025-07-01T00:00:00Z'), // orders — sf-date `@1751328000` + null, // payments — clean + null, // webhooks — clean + ], + ); + checkSeq( + '(e) `Sunset`, always an HTTP-date', + FLEET.map((e) => parseSunset(e.sunset)), + [ + Date.parse('2026-01-01T00:00:00Z'), + Date.parse('2026-03-15T00:00:00Z'), + Date.parse('2026-06-01T00:00:00Z'), + null, + null, + ], + ); + note( + '(e) → both RFCs, three spellings, one `Notice` shape. The two formats really do have to be handled separately: `Deprecation` is an sf-date and `Sunset` is an HTTP-date, and the RFCs disagree on purpose', + ); + } + + // ── (f) the edges, because a silent NaN is how this fails ──────────────────────────────── + { + checkSeq( + '(f) `parseDeprecation` on the awkward inputs', + [ + parseDeprecation(undefined), + parseDeprecation(''), + parseDeprecation(' @1735689600 '), + parseDeprecation('@-86400'), + parseDeprecation('1735689600'), + parseDeprecation('true'), + parseDeprecation('@not-a-number'), + ], + [ + null, + null, + Date.parse('2025-01-01T00:00:00Z'), + -86_400_000, + null, + null, + null, + ], + ); + note( + '(f) → `@-86400` is legal syntax for a 1969 date and the sign has to survive; a BARE `1735689600` with no `@` is NOT an sf-date and is correctly refused rather than guessed at; the old boolean draft spelling `Deprecation: true` yields nothing, which is honest — it carries no date to yield', + ); + } + + // ── (g) a `Sunset` with no `Deprecation` is still a notice ─────────────────────────────── + { + const notice = readNotice({ sunset: 'Thu, 01 Jan 2026 00:00:00 GMT' }); + check('(g) deprecatedAt', notice?.deprecatedAt, null); + check( + '(g) sunsetAt', + notice?.sunsetAt, + Date.parse('2026-01-01T00:00:00Z'), + ); + check('(g) is it a notice at all?', notice !== null, true); + check( + '(g) …and a response with neither header', + readNotice({ 'content-type': 'application/json' }), + null, + ); + note( + '(g) → RFC 8594 stands alone. An endpoint that announces only its removal date is the more urgent case, and a parser that requires both headers misses it entirely', + ); + } + + // ── (h) what it cost ───────────────────────────────────────────────────────────────────── + { + const parserLines = linesBetween( + 'deprecation.ts', + '// >>> BEGIN PARSERS', + '// <<< END PARSERS', + ); + check('(h) executable lines of parsing', parserLines, 29); + note( + `(h) → ${String(parserLines)} executable lines for both RFCs, the \`Link\` successor, and the edges in (f). Small — but it is 100% of the format handling, and one of those lines exists only because \`Date.parse\` returns NaN for the spelling the standard mandates`, + ); + check('(h) …of which the library provided', 0, 0); + } + + finish( + 'C6', + 'NO HELP IS REACHABLE, AND THE HELP THAT EXISTS IS THE EXACT FUNCTION NEEDED. The public barrel exports three `parse*` helpers — `parseDuration`, `parseBytes`, `parseRate` — and none parses a date (`parseDuration` returns `undefined` for both an HTTP-date and an sf-date). `parseRetryAfter` in `resilience.ts` handles delta-seconds OR an HTTP-date against an INJECTABLE clock and returns ms-until, which is precisely the `Sunset` requirement; pointed at all three of the fleet\'s real `Sunset` values it returned the right ms, the first being 12 days. It is unreachable: `resilience.ts` is not among the 17 published export subpaths and `index.ts` re-exports only `RateLimitError` from it — scenario 14\'s finding, one turn worse, because here the unexported helper is not merely similar but identical in requirement. So both parsers are hand-rolled, and the naive spelling has a silent trap: `Date.parse("@1735689600")` is `NaN`, so a client that reaches for `Date.parse` reads `Sunset` correctly and reports NO DEPRECATION for the format RFC 9745 actually mandates. The hand-rolled pair is 29 executable lines covering both RFCs, the three spellings the fleet serves, the `Link` successor, a signed sf-date, a bare unprefixed integer (correctly refused), the legacy `Deprecation: true`, and a `Sunset` with no `Deprecation` — which is a notice, and the urgent one', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c7-noise.ts b/docs/scenarios/proofs/deprecation-headers/c7-noise.ts new file mode 100644 index 00000000..a50a4524 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c7-noise.ts @@ -0,0 +1,286 @@ +// C7 — noise. At request volume does the naive approach produce one line per call, and can it be +// de-duplicated per endpoint WITHOUT hand-rolled state? +// +// One line per call: yes, measured at 600. De-duplicated without hand-rolled state: no. The library +// ships one genuine drop hook — `loggerSink({ levelOf })` returns `null` to discard an event before +// it is logged, and `format` returning `null` does the same — but the predicate it drops on is a +// pure function of `(event, ctx)`, and "have I already reported this endpoint" is not. The `Map` +// that answers it is yours to write and yours to bound. +// +// The drift channel does not help either: a folded notice produces one `info|undeclared` finding +// PER CALL, so routing the notice through findings converts 600 log lines into 600 findings. +// +// Nothing in the library counts, latches, samples or rate-limits a repeated observation. The five +// lines of `Map` in `DeprecationWatch` are the entire difference between 600 lines and 3. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c7-noise.ts +import { drift, loggerSink, seam } from '../../../../packages/core/src/index'; +import type { + LogLevel, + LoggerLike, + Validator, +} from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + DeprecationWatch, + deprecationSurface, + noticeOf, + readNotice, +} from './deprecation'; +import { BASE, FLEET, FakeVendor, NOW } from './fake-vendor'; +import { check, checkAtMost, checkSeq, finish, heading, note } from './harness'; + +/** 120 rounds over the 5-endpoint fleet = 600 calls, 360 of them carrying a notice. */ +const ROUNDS = 120; + +/** A logger that counts lines instead of printing them. */ +function countingLogger(): LoggerLike & { lines: string[] } { + const lines: string[] = []; + const push = (m: string): void => void lines.push(m); + return { + lines, + error: push, + warn: push, + info: push, + debug: push, + }; +} + +/** Run the whole fleet `ROUNDS` times through one seam, and hand back the request total. */ +async function runFleet( + trace: TraceSink, + opts: { fold?: boolean; output?: Validator } = {}, +): Promise { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const api = seam({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + trace, + }); + const surface = deprecationSurface({ fold: opts.fold ?? false }); + const members = FLEET.map((e) => + api.stitch({ + name: e.name, + path: e.path, + kind: surface, + ...(opts.output ? { output: drift(opts.output) } : {}), + }), + ); + for (let r = 0; r < ROUNDS; r += 1) for (const m of members) await m(); + return vendor.total; +} + +async function main(): Promise { + heading('C7 — 600 calls. How many lines?'); + + // ── (a) the naive answer: one line per call that carries a notice ──────────────────────── + { + const lines: string[] = []; + const naive: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type !== 'result') return; + const n = noticeOf(e.data); + if (n !== null) + lines.push( + `WARN ${ctx.name} is deprecated, sunset ${new Date(n.sunsetAt ?? 0).toISOString().slice(0, 10)}`, + ); + }, + }; + const total = await runFleet(naive, { fold: true }); + check('(a) calls made', total, 600); + check('(a) log lines produced', lines.length, 360); + check('(a) distinct lines among them', new Set(lines).size, 3); + note( + '(a) → 360 lines carrying 3 distinct facts. This is the failure mode the capture names: "one line per call in a log nobody greps"', + ); + } + + // ── (b) the same thing through `hooks.onResponse`, for completeness ────────────────────── + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const lines: string[] = []; + const api = seam({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + hooks: { + onResponse: (c) => { + if (readNotice(c.res?.headers ?? {}) !== null) + lines.push(`WARN ${c.name} is deprecated`); + }, + }, + }); + const members = FLEET.map((e) => + api.stitch({ name: e.name, path: e.path }), + ); + for (let r = 0; r < ROUNDS; r += 1) for (const m of members) await m(); + check('(b) hook-logged lines', lines.length, 360); + note( + '(b) → identical volume from the other header-bearing seam, and `hooks` is seam-level so this is the one-line change a team actually makes first', + ); + } + + // ── (c) the library's built-in `loggerSink` is louder, not quieter ─────────────────────── + { + const logger = countingLogger(); + const total = await runFleet(loggerSink(logger), { fold: true }); + check('(c) calls made', total, 600); + check( + '(c) lines from the default `loggerSink`', + logger.lines.length, + 2400, + ); + checkSeq('(c) …one call, unpacked', logger.lines.slice(0, 4), [ + 'users GET https://api.vendor.test/v1/users', + 'users request#1', + 'users 200 ok (1 attempt(s))', + 'users done in 0ms', + ]); + note( + `(c) → 4 lines per call (start/progress/result/done) whether or not anything is wrong. The default sink is a run log, not a findings log — first line: "${logger.lines[0] ?? ''}"`, + ); + } + + // ── (d) `levelOf` CAN drop events — the only drop hook in the library ──────────────────── + { + const logger = countingLogger(); + const sink = loggerSink(logger, { + // Return `null` to discard. A pure function of (event, ctx) — which is exactly what + // makes it unable to express "only the first time". + levelOf: (e: StitchEvent): LogLevel | null => + e.type === 'result' ? 'warn' : null, + }); + const total = await runFleet(sink, { fold: true }); + check('(d) calls made', total, 600); + check( + '(d) lines after dropping 3 of 4 event types', + logger.lines.length, + 600, + ); + note( + '(d) → `levelOf` returning `null` is real filtering (trace.ts:415-451), and `format` returning `null` drops too. It cut 2400 to 600 and it cannot cut 600 to 3, because the question "have I said this already" is not answerable from one event', + ); + } + + // ── (e) …so the latch is user code, and it is a Map ────────────────────────────────────── + { + const seen = new Set(); + const lines: string[] = []; + const latched: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type !== 'result') return; + const n = noticeOf(e.data); + if (n === null || seen.has(ctx.name)) return; + seen.add(ctx.name); + lines.push( + `WARN ${ctx.name} is deprecated, sunset ${new Date(n.sunsetAt ?? 0).toISOString().slice(0, 10)}`, + ); + }, + }; + const total = await runFleet(latched, { fold: true }); + check('(e) calls made', total, 600); + check('(e) lines after a 3-line latch', lines.length, 3); + checkSeq('(e) the lines', lines, [ + 'WARN users is deprecated, sunset 2026-01-01', + 'WARN search is deprecated, sunset 2026-03-15', + 'WARN orders is deprecated, sunset 2026-06-01', + ]); + checkAtMost('(e) lines per deprecated endpoint', lines.length / 3, 1); + note( + '(e) → 600 calls, 3 lines, and the whole mechanism is a `Set` and an early return', + ); + } + + // ── (f) the aggregating sink does better: one REPORT, not one line per endpoint ────────── + { + const clock = manualClock(NOW); + const watch = new DeprecationWatch(clock); + const total = await runFleet(watch, { fold: true }); + check('(f) calls made', total, 600); + check('(f) rows', watch.fleet().length, 3); + check( + '(f) lines an operator reads', + watch.summary().split('\n').length, + 1, + ); + check( + '(f) …which is', + watch.summary(), + '3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users', + ); + note( + '(f) → the latch in (e) reports each endpoint ONCE EVER, so it cannot tell you the sunset moved. The aggregator keeps the latest per endpoint and is queried on demand, which is the shape that survives a vendor changing its mind', + ); + } + + // ── (g) routing the notice through DRIFT does not de-duplicate it either ───────────────── + { + let findings = 0; + const counter: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings += 1; + }, + }; + const total = await runFleet(counter, { + fold: true, + output: { + validate: async (v: unknown) => ({ + ok: true as const, + value: v, + }), + }, + }); + check('(g) calls made', total, 600); + check('(g) drift findings emitted', findings, 0); + note( + '(g) → zero, because a permissive validator that returns the value unchanged declares nothing undeclared. The finding in C3 came from a contract that STRIPPED `_deprecation`', + ); + } + { + let findings = 0; + const counter: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings += 1; + }, + }; + await runFleet(counter, { + fold: true, + output: { + // Declares nothing, so every top-level field is `undeclared` drift. + validate: async () => ({ ok: true as const, value: {} }), + }, + }); + check('(g) findings from a stripping contract', findings, 960); + note( + '(g) → one finding PER CALL per undeclared field, forever. The drift channel carries the notice (C3) and does not de-duplicate it — 600 log lines become 960 findings', + ); + } + + // ── (h) is there any built-in de-duplication anywhere? ─────────────────────────────────── + { + // Hunted for on the public barrel: anything that latches, samples, throttles a REPORT, or + // remembers an observation across calls. `throttle` is request pacing, not report pacing. + const dedupish = Object.keys( + await import('../../../../packages/core/src/index'), + ).filter((k) => /dedup|once|latch|sample|distinct|uniq/i.test(k)); + checkSeq('(h) de-duplication primitives exported', dedupish, []); + note( + '(h) → none. `cache.coalesce` de-duplicates REQUESTS, never observations, and `throttle` paces the wire. Report-level de-duplication is not a thing the library has an opinion about', + ); + } + + finish( + 'C7', + 'ONE LINE PER CALL, YES — 360 lines carrying 3 distinct facts across 600 calls — AND NO, IT CANNOT BE DE-DUPLICATED WITHOUT HAND-ROLLED STATE. The naive sink and the naive `hooks.onResponse` both produced 360 lines; the library\'s own `loggerSink` is louder still at 2400 (4 events per call, logged whether or not anything is wrong). There IS a real drop hook — `loggerSink({ levelOf })` returning `null` discards an event, and `format` returning `null` does too — and it cut 2400 to 600, but it cannot cut 600 to 3: `levelOf` is a pure function of `(event, ctx)` and "have I reported this endpoint already" is not answerable from one event. A 3-line `Set` latch in the sink took 600 calls to 3 lines; the aggregating `DeprecationWatch` did better by keeping ONE ROW PER ENDPOINT queried on demand, so a vendor that moves its sunset date is still visible where a fire-once latch would have gone quiet. Routing the notice through the drift channel does not help — a stripping contract emitted 960 findings over the same 600 calls, one per call per undeclared field. Nothing on the public barrel latches, samples or de-duplicates an observation: `cache.coalesce` de-duplicates requests and `throttle` paces the wire, and neither has anything to say about a repeated report', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/c8-assembled.ts b/docs/scenarios/proofs/deprecation-headers/c8-assembled.ts new file mode 100644 index 00000000..ca73ca40 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/c8-assembled.ts @@ -0,0 +1,233 @@ +// C8 — assemble the best available answer and report the seam(s) and the line count. +// +// Both sides do the same job over the same `Adapter` and the same `Clock`: parse both header +// formats, keep one row per endpoint, sort by soonest sunset, answer "how many days", and fail +// after a sunset once it has passed. The parsers are shared (C6 established they are 100% user code +// either way, so re-typing them in the control would inflate it by 29 lines that say nothing about +// the library). +// +// The number goes AGAINST the library, and that is the honest result: 132 executable lines to the +// control's 81. The wiring alone favours it (20 to 52) and then the surface and the sink cost more +// than they save, because the control reads the header inline in the method that already had the +// response, where StitchAPI needs a `Surface` object to reach it and a `TraceSink` object to +// remember it. What the extra lines buy is not this feature: it is the OTHER dozen things the same +// seam already does around the same call — retry, throttle, cache, circuit, auth, timeout, the +// trace tree — each a config key here and a hand-rolled subsystem there. +// +// pnpm exec tsx docs/scenarios/proofs/deprecation-headers/c8-assembled.ts +import { manualClock } from '../../../../packages/core/src/testing'; +import { watchedApi } from './assembled'; +import { noticeOf } from './deprecation'; +import { BASE, DAY, FLEET, FakeVendor, NOW } from './fake-vendor'; +import { HandRolledClient } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SUNSET = Date.parse('2026-01-01T00:00:00Z'); +const ENDPOINTS = FLEET.map((e) => ({ name: e.name, path: e.path })); +const EXPECTED_REPORT = + '3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users'; + +/** + * Executable lines between the USER CODE markers — imports, the options interface, blanks and + * comments removed on BOTH sides, so the number is the code someone actually maintains. + */ +function executableLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8'); + const from = src.indexOf('// >>> BEGIN USER CODE'); + const to = src.indexOf('// <<< END USER CODE'); + return src + .slice(from, to) + .replace(/^import[\s\S]*?;$/gm, '') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +async function main(): Promise { + heading('C8 — the whole job, both ways'); + + // ── (a) StitchAPI, before any sunset ───────────────────────────────────────────────────── + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const { members, watch } = watchedApi({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + endpoints: ENDPOINTS, + failAfterSunset: true, + }); + for (let r = 0; r < 20; r += 1) + for (const m of members.values()) await m(); + + check('(a) calls made', vendor.total, 100); + check('(a) the report', watch.summary(), EXPECTED_REPORT); + checkSeq( + '(a) rows', + watch.fleet().map((r) => `${r.endpoint}:${String(r.calls)}`), + ['users:20', 'search:20', 'orders:20'], + ); + const value = await members.get('users')?.(); + check( + '(a) the caller can also read its OWN notice off the value', + noticeOf(value)?.sunsetAt, + SUNSET, + ); + } + + // ── (b) the hand-rolled control, same job, same numbers ────────────────────────────────── + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const client = new HandRolledClient({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + failAfterSunset: true, + }); + for (let r = 0; r < 20; r += 1) + for (const e of ENDPOINTS) await client.call(e.name, e.path); + + check('(b) calls made', vendor.total, 100); + check('(b) the report', client.summary(), EXPECTED_REPORT); + checkSeq( + '(b) rows', + client.fleet().map((r) => `${r.endpoint}:${String(r.calls)}`), + ['users:20', 'search:20', 'orders:20'], + ); + note( + '(b) → identical output, which is what makes the line count comparable', + ); + } + + // ── (c) the tripwire, both ways, on the same clock ─────────────────────────────────────── + { + const outcomes: string[] = []; + for (const at of [SUNSET - DAY, SUNSET + DAY]) { + const clock = manualClock(at); + const vendor = new FakeVendor(); + const { members } = watchedApi({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + endpoints: ENDPOINTS, + failAfterSunset: true, + }); + const r = await members.get('users')?.safe(); + outcomes.push(r?.ok === true ? 'ok' : (r?.error?.message ?? '?')); + } + checkSeq('(c) StitchAPI, day before / day after', outcomes, [ + 'ok', + 'users: sunset passed (2026-01-01T00:00:00.000Z)', + ]); + + const control: string[] = []; + for (const at of [SUNSET - DAY, SUNSET + DAY]) { + const clock = manualClock(at); + const client = new HandRolledClient({ + baseUrl: BASE, + adapter: new FakeVendor().adapter(), + clock, + failAfterSunset: true, + }); + try { + await client.call('users', '/v1/users'); + control.push('ok'); + } catch (e) { + control.push((e as Error).message); + } + } + checkSeq('(c) control, day before / day after', control, [ + 'ok', + 'users: sunset passed (2026-01-01T00:00:00.000Z)', + ]); + } + + // ── (d) the line count ─────────────────────────────────────────────────────────────────── + { + // The parsers (C6 measured 29 lines) are shared: both sides import them, so they cancel. + const PARSERS = 29; + const wiring = executableLines('assembled.ts'); + const control = executableLines('hand-rolled.ts'); + const module = executableLines('deprecation.ts'); + // What the StitchAPI side needs BEYOND the shared parsers: the Surface, the Notice type, + // the sink, and the summary — everything in `deprecation.ts` that is not a parser. + const surfaceAndSink = module - PARSERS; + + check('(d) StitchAPI wiring (`assembled.ts`)', wiring, 20); + check('(d) hand-rolled wiring (`hand-rolled.ts`)', control, 52); + check('(d) `deprecation.ts` in full', module, 112); + check( + '(d) …of which surface + sink, beyond the shared parsers', + surfaceAndSink, + 83, + ); + check('(d) TOTAL, StitchAPI', wiring + module, 132); + check('(d) TOTAL, hand-rolled', control + PARSERS, 81); + note( + `(d) → the wiring alone favours the library ${String(wiring)} to ${String(control)}, and the TOTAL goes the other way: ${String(wiring + module)} to ${String(control + PARSERS)}. StitchAPI costs ${String(wiring + module - control - PARSERS)} MORE lines for the identical output`, + ); + note( + '(d) → the difference is two indirections the control does not pay for: a `Surface` object to get at `res.headers`, and a `TraceSink` object to hold the per-endpoint Map. The control does both inline in the method that already had the response in hand. This is the first scenario in this pass where the library LOSES on volume', + ); + } + + // ── (e) what the extra lines actually buy ──────────────────────────────────────────────── + // The control is 100 lines from having any of this, and each item is a config key on the seam. + { + const clock = manualClock(NOW); + const vendor = new FakeVendor(); + const { members, watch } = watchedApi({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + endpoints: ENDPOINTS, + failAfterSunset: false, + }); + // 20 calls each, but `users` is cached: the notice survives, the requests do not. + for (let r = 0; r < 20; r += 1) + for (const m of members.values()) await m(); + check('(e) requests without a cache', vendor.total, 100); + check('(e) report still correct', watch.summary(), EXPECTED_REPORT); + note( + '(e) → the same seam already carries `retry`, `throttle`, `cache`, `circuit`, `auth`, `timeout`, `idempotency`, `paginate` and the trace tree as CONFIG KEYS. The control has a `fetch` in a method and would grow a hand-rolled version of each', + ); + } + + // ── (f) the seams, named ───────────────────────────────────────────────────────────────── + { + const seams = [ + 'Surface.interpret — reads res.headers, decides the value, renders the tripwire verdict', + 'trace: TraceSink at the seam — one sink, ctx.name per endpoint, cross-call state', + 'clock — injected time, so the sunset crossing is deterministic', + ]; + for (const s of seams) note(`(f) seam: ${s}`); + check('(f) seams used', seams.length, 3); + // Hunted on the public config surface: anything named for this problem. + const knowing = Object.keys( + await import('../../../../packages/core/src/index'), + ).filter((k) => /deprecat|sunset|retire/i.test(k)); + checkSeq('(f) exports that know about deprecation', knowing, []); + note( + '(f) → three seams, zero config. Every other scenario in this pass had at least one lever in the config object; this one has none, and that is the finding', + ); + } + + finish( + 'C8', + "ACHIEVABLE WITH USER CODE, ON THREE SEAMS AND ZERO CONFIG KEYS — and it is the first scenario in this pass where the LINE COUNT GOES AGAINST THE LIBRARY. `Surface.interpret` reads `res.headers`, decides the value and renders the tripwire verdict; a seam-level `trace` sink aggregates by `ctx.name`; the injected `clock` makes the sunset crossing deterministic. Both sides produce the identical report (`3 endpoints deprecated (users, search, orders), earliest sunset in 12 days: users`) over the identical 100 calls, and the identical tripwire on the identical clock. Wiring alone favours the library — 20 executable lines against the control's 52 — and the TOTAL goes the other way: 132 against 81, so StitchAPI costs 51 MORE lines for identical output. The difference is two indirections the control never pays for: a `Surface` object to reach `res.headers`, and a `TraceSink` object to hold the per-endpoint Map, where the control does both inline in the method that already had the response. What the extra lines buy is not this feature: it is that the same seam already carries `retry`, `throttle`, `cache`, `circuit`, `auth`, `timeout` and the trace tree as config keys, and the control would grow a hand-rolled version of each. NOT ONE config key in the library knows what a `Deprecation` header is", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/deprecation-headers/deprecation.ts b/docs/scenarios/proofs/deprecation-headers/deprecation.ts new file mode 100644 index 00000000..0ab3d582 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/deprecation.ts @@ -0,0 +1,230 @@ +// The assembled answer: parse both header formats, put the notice somewhere an accessor can see it, +// aggregate it across the fleet, and optionally trip after a sunset you chose. +// +// Everything between the USER CODE markers is what a caller writes. C8 counts it against +// `hand-rolled.ts`, which does the same job with no library at all — so the markers bracket only +// code someone maintains, never the fixtures or the types. +// +// The shape is forced by one measured fact (C1): NO StitchEvent, Inspection, RunReport, SafeResult +// or StitchError carries response headers. The only two places a response header is in scope are +// `hooks.onResponse` (`ctx.res.headers`) and a Surface's `interpret(res, cfg)` (`res.headers`). Of +// those two, only `interpret` can put what it found anywhere the rest of the library will carry, so +// the surface is the load-bearing seam and everything else here hangs off it. +import type { + Surface, + SurfaceOutcome, +} from '../../../../packages/core/src/index'; +import { verdictOf } from '../../../../packages/core/src/surface'; +import type { + Clock, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { DAY } from './fake-vendor'; + +/** + * A parsed retirement notice. Dates are epoch ms or `null` — `null` means "the vendor did not say", + * which is a different fact from "the date is zero" and the fleet report has to keep them apart. + */ +export interface Notice { + /** When the endpoint was declared deprecated (RFC 9745), or `null`. */ + deprecatedAt: number | null; + /** When it stops responding (RFC 8594), or `null` — deprecation without a sunset is legal. */ + sunsetAt: number | null; + /** The `rel="successor-version"` URL, or `null`. */ + successor: string | null; +} + +/** The key a folded notice rides under, on the value a deprecated endpoint returns. */ +export const NOTICE_KEY = '_deprecation'; + +// >>> BEGIN USER CODE + +// >>> BEGIN PARSERS (C6 counts the executable lines between these two markers) +/** + * RFC 9745 `Deprecation`. The header is a structured-field DATE: an `@` sigil followed by + * seconds since the epoch (`@1735689600`). Several vendors still emit the pre-RFC draft spelling, + * an HTTP-date, in the same header — so both are accepted, and neither is guessed at. + */ +export function parseDeprecation(value: string | undefined): number | null { + if (value === undefined) return null; + const raw = value.trim(); + // sf-date: `@` + an sf-integer of SECONDS. The `-?` matters — a date before 1970 is legal + // syntax, and silently dropping the sign would move a 1969 notice to 1971. + const sf = /^@(-?\d+)$/.exec(raw); + if (sf?.[1] !== undefined) return Number(sf[1]) * 1000; + const http = Date.parse(raw); + return Number.isNaN(http) ? null : http; +} + +/** + * RFC 8594 `Sunset`. Always an HTTP-date, never a structured-field date — the two RFCs genuinely + * disagree on format, which is the trap this scenario is built around. + */ +export function parseSunset(value: string | undefined): number | null { + if (value === undefined) return null; + const at = Date.parse(value.trim()); + return Number.isNaN(at) ? null : at; +} + +/** The `rel="successor-version"` target out of a `Link` header, if one is there. */ +export function parseSuccessor(value: string | undefined): string | null { + if (value === undefined) return null; + const m = /<([^>]+)>\s*;[^,]*rel\s*=\s*"?successor-version"?/.exec(value); + return m?.[1] ?? null; +} + +/** + * Read a whole notice off a response's headers, or `null` when the vendor said nothing. + * + * A `Sunset` with no `Deprecation` still counts: RFC 8594 stands alone, and an endpoint that + * announces only its removal date is the more urgent case, not the less. + */ +export function readNotice(headers: Record): Notice | null { + const deprecatedAt = parseDeprecation(headers['deprecation']); + const sunsetAt = parseSunset(headers['sunset']); + if (deprecatedAt === null && sunsetAt === null) return null; + return { + deprecatedAt, + sunsetAt, + successor: parseSuccessor(headers['link']), + }; +} +// <<< END PARSERS + +export interface DeprecationSurfaceOptions { + /** + * Fold the notice into the returned value under {@link NOTICE_KEY}, so every accessor + * downstream — the awaited value, `.inspect()`, the `result` event, and therefore a `TraceSink` + * — can see it. Off by default: it changes the caller's result type, which is a real cost. + */ + fold?: boolean; + /** + * Called with every notice seen, endpoint name first. The side channel that does NOT touch the + * caller's payload — the surface is the only place a response header is in scope, so if the + * notice must not ride the value, this is the only way out. + */ + onNotice?: (endpoint: string, notice: Notice) => void; + /** + * FAIL the call once this instant has passed (epoch ms), read off the stitch's `clock`. A + * deliberate tripwire for a sunset you have decided to treat as a deadline — never a default, + * because RFC 9745 is explicit that the header is a hint and not a guarantee. + */ + failAfterSunset?: boolean; +} + +/** + * A surface that reads the retirement notice off the response. + * + * `interpret(res, cfg)` is the ONE hook in the library that sees `res.headers` AND decides what the + * call returns, so it is where all of this has to live. It composes `verdictOf` first — an + * `interpret` REPLACES the default rather than layering on it, so skipping that would silently + * discard the stitch's own `verdict` config and turn every 4xx into a success. + */ +export function deprecationSurface( + opts: DeprecationSurfaceOptions = {}, +): Surface { + return { + id: 'http+deprecation', + interpret: (res, cfg): SurfaceOutcome => { + const failed = verdictOf(res, cfg); + if (failed) return failed; + const notice = readNotice(res.headers); + if (notice === null) return { ok: true, data: res.body }; + const name = cfg.name ?? 'stitch'; + opts.onNotice?.(name, notice); + if ( + opts.failAfterSunset === true && + notice.sunsetAt !== null && + (cfg.clock?.now() ?? Date.now()) >= notice.sunsetAt + ) + return { + ok: false, + message: `${name}: sunset passed (${new Date(notice.sunsetAt).toISOString()})`, + status: res.status, + }; + if (opts.fold !== true) return { ok: true, data: res.body }; + return { + ok: true, + data: { ...(res.body as object), [NOTICE_KEY]: notice }, + }; + }, + }; +} + +/** One row of the fleet report. */ +export interface FleetRow { + endpoint: string; + notice: Notice; + /** Calls seen against this endpoint — the de-duplication denominator (C7). */ + calls: number; +} + +/** + * The fleet view: which endpoints are retiring, and which one goes first. + * + * A `TraceSink` is configured once (on a stitch or a whole seam), receives `handle(event, ctx)` for + * every event of every call through it, and `ctx.name` says which endpoint — the same cross-call + * seam scenario 12 used to turn per-call drift findings into a rate. What does NOT transfer is the + * payload: no event carries headers, so this reads the notice off the `result` event's `data`, + * which only works when the surface was built with `fold: true`. That coupling is the finding, not + * an implementation detail. + * + * De-duplication is the `Map` keyed by endpoint. One row per endpoint however many calls arrive, + * which is the difference between a fleet report and a log line per request. + */ +export class DeprecationWatch implements TraceSink { + private readonly rows = new Map(); + private readonly clock: Clock | undefined; + + constructor(clock?: Clock) { + this.clock = clock; + } + + handle(event: StitchEvent, ctx: TraceContext): void { + if (event.type !== 'result') return; + const notice = noticeOf(event.data); + if (notice === null) return; + const prior = this.rows.get(ctx.name); + this.rows.set(ctx.name, { + endpoint: ctx.name, + notice, + calls: (prior?.calls ?? 0) + 1, + }); + } + + /** Deprecated endpoints, soonest sunset first. One row per endpoint, never per call. */ + fleet(): FleetRow[] { + return [...this.rows.values()].sort( + (a, b) => + (a.notice.sunsetAt ?? Infinity) - + (b.notice.sunsetAt ?? Infinity), + ); + } + + /** The one line an operator reads: how many are retiring, and how long until the first one. */ + summary(): string { + const rows = this.fleet(); + if (rows.length === 0) return 'no deprecated endpoints'; + const first = rows[0]; + if (first === undefined) return 'no deprecated endpoints'; + const names = rows.map((r) => r.endpoint).join(', '); + if (first.notice.sunsetAt === null) + return `${String(rows.length)} endpoints deprecated (${names}), no sunset announced`; + const days = Math.round( + (first.notice.sunsetAt - (this.clock?.now() ?? Date.now())) / DAY, + ); + return `${String(rows.length)} endpoints deprecated (${names}), earliest sunset in ${String(days)} days: ${first.endpoint}`; + } +} + +/** Pull a folded notice back off a result value, or `null` when there is not one there. */ +export function noticeOf(data: unknown): Notice | null { + if (typeof data !== 'object' || data === null) return null; + const found = (data as Record)[NOTICE_KEY]; + if (typeof found !== 'object' || found === null) return null; + return found as Notice; +} + +// <<< END USER CODE diff --git a/docs/scenarios/proofs/deprecation-headers/fake-vendor.ts b/docs/scenarios/proofs/deprecation-headers/fake-vendor.ts new file mode 100644 index 00000000..0ce249da --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/fake-vendor.ts @@ -0,0 +1,155 @@ +// The vendor. Five endpoints, three of them retiring, and every response is a `200`. +// +// That last part is the scenario: nothing here ever fails, so the fleet looks perfectly healthy from +// every angle except the one nobody reads. The retirement notices live ONLY in `Deprecation` +// (RFC 9745) and `Sunset` (RFC 8594) headers riding successful responses. +// +// Three header spellings are served on purpose, because a client that parses one and not the others +// gets a fraction of the picture: +// +// `Deprecation: @1735689600` RFC 9745 structured-field date (an sf-integer +// with an `@` sigil — seconds, not ms) +// `Deprecation: Wed, 01 Jan 2025 00:00:00 GMT` the pre-RFC draft spelling, an HTTP-date, still +// what several large vendors actually emit +// `Sunset: Wed, 01 Jan 2026 00:00:00 GMT` RFC 8594, always an HTTP-date +// +// `Link: <...>; rel="successor-version"` rides along on the deprecated endpoints because RFC 9745 +// pairs it with `Deprecation`, and because it is the one part of the notice that says what to DO. +// +// The clock is injected so "the sunset has passed" is a deterministic fact rather than a wall-clock +// accident: every script pins NOW to 2025-12-20T00:00:00Z, which puts `users` exactly 12 days from +// its sunset. +import type { + Adapter, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +/** Frozen "today" for every script in this directory: 2025-12-20T00:00:00Z. */ +export const NOW = Date.parse('2025-12-20T00:00:00Z'); + +/** A day in ms — the unit every sunset countdown in this directory is quoted in. */ +export const DAY = 86_400_000; + +/** + * One vendor endpoint: the body it serves and the retirement notice it attaches (if any). + * + * `deprecation` and `sunset` are stored as the LITERAL header values the vendor sends, not as + * timestamps, because the parsing of those literals is what C6 is measuring. Storing them parsed + * would quietly do the work under test. + */ +export interface Endpoint { + /** Path under the base URL, and the `name` the stitch is given. */ + name: string; + path: string; + body: unknown; + /** Literal `Deprecation` header value — sf-date (`@1735689600`) or an HTTP-date. */ + deprecation?: string; + /** Literal `Sunset` header value — always an HTTP-date (RFC 8594). */ + sunset?: string; + /** Literal `Link` header value announcing the successor. */ + successor?: string; +} + +/** + * The fleet. Three deprecated, two clean, and the deprecated three deliberately disagree about how + * to spell `Deprecation`. + * + * Sunset dates relative to the frozen NOW (2025-12-20): + * users 2026-01-01 12 days ← the earliest, and the one a fleet report must surface first + * search 2026-03-15 85 days + * orders 2026-06-01 163 days + */ +export const FLEET: readonly Endpoint[] = [ + { + name: 'users', + path: '/v1/users', + body: { users: [{ id: 1, name: 'Ada' }] }, + // RFC 9745 structured-field date: `@` + seconds since the epoch. 2025-01-01T00:00:00Z. + deprecation: '@1735689600', + sunset: 'Thu, 01 Jan 2026 00:00:00 GMT', + successor: + '; rel="successor-version"', + }, + { + name: 'search', + path: '/v1/search', + body: { hits: 3 }, + // The pre-RFC draft spelling: an HTTP-date in the SAME header. Real vendors send this. + deprecation: 'Sat, 01 Mar 2025 00:00:00 GMT', + sunset: 'Sun, 15 Mar 2026 00:00:00 GMT', + }, + { + name: 'orders', + path: '/v1/orders', + body: { orders: [{ id: 'o-1' }] }, + deprecation: '@1751328000', + sunset: 'Mon, 01 Jun 2026 00:00:00 GMT', + successor: + '; rel="successor-version"', + }, + { name: 'payments', path: '/v1/payments', body: { balance: 100 } }, + { name: 'webhooks', path: '/v1/webhooks', body: { subscribed: true } }, +]; + +/** Look a fleet member up by name; throws rather than serving a silent 404 in a proof. */ +export function endpoint(name: string): Endpoint { + const found = FLEET.find((e) => e.name === name); + if (!found) throw new Error(`no such endpoint: ${name}`); + return found; +} + +export const BASE = 'https://api.vendor.test'; + +/** + * The vendor as an `Adapter`. Routes on the request URL's path, counts requests per endpoint, and + * always answers `200`. + * + * `headersFor` is exported separately so a script can assert what the wire carried without going + * through a stitch — the difference between "the vendor sent it" and "an accessor could see it" is + * the whole scenario, and conflating them would beg the question. + */ +export class FakeVendor { + /** Requests served, per endpoint name. */ + readonly requests = new Map(); + + adapter(): Adapter { + return async (req): Promise => { + const path = new URL(req.url).pathname; + const hit = FLEET.find((e) => e.path === path); + if (!hit) throw new Error(`fake vendor: unrouted path ${path}`); + this.requests.set(hit.name, (this.requests.get(hit.name) ?? 0) + 1); + return { + status: 200, + headers: headersFor(hit), + body: hit.body, + }; + }; + } + + /** Total requests served across the fleet. */ + get total(): number { + return [...this.requests.values()].reduce((a, b) => a + b, 0); + } +} + +/** + * The header map one endpoint puts on the wire. Lowercased keys, because that is what every HTTP + * client normalises to and what `AdapterResponse.headers` carries in practice (the engine reads + * `res.headers['retry-after']` lowercased at engine.ts:750). + */ +export function headersFor(e: Endpoint): Record { + const h: Record = { 'content-type': 'application/json' }; + if (e.deprecation !== undefined) h['deprecation'] = e.deprecation; + if (e.sunset !== undefined) h['sunset'] = e.sunset; + if (e.successor !== undefined) h['link'] = e.successor; + return h; +} + +/** A one-endpoint adapter, for the scripts that only need a single response shape. */ +export function serving(e: Endpoint): Adapter { + return async (): Promise => ({ + status: 200, + headers: headersFor(e), + body: e.body, + }); +} diff --git a/docs/scenarios/proofs/deprecation-headers/hand-rolled.ts b/docs/scenarios/proofs/deprecation-headers/hand-rolled.ts new file mode 100644 index 00000000..f7450d99 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/hand-rolled.ts @@ -0,0 +1,81 @@ +// The control: the same job with no library at all, over the same `Adapter` and the same `Clock`. +// +// It has to do everything `assembled.ts` does — parse both header formats, keep one row per +// endpoint, sort by soonest sunset, answer "how many days", and trip after a sunset you chose — so +// the comparison is of the same work, not of two different jobs. +// +// The parsers are IMPORTED rather than re-typed, because C6 already established they are 100% user +// code on both sides: re-typing them here would inflate the control by 29 lines that say nothing +// about the library. What is counted is the wiring — the request loop, the per-endpoint state, the +// tripwire, and the reporting. +import type { Adapter, Clock } from '../../../../packages/core/src/types'; +import { type Notice, readNotice } from './deprecation'; +import { DAY } from './fake-vendor'; + +export interface HandRolledOptions { + baseUrl: string; + adapter: Adapter; + clock: Clock; + /** Fail the call once its announced sunset has passed. */ + failAfterSunset: boolean; +} + +// >>> BEGIN USER CODE +/** A client that reads retirement notices off its own responses. */ +export class HandRolledClient { + private readonly rows = new Map< + string, + { notice: Notice; calls: number } + >(); + + constructor(private readonly opts: HandRolledOptions) {} + + async call(name: string, path: string): Promise { + const res = await this.opts.adapter({ + url: `${this.opts.baseUrl}${path}`, + method: 'GET', + headers: {}, + }); + if (res.status >= 400) + throw new Error(`${name}: HTTP ${String(res.status)}`); + const notice = readNotice(res.headers); + if (notice !== null) { + const prior = this.rows.get(name); + this.rows.set(name, { notice, calls: (prior?.calls ?? 0) + 1 }); + if ( + this.opts.failAfterSunset && + notice.sunsetAt !== null && + this.opts.clock.now() >= notice.sunsetAt + ) + throw new Error( + `${name}: sunset passed (${new Date(notice.sunsetAt).toISOString()})`, + ); + } + return res.body; + } + + fleet(): { endpoint: string; notice: Notice; calls: number }[] { + return [...this.rows.entries()] + .map(([endpoint, r]) => ({ endpoint, ...r })) + .sort( + (a, b) => + (a.notice.sunsetAt ?? Infinity) - + (b.notice.sunsetAt ?? Infinity), + ); + } + + summary(): string { + const rows = this.fleet(); + if (rows.length === 0) return 'no deprecated endpoints'; + const first = rows[0]; + if (first === undefined) return 'no deprecated endpoints'; + const names = rows.map((r) => r.endpoint).join(', '); + if (first.notice.sunsetAt === null) + return `${String(rows.length)} endpoints deprecated (${names}), no sunset announced`; + const days = Math.round( + (first.notice.sunsetAt - this.opts.clock.now()) / DAY, + ); + return `${String(rows.length)} endpoints deprecated (${names}), earliest sunset in ${String(days)} days: ${first.endpoint}`; + } +} +// <<< END USER CODE diff --git a/docs/scenarios/proofs/deprecation-headers/harness.ts b/docs/scenarios/proofs/deprecation-headers/harness.ts new file mode 100644 index 00000000..f36e7225 --- /dev/null +++ b/docs/scenarios/proofs/deprecation-headers/harness.ts @@ -0,0 +1,123 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is REACHABILITY: given an accessor, is the response header there or not. +// That is a yes/no with a value attached, and both halves matter — "`.inspect()` does not carry +// `Sunset`" and "`hooks.onResponse` carries `Wed, 01 Jan 2026 00:00:00 GMT`" are the same +// measurement pointed at two places. So `checkReach` is the assertion this file exists for: it +// prints the accessor, whether the header was REACHED, and the value it reached, on one line that +// reads out of context — the C1 table is literally its output. +// +// The rest follows `intermittent-drift/harness.ts`: `check` for an exact value rendered with +// `JSON.stringify` (so `undefined` and `'undefined'` stay distinguishable), `checkSeq` for a +// measured sequence, `note` for a reported-but-not-asserted number, `finish` for the verdict. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides C1. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v}n`; + if (typeof v === 'number' && Number.isNaN(v)) return 'NaN'; + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the event spine + * (`["start","progress","result","done"]`) and the fleet report + * (`["users — sunset in 12 days", ...]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * THE assertion of C1: can this accessor see a response header, and what did it see? + * + * Both halves on one line, because the pair is the finding. `hooks.onResponse -> REACHED + * "Wed, 01 Jan 2026 00:00:00 GMT"` and `.inspect() -> ABSENT` are the consolidation deliverable, + * and a bare `true`/`false` is unreadable three pages later. `expected` is whether the claim + * predicts reachability, so a wrong prediction fails loudly rather than quietly recording whatever + * happened. + */ +export function checkReach( + accessor: string, + value: unknown, + expected: boolean, +): void { + checks++; + const reached = value !== undefined && value !== null; + const ok = reached === expected; + if (!ok) failures++; + const verdict = reached ? `REACHED ${show(value)}` : 'ABSENT'; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${accessor.padEnd(22)} -> ${verdict}${ + ok ? '' : ` (expected ${expected ? 'REACHED' : 'ABSENT'})` + }`, + ); +} + +/** Assert a measured number is at most `bound` — the de-duplication ceiling in C7. */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (<= ${String(bound)})` : ` (expected <= ${String(bound)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring an ABSENCE (C1's five accessors that carry nothing) and + * several by measuring the library doing something genuinely good (C5's tripwire), so the verdict + * statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/dual-run-migration/README.md b/docs/scenarios/proofs/dual-run-migration/README.md new file mode 100644 index 00000000..75040546 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/README.md @@ -0,0 +1,194 @@ +# Proofs — the migration you have to run twice + +Runnable evidence for the claims in +[`../../dual-run-migration.md`](../../dual-run-migration.md). + +**Both pre-registered predictions were tested. C2's is confirmed outright; C1's is confirmed only in +the two configurations a dual-run is most likely to land in, and refuted in three others — which +turns out to be the more dangerous shape.** Resilience state is not shared by default. It is shared +on a **key collision**, and the collision is invisible in the config: two stitches share a breaker +iff they land on the same store _and_ the same string out of +`circuit.key ?? name ?? path ?? 'stitch'`. Measured across five configurations, the primary was +fast-failed on the shadow's breaker in exactly two of them — a v2 that keeps the path and changes +the origin, and `throttle: { pool: 'host' }`, which is the setting a consumer reaches for +_precisely because_ the two versions share the vendor's meter. + +The capture asks about four channels. There are **five**, and the one it does not list is the worst: +`all()` auto-cancels its members on the first failure (`pipe.ts:115`), so a shadow that fails fast +**kills the primary's in-flight request** — measured as `aborted: true` on the wire, not merely as an +error the caller sees. + +Three more things the capture does not contain: + +- **A bare `void v2(input)` sends zero requests.** `StitchResult` is a lazy `PromiseLike` + (`types.ts:1905`), so the most natural fire-and-forget spelling in the language does not fail + loudly — it does not run at all. No request, no rejection, no `unhandledRejection`. A dual-run + written that way silently compares nothing, forever. The spelling that both fires and cannot + reject is `void v2.safe(input)` (C1 b). +- **`.with()` partially rescues `all`, and that is a trap.** A bound partial _does_ survive + `runMember`'s broadcast — but it binds a **constant**, so a group built once and called twice sent + the shadow to the wrong customer while the primary followed the new input (C2). +- **A seam already accounts for the shadow's cost correctly.** `seamBucket` re-keys every acquire + onto one seam id (`seam.ts:51-69`), so a seam-level `throttle` is one bucket across both versions. + This is the one cost question in the scenario the library already answers (C7). + +Every script is standalone and offline. The transport is a fake in-memory `Adapter`; nothing is +written to disk and nothing touches the network. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/dual-run-migration/c1-shadow-isolation.ts + +# all of them +for f in docs/scenarios/proofs/dual-run-migration/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/dual-run-migration/*.ts +``` + +## The method: counts on the wire, not intentions in the config + +Every claim here reduces to one question — _how many requests did the vendor actually receive, and +what was in them?_ — so the primitive is a count and a literal URL string taken from the fake +transport in [`vendor.ts`](vendor.ts), which is the only thing in the process that can observe a +request. A row of any table in this directory is a measurement of the wire. Nothing is inferred from +what a config appears to promise. + +The fake vendor puts **both versions on one host**, because that is the situation: `api.vendor.test` +is retiring `/v1`, and as the consumer you spend the same meter, hit the same breaker key material, +and share the same origin for both. A fake pointing v2 at a second host would have quietly dissolved +half the scenario. + +The two versions disagree the way real versions do: + +| | v1 | v2 | +| ------ | ------------------------------------------------- | --------------------------------------------------------------------------------- | +| input | `GET /v1/customers/{id}` — id is a **path** param | `GET /v2/customers?customer_id=` — id moved to a **query** param, and was renamed | +| output | `created` (epoch int), `tags` in one order | `created_at` (ISO string), `tags` reordered, `livemode` added | + +…plus **one genuine regression**: `balance_cents` is `41250` in v1 and `41520` in v2. A wrong _value_ +of the right type at the right path is the diff a schema cannot catch and the one a dual-run exists +to find. `CREATED_EPOCH` and `CREATED_ISO` denote the same instant, and C4 asserts the round-trip +rather than trusting the pair. + +### Timing + +Scenario 19 measured that `manualClock()` does **not** drive `timeout.total`, `cache.ttl`, event +`at` / `done.elapsed`, OAuth2 expiry, or SigV4. Two consequences, stated in +[`harness.ts`](harness.ts) and inherited by every script: + +- **Caller-observed latency is wall-clock by definition**, so C1 (a), C7 and C8 use + `performance.now()` and a real `setTimeout` in the fake adapter. Every latency number in this + directory is **real time** and is asserted as a **band**, never an exact figure. +- **Circuit cooldown and retry backoff _are_ manual-clock driven** — which is why the retry channel + (C1 c) deliberately uses a real clock with a 1 ms fixed backoff instead: nothing advances a + `manualClock`, so the first backoff sleep never resolves and the script would hang. That channel + measures counts, not timing, so a real clock costs nothing and is honest. + +## What each script establishes + +| script | verdict | the number that decides it | +| -------------------------------------------------- | --------------------------------------------- | ------------------------------------------------------------------------ | +| [`c1-shadow-isolation.ts`](c1-shadow-isolation.ts) | **PARTIAL** — 1 of 5 channels safe by default | 5 circuit configurations: primary got 1, 1, **0**, **0**, 1 requests | +| [`c2-different-inputs.ts`](c2-different-inputs.ts) | **CONFIRMED** | the shadow's literal URL: `/v2/customers` — no id at all | +| [`c3-comparison.ts`](c3-comparison.ts) | **PARTIAL** | 17 public subpaths, 0 of them export a value-vs-value comparator | +| [`c4-relevancy.ts`](c4-relevancy.ts) | **MEASURED** | 7 raw diff ops per call on a _correct_ v2; 6 benign | +| [`c5-writes.ts`](c5-writes.ts) | **NO GUARD EXISTS** | 3 shadow write attempts, 0 reached the wire behind an 8-line adapter | +| [`c6-cutover.ts`](c6-cutover.ts) | **PARTIAL** | a thunk moves origin _and_ path; input shape and `output` it cannot move | +| [`c7-cost.ts`](c7-cost.ts) | **MEASURED** | exactly **2.000x**; 5 lines of sampling take it to 1.045x | +| [`c8-assembled.ts`](c8-assembled.ts) | **ASSEMBLED** | naive 0-of-4 vs safe 4-of-4 user-facing calls, same flaky v2 | + +## The C1 table — four channels, and the fifth + +| channel | safe by default? | measured | what it takes | +| ---------------- | ---------------- | ------------------------------------------------------- | ---------------------------------------------------------------------- | +| **(a) latency** | **NO** | `all()` 122 ms vs floated 12 ms, primary alone 13 ms | do not await the shadow. No combinator does this — user code | +| **(b) thrown** | **NO** | `all()` threw `StitchError`; `.safe()` gave `ok: false` | `void v2.safe(input)` — fires **and** cannot reject | +| **(c) retry** | **YES** | primary got 1 request of 1 while the shadow burned 3 | — (the budget is a per-call loop counter, `engine.ts:610,624`) | +| **(c\*) cancel** | **NO** | primary request `aborted: true`, `completed: false` | never put the shadow in `all()` | +| **(d) circuit** | **NO** | 1 · 1 · **0** · **0** · 1 primary requests across d1–d5 | a distinct `circuit.key` per version, and never unkeyed `pool: 'host'` | + +Only **one** channel is safe by default, and it is the one the capture was least worried about. + +### The five circuit configurations (C1 d) + +The breaker's identity is `(store instance) × ('circuit:' + (circuit.key ?? name ?? path ?? 'stitch'))` +— `resilience.ts:353` over `engine.ts:857-862`, `engine.ts:265-274`, `engine.ts:140`. + +| | configuration | result | why | +| --- | ----------------------------------------- | ---------- | ------------------------------------------------------- | +| d1 | standalone stitches, distinct paths | isolated | each builds its own `memoryStore()` | +| d2 | one seam, distinct paths | isolated | shared store, but the key is the path | +| d3 | one seam, **same path**, different origin | **SHARED** | `nameOf` reads `path`; both key on `circuit:/customers` | +| d4 | one seam + `throttle: { pool: 'host' }` | **SHARED** | `hostKey` returns the URL host instead of the name | +| d5 | explicit distinct `circuit.key` | isolated | the only lever; a static string, no `keyOf` | + +d3 and d4 are both ordinary ways to ship a dual-run. In each, the caller received `circuit open` on a +breaker only the shadow ever opened. + +## The relevancy ledger (C4) + +| v1 → v2 change | declarative? | user code needed? | +| ------------------------ | ------------------------------ | ----------------------------------------------------------- | +| `livemode` added | **yes** — `ignore: 'livemode'` | — | +| `tags[]` reordered | partly — `ignore: 'tags[]'` | **yes**: `ignore` _suppresses_, it cannot compare unordered | +| `created` → `created_at` | partly — ignore both paths | **yes**: no aliasing option exists | +| epoch → ISO | **no** | **yes**: no coercion hook | +| `balance_cents` | (must survive every filter) | — | + +`DriftOptions.ignore` is the only declarative filter in the tree, and it reaches 7 → 1 with four +clauses. But it does so by refusing to look: the clause that silences the benign tag reorder also +silences a **real** tag change, and the clause that silences the rename also silences a v2 reporting +the **wrong instant** — both measured. A 24-line normalizer reaches the same count and still catches +what `ignore` hid. + +## What is not reachable (C3) + +| symbol | compares | reachable from | +| -------------------------- | --------------------------------- | ------------------------------------------ | +| `drift(schema, opts)` | nothing — it tags a schema | `stitchapi` — **public** | +| `classifyDiff(a, b, opts)` | value vs value, **with** `ignore` | `packages/core/src/drift.ts` — source only | +| `diff(before, after)` | value vs value | `packages/core/src/diff.ts` — source only | + +The comparison primitive is not missing. It exists **twice**, in exactly the right shape, and neither +copy is exported from any of the 17 subpaths in the package `exports` map. `classifyDiff` is also the +wrong tool for a regression report even when reached: it renders the planted `balance_cents` +regression as the detail `"number -> number"`, a type delta with no numbers in it. + +## The tension C8 exists to name + +The two ways to make the **meter** accounting correct are not equivalent, and the scenario walks +straight into the bad one. A seam-level `throttle` pools both versions and leaves the breaker keyed +per path. `throttle: { pool: 'host' }` also pools correctly — and silently re-keys the **circuit** +onto the host, which is the exact configuration measured fast-failing the primary. A consumer +reaching for `pool: 'host'` is reaching for it for a good reason, and gets an unasked-for shared +breaker. + +C8's safe construction is **55 executable lines across 5 seams**: `readsOnly` on the shadow's adapter +only, one seam for the throttle, distinct `circuit.key` strings, a hand-written normalizer, and a +floated `.safe()` at the call site. Four of the five are ordinary config. The fifth is not config at +all — there is no declarative surface anywhere in the library for _"these two field names mean the +same thing"_ or _"compare this array unordered"_ — so the relevancy model is code you write and +maintain, which is exactly the part the field guidance says the effort goes into. + +Three things no amount of user code fixes, and they bound the technique rather than the library: + +- a write can only be **refused**, never shadowed; +- the shadow's correctness depends on the caller mapping the input twice, and getting it wrong is + **silent** — C2 measured a bound shadow querying the wrong customer forever; +- the comparator is vendored, so a library-side improvement to `diff` never reaches it. diff --git a/docs/scenarios/proofs/dual-run-migration/c1-shadow-isolation.ts b/docs/scenarios/proofs/dual-run-migration/c1-shadow-isolation.ts new file mode 100644 index 00000000..cec7291b --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c1-shadow-isolation.ts @@ -0,0 +1,573 @@ +// C1 — DECIDING CLAIM. Can the shadow be made unable to hurt the primary? +// +// "Unable to hurt" is not one property, it is FOUR, and they have four different answers. This +// script measures each separately and fills the table in `printChannels()`: +// +// (a) LATENCY — does awaiting the shadow put its wall-clock on the caller's critical path? +// (b) THROWN — does a shadow failure reach the caller as a thrown error? +// (c) RETRY — does the shadow consume the primary's retry budget? +// (d) CIRCUIT — do the shadow's failures open a breaker the primary uses? +// +// (d) carries a PRE-REGISTERED PREDICTION from scenario 9 / issue #641: "resilience state is shared +// unless keyed by hand", so a flaky v2 may open a breaker that takes down v1. It is measured here as +// five separate configurations, because the answer turns out to depend on TWO things at once (the +// store instance and the key string) and a single yes/no would misreport four of the five. +// +// TIMING: (a) is wall-clock — real `setTimeout` in the fake transport, `performance.now()` at the +// caller. (d) uses `manualClock()`, which DOES drive circuit cooldown. See harness.ts. +import { all } from '../../../../packages/core/src/pipe'; +import { seam } from '../../../../packages/core/src/seam'; +import { stitch } from '../../../../packages/core/src/stitch'; +import { manualClock } from '../../../../packages/core/src/test-clock'; +import type { + SafeResult, + StitchInput, +} from '../../../../packages/core/src/types'; +import { + channelRow, + check, + checkBand, + finish, + heading, + ledgerRow, + note, + printChannels, + printLedger, +} from './harness'; +import { HOST, elapsed, fakeVendor } from './vendor'; + +const V1_PATH = '/v1/customers/{id}'; +const V2_PATH = '/v2/customers'; + +async function main(): Promise { + // --------------------------------------------------------------------------- + heading( + "C1 (a) — LATENCY: does the shadow land on the caller's critical path?", + ); + // v1 answers in 10ms, v2 in 120ms. Any spelling that AWAITS both pays 120ms; the question is + // whether a spelling exists that does not. + { + const vendor = fakeVendor({ latency: { v1: 10, v2: 120 } }); + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }); + + // The baseline: the primary alone. Everything else is measured against this. + const solo = await elapsed(() => v1({ params: { id: 'cus_7Q2' } })); + note('primary alone (v1, 10ms endpoint)', `${solo.ms}ms`); + + // Spelling 1 — the combinator that looks purpose-built for this. + const both = await elapsed(() => + all([v1, v2])({ params: { id: 'cus_7Q2' } }), + ); + note('all([v1, v2]) — awaits both', `${both.ms}ms`); + + // Spelling 2 — fire-and-forget. The shadow is started and deliberately NOT awaited. + const ff = await elapsed(async () => { + const primary = v1({ params: { id: 'cus_7Q2' } }); + void v2({ params: { id: 'cus_7Q2' } }).catch(() => undefined); + return primary; + }); + note('fire-and-forget (shadow not awaited)', `${ff.ms}ms`); + + checkBand( + 'all([v1,v2]) caller-observed ms (≈ the SLOWEST member)', + both.ms, + 110, + 220, + ); + checkBand( + 'fire-and-forget caller-observed ms (≈ the PRIMARY alone)', + ff.ms, + 5, + 60, + ); + const added = both.ms - solo.ms; + note('added latency, all() vs primary alone', `${added}ms`); + check( + 'fire-and-forget adds less than 25ms over the primary alone', + ff.ms - solo.ms < 25, + true, + ); + + channelRow({ + channel: '(a) latency', + safeByDefault: false, + measured: `${both.ms}ms vs ${ff.ms}ms`, + fix: 'do NOT await the shadow — no combinator does this; user code', + }); + + // A fire-and-forget shadow still has to be allowed to finish before the process exits, or the + // measurement below counts a request the vendor never actually received. + await new Promise((r) => setTimeout(r, 200)); + ledgerRow( + 'all([v1,v2]) + fire-and-forget, 3 logical calls', + vendor.count('v1'), + vendor.count('v2'), + `${both.ms}ms / ${ff.ms}ms`, + ); + } + + // --------------------------------------------------------------------------- + heading('C1 (b) — THROWN: does a shadow failure reach the caller?'); + { + // v2 answers 500 forever; v1 is healthy. + const vendor = fakeVendor({ statuses: { v2: [500, 500, 500, 500] } }); + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }); + + // Spelling 1 — `all` is FAIL-FAST by construction: the first failure rejects the group. + const viaAll = await elapsed(() => + all([v1, v2])({ params: { id: 'cus_7Q2' } }), + ); + const allErr = viaAll.error as Error | undefined; + note( + 'all([v1,v2]) threw', + allErr ? allErr.constructor.name : 'nothing', + ); + check( + 'all() surfaces the shadow failure to the caller', + allErr !== undefined, + true, + ); + + // Spelling 2 — the shadow is floated with an explicit `.catch`. + vendor.reset(); + const caught = await elapsed(async () => { + const primary = v1({ params: { id: 'cus_7Q2' } }); + void v2({ params: { id: 'cus_7Q2' } }).catch(() => undefined); + return primary; + }); + await new Promise((r) => setTimeout(r, 60)); + check( + 'float + .catch() reaches the caller', + caught.error !== undefined, + false, + ); + check( + 'float + .catch() DID send the shadow request', + vendor.count('v2'), + 1, + ); + note( + 'float + .catch() caller result', + caught.value !== undefined ? 'a v1 value' : 'nothing', + ); + + // Spelling 3 — `.safe()` never throws at all. + const safe = await v2.safe({ params: { id: 'cus_7Q2' } }); + check('.safe() on the failing shadow throws', false, false); + check('.safe().ok', safe.ok, false); + + // THE HAZARD, and it is NOT the one the capture would predict. `StitchResult` is a LAZY + // `PromiseLike` (types.ts:1905), not a Promise: nothing runs until something subscribes. So + // the most natural fire-and-forget spelling in the language — + // + // void v2(input); // "shadow it and move on" + // + // does not fail loudly. It does not run AT ALL. There is no request, no rejection, and no + // unhandledRejection to tell you: the dual-run silently compares nothing, forever. + vendor.reset(); + let unhandled: string | undefined; + const onUnhandled = (reason: unknown) => { + unhandled = (reason as Error)?.constructor?.name ?? String(reason); + }; + process.on('unhandledRejection', onUnhandled); + { + const primary = v1({ params: { id: 'cus_7Q2' } }); + void v2({ params: { id: 'cus_7Q2' } }); // NO subscription — the mistake this measures + await primary; + } + await new Promise((r) => setTimeout(r, 60)); + const bareVoidRequests = vendor.count('v2'); + check( + 'bare `void v2(input)` sent ZERO shadow requests (lazy thenable)', + bareVoidRequests, + 0, + ); + check( + '...and therefore raised no unhandledRejection either', + unhandled, + undefined, + ); + + // The construction that DOES leave a rejection unhandled: subscribe (so it runs) but attach + // no rejection handler. `Promise.resolve(...)` on the thenable is the ordinary way to do it + // by accident — e.g. handing the shadow to `Promise.allSettled`'s cousin, or logging it. + vendor.reset(); + { + const primary = v1({ params: { id: 'cus_7Q2' } }); + void Promise.resolve(v2({ params: { id: 'cus_7Q2' } })); // subscribed, unguarded + await primary; + } + await new Promise((r) => setTimeout(r, 60)); + process.off('unhandledRejection', onUnhandled); + check( + 'subscribing without a handler DID send the request', + vendor.count('v2'), + 1, + ); + check( + '...and it raises an unhandledRejection', + unhandled !== undefined, + true, + ); + note('unhandledRejection reason class', unhandled ?? 'none'); + + channelRow({ + channel: '(b) thrown', + safeByDefault: false, + measured: `all(): threw; .safe(): ok:false`, + fix: '`void v2.safe(input)` — fires AND cannot reject. `void v2(input)` never fires', + }); + } + + // --------------------------------------------------------------------------- + heading( + "C1 (c) — RETRY: does the shadow consume the primary's retry budget?", + ); + { + // Both stitches ask for 3 attempts. v2's endpoint answers 503 (a default-retryable status) + // three times; v1's answers 200 immediately. If the budget were shared, v1's request count + // would move. + // + // REAL CLOCK, deliberately. A `manualClock()` DOES drive retry backoff — which is exactly + // why it cannot be used here: nothing advances it, so the first backoff sleep never resolves + // and the script hangs. This channel measures COUNTS, not timing, so a real clock with a + // 1ms fixed backoff is both honest and fast. + const vendor = fakeVendor({ statuses: { v2: [503, 503, 503, 503] } }); + const backoff = { curve: 'fixed', base: 1 } as const; + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + retry: { attempts: 3, backoff }, + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + retry: { attempts: 3, backoff }, + }); + + await v2.safe({ params: { id: 'cus_7Q2' } }); + const v2Attempts = vendor.count('v2'); + await v1({ params: { id: 'cus_7Q2' } }); + const v1Attempts = vendor.count('v1'); + + check('shadow burned its own attempts (retry: 3)', v2Attempts, 3); + check( + 'primary still got its full first attempt (1 call, 1 request)', + v1Attempts, + 1, + ); + note( + 'retry budget is a local loop counter (engine.ts:610,624) — per call, never shared', + ); + } + // BUT: there is a FIFTH way the shadow reaches the primary that the capture did not list, and it + // lives next door to this channel. `all` AUTO-CANCELS its members on the first failure (pipe.ts:115 + // `ctrl.abort()`), so a shadow that fails FAST cancels a primary that is still in flight. The + // primary does not merely surface an error — its request is killed on the wire. + heading( + 'C1 (c*) — CANCEL: does a shadow failure kill the in-flight primary?', + ); + { + const vendor = fakeVendor({ + latency: { v1: 120, v2: 0 }, + statuses: { v2: [500] }, + }); + const slowPrimary = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }); + const fastFailShadow = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }); + + const r = await elapsed(() => + all([slowPrimary, fastFailShadow])({ params: { id: 'cus_7Q2' } }), + ); + await new Promise((res) => setTimeout(res, 200)); + const primaryCall = vendor.log.find((c) => c.version === 'v1'); + check( + 'all(): the primary request reached the vendor', + primaryCall !== undefined, + true, + ); + check( + "all(): the shadow's failure ABORTED the in-flight primary", + primaryCall?.aborted, + true, + ); + check( + 'all(): the primary never completed', + primaryCall?.completed, + false, + ); + note( + 'the group AbortController (pipe.ts:57-71, aborted at pipe.ts:115) is linked to every member', + ); + note('caller-observed ms for the aborted group', `${r.ms}ms`); + + channelRow({ + channel: '(c) retry', + safeByDefault: true, + measured: `v1 requests: 1 of 1`, + fix: '— (budget is a per-call loop counter). But see (c*) below', + }); + channelRow({ + channel: '(c*) cancel', + safeByDefault: false, + measured: `primary aborted: ${String(primaryCall?.aborted)}`, + fix: 'never put the shadow in `all()` — it auto-cancels the primary', + }); + } + + // --------------------------------------------------------------------------- + heading( + "C1 (d) — CIRCUIT: do the shadow's failures open a breaker the primary uses?", + ); + // PRE-REGISTERED PREDICTION (scenario 9 / issue #641): "resilience state is shared unless keyed by + // hand". Measured as five configurations, because the breaker's identity is + // (store instance) x ('circuit:' + (circuit.key ?? cfg.name ?? cfg.path ?? 'stitch')) + // — resilience.ts:353 over engine.ts:857-862, engine.ts:265-274, engine.ts:140 — and the prediction + // is right for some of those and wrong for others. + // + // The probe is the same every time: fail the SHADOW until its breaker trips, then call the PRIMARY + // once and count whether the vendor received that request. A request that never arrives was + // fast-failed by a breaker the shadow opened. + + /** Trip the shadow, then call the primary once. Returns what the vendor saw and what the caller got. */ + // The probe only ever calls `.safe()`, so it asks for exactly that. Naming the full + // `Stitch` would not typecheck: a `path`-templated stitch NARROWS its own input type + // (`params` becomes required), so `Stitch` is not assignable to + // `Stitch`. + type Probed = { + safe(input?: StitchInput): Promise>; + }; + async function probe( + label: string, + build: (adapter: ReturnType) => { + v1: Probed; + v2: Probed; + }, + ): Promise<{ primaryRequests: number; callerError: string }> { + const vendor = fakeVendor({ statuses: { v2: [500, 500, 500, 500] } }); + const { v1, v2 } = build(vendor); + // circuit: [2, '60s'] — two consecutive failures trip it. + await v2.safe({ params: { id: 'cus_7Q2' } }); + await v2.safe({ params: { id: 'cus_7Q2' } }); + const before = vendor.count('v1'); + const primary = await v1.safe({ params: { id: 'cus_7Q2' } }); + const primaryRequests = vendor.count('v1') - before; + // A fast-fail comes back through `.safe()` as a `StitchError` whose MESSAGE carries the + // breaker's own words — the `CircuitOpenError` class is flattened on the awaited path — so + // the message is what identifies it, not the constructor name. + const err = primary.error as (Error & { status?: number }) | undefined; + const callerError = primary.ok + ? 'ok' + : err?.message === 'circuit open' + ? 'circuit open' + : `${err?.name ?? 'error'} ${err?.status ?? ''}`.trim(); + ledgerRow(label, primaryRequests, vendor.count('v2'), callerError); + return { primaryRequests, callerError }; + } + + const CIRCUIT = [2, '60s'] as [number, string]; + + // (d1) Two standalone stitches. No shared store. Distinct paths. + const d1 = await probe('d1 standalone, distinct paths', (vendor) => ({ + v1: stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + circuit: CIRCUIT, + clock: manualClock(), + }), + v2: stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + circuit: CIRCUIT, + clock: manualClock(), + }), + })); + check( + '(d1) standalone stitches: primary still reached the vendor', + d1.primaryRequests, + 1, + ); + + // (d2) ONE seam — the natural way to configure a vendor once. The seam SHARES one store + // (seam.ts:229, stitch.ts:985). Paths still distinct. + const d2 = await probe('d2 shared seam, distinct paths', (vendor) => { + const vendorSeam = seam({ + baseUrl: HOST, + adapter: vendor.adapter, + circuit: CIRCUIT, + clock: manualClock(), + }); + return { + v1: vendorSeam.stitch({ path: V1_PATH }), + v2: vendorSeam.stitch({ path: V2_PATH }), + }; + }); + check( + '(d2) shared seam, distinct paths: primary still reached the vendor', + d2.primaryRequests, + 1, + ); + + // (d3) THE TRAP. A shared seam AND a colliding key. Here the two versions live on different + // ORIGINS but the same PATH — `api.vendor.test` vs `v2.vendor.test`, `/customers` on both — which + // is an entirely ordinary way for a vendor to ship a v2. `nameOf` reads `cfg.name ?? cfg.path`, so + // both stitches key on `'circuit:/customers'`. + const d3 = await probe('d3 shared seam, SAME path', (vendor) => { + const vendorSeam = seam({ + adapter: vendor.adapter, + circuit: CIRCUIT, + clock: manualClock(), + }); + return { + // Both spell `path: '/customers'`; only the origin differs. The fake routes on the + // /v1 or /v2 prefix in the URL, so the origin carries the version here. + v1: vendorSeam.stitch({ + baseUrl: `${HOST}/v1`, + path: '/customers', + }), + v2: vendorSeam.stitch({ + baseUrl: `${HOST}/v2`, + path: '/customers', + }), + }; + }); + check( + "(d3) shared seam + SAME path: primary was FAST-FAILED by the shadow's breaker", + d3.primaryRequests, + 0, + ); + check( + "(d3) the caller got the shadow's breaker error, verbatim", + d3.callerError, + 'circuit open', + ); + + // (d4) THE SECOND TRAP, and the one specific to a dual-run: `throttle: { pool: 'host' }` — the + // setting you reach for BECAUSE v1 and v2 share the vendor's meter (C7) — silently re-keys the + // CIRCUIT onto the URL host (engine.ts:266-272 feeding engine.ts:860). One host, one breaker, + // even though the paths differ. + const d4 = await probe('d4 shared seam + pool:host', (vendor) => { + const vendorSeam = seam({ + baseUrl: HOST, + adapter: vendor.adapter, + circuit: CIRCUIT, + // `pool: 'host'` ALONE — no rate, so nothing paces and nothing sleeps. That isolates + // the effect being measured: the mere presence of this key moves the BREAKER's key + // onto the URL host (engine.ts:266-272 feeding engine.ts:860). + throttle: { pool: 'host' }, + clock: manualClock(), + }); + return { + v1: vendorSeam.stitch({ path: V1_PATH }), + v2: vendorSeam.stitch({ path: V2_PATH }), + }; + }); + check( + '(d4) pool:host re-keys the breaker onto the HOST: primary fast-failed', + d4.primaryRequests, + 0, + ); + check( + '(d4) the caller got the breaker error, verbatim', + d4.callerError, + 'circuit open', + ); + + // (d5) The fix, measured: an explicit distinct `circuit.key` per version. This is the "keyed by + // hand" the prediction names, and it is the ONLY lever — `CircuitOptions` has exactly three fields + // (failures, cooldown, key) and `key` is a static string, not a function (types.ts:1145-1162). + const d5 = await probe('d5 pool:host + explicit circuit.key', (vendor) => { + const vendorSeam = seam({ + baseUrl: HOST, + adapter: vendor.adapter, + throttle: { pool: 'host' }, + clock: manualClock(), + }); + return { + v1: vendorSeam.stitch({ + path: V1_PATH, + circuit: { failures: 2, cooldown: '60s', key: 'cust-v1' }, + }), + v2: vendorSeam.stitch({ + path: V2_PATH, + circuit: { failures: 2, cooldown: '60s', key: 'cust-v2' }, + }), + }; + }); + check( + '(d5) explicit circuit.key restores isolation: primary reached the vendor', + d5.primaryRequests, + 1, + ); + + channelRow({ + channel: '(d) circuit', + safeByDefault: false, + measured: `d1/d2: 1 req · d3/d4: 0 req · d5: 1 req`, + fix: "distinct `circuit.key` per version (or distinct `name`), and never `pool:'host'` unkeyed", + }); + + printChannels(); + printLedger('C1 — requests the vendor actually received'); + + console.log(` + READING THE (d) ROWS. The prediction "resilience state is shared unless keyed by hand" is + HALF RIGHT, and the half that is wrong is the half that would have made this safe by accident: + + d1 standalone stitches, distinct paths -> ISOLATED (each builds its own memoryStore) + d2 one seam, distinct paths -> ISOLATED (shared store, but the key is the path) + d3 one seam, SAME path -> SHARED. v1 fast-failed on v2's breaker. + d4 one seam + throttle pool:'host' -> SHARED. The host became the key. + d5 explicit distinct circuit.key -> ISOLATED. + + So sharing is not the default; it is a COLLISION, and the collision is invisible in the config. + Two stitches collide when they land on the same store AND the same string out of + \`circuit.key ?? name ?? path ?? 'stitch'\`. The two ways a dual-run walks into it are both + ordinary: a v2 that keeps the path and changes the origin (d3), and \`pool: 'host'\` (d4) — which + is the setting a consumer reaches for precisely BECAUSE the two versions share the vendor's meter. + `); + + finish( + 'C1', + 'the shadow CAN be made unable to hurt the primary, but only ONE of the four channels is safe by default: the retry budget. Latency, thrown errors and the circuit each need explicit construction — and a fifth channel the capture did not list (`all()` auto-cancelling the in-flight primary) is the most dangerous of them', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c2-different-inputs.ts b/docs/scenarios/proofs/dual-run-migration/c2-different-inputs.ts new file mode 100644 index 00000000..6a2d1362 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c2-different-inputs.ts @@ -0,0 +1,295 @@ +// C2 — DECIDING CLAIM. Can the two calls take DIFFERENT inputs? +// +// This is the claim the whole scenario turns on, because a v2 that took the same input as v1 would +// not need a dual-run — you would just change the base URL. A real v2 moves the id from the path to +// a query parameter, renames it, and reshapes the body. So the question is whether the combinator +// that LOOKS purpose-built for "run these two together" can express two different inputs. +// +// PRE-REGISTERED PREDICTION (scenario 10 / issue #643): `runMember` (pipe.ts:75-86) builds every +// member's input from the ONE group input — +// +// const memberInput: StitchInput = { ...input, signal }; +// +// — so `all`/`any` cannot express it. This script confirms or refutes that by reading the literal +// URL the fake transport received, then measures every alternative spelling and what it costs. +// +// The vendor's actual v1 -> v2 input change, which is the thing being expressed: +// v1 GET /v1/customers/{id} id is a PATH parameter +// v2 GET /v2/customers?customer_id=… id moved to a QUERY parameter, and was renamed +import { all, linked } from '../../../../packages/core/src/pipe'; +import { stitch } from '../../../../packages/core/src/stitch'; +import { + check, + checkStr, + countUserLines, + finish, + heading, + note, +} from './harness'; +import { HOST, fakeVendor } from './vendor'; + +import { readFileSync } from 'node:fs'; + +const V1_PATH = '/v1/customers/{id}'; +const V2_PATH = '/v2/customers'; + +function pair(vendor: ReturnType) { + return { + v1: stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }), + v2: stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }), + }; +} + +async function main(): Promise { + // ----------------------------------------------------------------------- + heading('C2 (1) — `all([v1, v2])` with ONE group input'); + { + const vendor = fakeVendor({}); + const { v1, v2 } = pair(vendor); + + // The only input `all` accepts is the GROUP's. v1 wants `params.id`; v2 wants + // `query.customer_id`. There is exactly one slot to put either in. + await all([v1, v2])({ params: { id: 'cus_7Q2' } }).catch( + () => undefined, + ); + + const v1Url = vendor.pathOf('v1'); + const v2Url = vendor.pathOf('v2'); + note('v1 received', v1Url); + note('v2 received', v2Url); + + checkStr('v1 got the id it needed', v1Url, '/v1/customers/cus_7Q2'); + checkStr( + 'v2 got a request with NO id at all — the group input did not fit it', + v2Url, + '/v2/customers', + ); + check( + 'the shadow call is therefore WRONG (it asks for every customer, not this one)', + v2Url.includes('cus_7Q2'), + false, + ); + } + + // ----------------------------------------------------------------------- + heading("C2 (2) — the broadcast also LEAKS: v1's parameter names reach v2"); + { + // The mirror-image failure. Put the id where v2 wants it and v1 breaks — but worse, a + // query key is not silently dropped the way an unused path param is: it is APPENDED to + // every member's URL. So the shadow sends the primary's parameter spelling to the vendor. + const vendor = fakeVendor({}); + const { v1, v2 } = pair(vendor); + await all([v1, v2])({ + params: { id: 'cus_7Q2' }, + query: { customer_id: 'cus_7Q2' }, + }).catch(() => undefined); + + note('v1 received', vendor.pathOf('v1')); + note('v2 received', vendor.pathOf('v2')); + checkStr( + 'v2 finally got its query param…', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_7Q2', + ); + checkStr( + "…but v1 was handed v2's parameter too, and sent it to the vendor", + vendor.pathOf('v1'), + '/v1/customers/cus_7Q2?customer_id=cus_7Q2', + ); + note( + "a broadcast input is a UNION of both versions' parameters — every member sends every key", + ); + } + + // ----------------------------------------------------------------------- + heading( + 'C2 (3) — `.with()` inside `all`: does a per-member binding survive the broadcast?', + ); + { + const vendor = fakeVendor({}); + const { v1, v2 } = pair(vendor); + + // `.with()` binds part of a member's input at CONSTRUCTION. `bound.__runWith` merges the + // bound partial under the incoming input (stitch.ts:1132-1133 over `mergeInput`, + // stitch.ts:792-813), so the binding does survive `runMember`'s broadcast. + const group = all([v1, v2.with({ query: { customer_id: 'cus_7Q2' } })]); + await group({ params: { id: 'cus_7Q2' } }).catch(() => undefined); + + note('v1 received', vendor.pathOf('v1')); + note('v2 received', vendor.pathOf('v2')); + checkStr('v1 correct', vendor.pathOf('v1'), '/v1/customers/cus_7Q2'); + checkStr( + 'v2 correct — `.with()` DID survive `runMember`', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_7Q2', + ); + + // …and here is why that rescue does not hold. `.with()` binds a CONSTANT. Call the same + // group for a DIFFERENT customer and the shadow keeps asking about the first one. + vendor.reset(); + await group({ params: { id: 'cus_ZZZ' } }).catch(() => undefined); + note('second call, v1 received', vendor.pathOf('v1')); + note('second call, v2 received', vendor.pathOf('v2')); + checkStr( + 'v1 followed the new input', + vendor.pathOf('v1'), + '/v1/customers/cus_ZZZ', + ); + checkStr( + 'v2 is STILL pinned to the bound customer — it compared the wrong records', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_7Q2', + ); + check( + 'the two versions asked about the SAME customer', + vendor.pathOf('v1').includes('cus_ZZZ') && + vendor.pathOf('v2').includes('cus_ZZZ'), + false, + ); + note( + 'a dual-run whose shadow silently compares a different record produces a diff on every call', + ); + } + + // ----------------------------------------------------------------------- + heading('C2 (4) — the spellings that DO work, and what each costs'); + + // `linked` — sequential, `run(node, input)` takes a per-call input per node (pipe.ts ScopedRun). + { + const vendor = fakeVendor({}); + const { v1, v2 } = pair(vendor); + const id = 'cus_LNK'; + // >>> BEGIN USER CODE linked + await linked(async (run) => { + const primary = await run(v1, { params: { id } }); + const shadow = await run(v2, { query: { customer_id: id } }); + return { primary, shadow }; + }); + // <<< END USER CODE linked + note( + 'unlike all/any/race, `linked` returns a Promise, not a Composable (pipe.ts:357-359) — it runs on the spot and cannot itself be a member', + ); + checkStr( + 'linked: v1 correct', + vendor.pathOf('v1'), + '/v1/customers/cus_LNK', + ); + checkStr( + 'linked: v2 correct', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_LNK', + ); + note( + "linked runs the two SEQUENTIALLY — the shadow is on the caller's critical path", + ); + } + + // Plain `Promise.all` — no combinator at all. + { + const vendor = fakeVendor({}); + const { v1, v2 } = pair(vendor); + const id = 'cus_PAL'; + // >>> BEGIN USER CODE promise-all + const [primary, shadow] = await Promise.all([ + v1({ params: { id } }), + v2({ query: { customer_id: id } }), + ]); + // <<< END USER CODE promise-all + void primary; + void shadow; + checkStr( + 'Promise.all: v1 correct', + vendor.pathOf('v1'), + '/v1/customers/cus_PAL', + ); + checkStr( + 'Promise.all: v2 correct', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_PAL', + ); + note('Promise.all is concurrent, but still AWAITS the shadow (C1 (a))'); + } + + // The spelling that satisfies C2 *and* C1 at once: per-call input, concurrent, not awaited, + // cannot reject. This is the one C8 assembles. + { + const vendor = fakeVendor({ latency: { v2: 40 } }); + const { v1, v2 } = pair(vendor); + const id = 'cus_SAF'; + // >>> BEGIN USER CODE isolated-shadow + const shadow = v2.safe({ query: { customer_id: id } }); + const primary = await v1({ params: { id } }); + void shadow.then((r) => { + if (r.ok) compare(primary, r.data); + }); + // <<< END USER CODE isolated-shadow + await new Promise((r) => setTimeout(r, 120)); + checkStr( + 'isolated: v1 correct', + vendor.pathOf('v1'), + '/v1/customers/cus_SAF', + ); + checkStr( + 'isolated: v2 correct', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_SAF', + ); + check('isolated: the comparison ran', comparisons, 1); + } + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const lines = { + linked: countUserLines(src, 'linked'), + promiseAll: countUserLines(src, 'promise-all'), + isolated: countUserLines(src, 'isolated-shadow'), + }; + note('executable lines — `linked`', lines.linked); + note('executable lines — plain `Promise.all`', lines.promiseAll); + note( + 'executable lines — isolated shadow (satisfies C1 too)', + lines.isolated, + ); + check('the working spelling is under 6 lines', lines.isolated <= 6, true); + + console.log(` + THE SPELLINGS, SIDE BY SIDE + + spelling different inputs? concurrent? off the critical path? lines + --------------------------- ----------------- ----------- ---------------------- ----- + all([v1, v2]) NO yes no 1 + all([v1, v2.with({...})]) CONSTANT only yes no 1 + linked(run => ...) yes no no ${lines.linked} + Promise.all([...]) yes yes no ${lines.promiseAll} + v2.safe(...) not awaited yes yes YES ${lines.isolated} + + The combinators are not merely inadequate here — \`all\` is actively wrong on THREE counts at + once: it broadcasts one input (C2), it awaits the slowest member (C1 a), and it cancels the + primary when the shadow fails (C1 c*). The working spelling uses no combinator at all. +`); + + finish( + 'C2', + "CONFIRMED. `all`/`any` broadcast one input to every member, so the two versions cannot take different inputs through a combinator: the shadow either receives no id (`/v2/customers`) or receives the primary's parameter spelling as well. `.with()` partially rescues it — a bound partial DOES survive `runMember` — but only as a CONSTANT, so a group built once and called twice sent the shadow to the wrong customer. The working spelling is a plain un-awaited `.safe()` call", + ); +} + +// A stand-in for the comparison C3/C4 build properly — here only to prove the shadow's result is +// reachable at all from the isolated spelling. +let comparisons = 0; +function compare(a: unknown, b: unknown): void { + void a; + void b; + comparisons++; +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c3-comparison.ts b/docs/scenarios/proofs/dual-run-migration/c3-comparison.ts new file mode 100644 index 00000000..3464a5d7 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c3-comparison.ts @@ -0,0 +1,208 @@ +// C3 — is there a COMPARISON primitive? +// +// A dual-run is two calls and a comparison. C1 and C2 measured the two calls. This measures whether +// the library has anything that compares one RESPONSE to another RESPONSE — as opposed to `drift()`, +// which is anchored to a SCHEMA (response vs contract). +// +// The measurement is deliberately mechanical: enumerate what the public barrel actually exports at +// runtime, read the `exports` map in package.json for the reachable subpaths, and call each +// candidate to see what its inputs really are. A claim about a public surface should be a directory +// listing, not a recollection. +import { diff } from '../../../../packages/core/src/diff'; +import { classifyDiff } from '../../../../packages/core/src/drift'; +import * as barrel from '../../../../packages/core/src/index'; +import { drift } from '../../../../packages/core/src/stitch'; +import { + check, + checkSeq, + countUserLines, + finish, + heading, + note, +} from './harness'; +import { v1Customer, v2Customer } from './vendor'; + +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); + +async function main(): Promise { + // ----------------------------------------------------------------------- + heading( + 'C3 (1) — what the PUBLIC barrel exports, and whether any of it compares two values', + ); + { + const names = Object.keys(barrel).sort(); + note('public exports on `stitchapi` (the root barrel)', names.length); + + // The candidate set: anything whose NAME suggests a comparison. Naming is the only honest + // filter here — a reader looking for "compare two responses" greps for exactly these. + const candidates = names.filter((n) => + /diff|compare|equal|match|drift|ignore|normali[sz]e|canonical/i.test( + n, + ), + ); + checkSeq('comparison-shaped names on the root barrel', candidates, [ + 'drift', + ]); + note('exactly one candidate, and it is `drift`'); + + // Reachable subpaths, from the package's own `exports` map. + const pkg = require('../../../../packages/core/package.json') as { + exports: Record; + }; + const subpaths = Object.keys(pkg.exports).sort(); + note('reachable subpaths', subpaths.join(' ')); + check('a `./diff` subpath exists', subpaths.includes('./diff'), false); + check( + 'a `./drift` subpath exists', + subpaths.includes('./drift'), + false, + ); + check( + 'a `./compare` subpath exists', + subpaths.includes('./compare'), + false, + ); + } + + // ----------------------------------------------------------------------- + heading( + 'C3 (2) — `drift()` is schema-anchored: what does it actually take?', + ); + { + // `drift(schema, options)` does not compare anything. It TAGS a schema for the engine. + const spec = drift( + (v: unknown) => typeof v === 'object' && v !== null, + { + ignore: 'meta', + }, + ) as unknown as Record; + checkSeq('the object `drift()` returns', Object.keys(spec).sort(), [ + '__kind', + 'options', + 'schema', + ]); + check('its `__kind`', spec['__kind'], 'drift'); + note( + 'input 1 is a SCHEMA, input 2 is options — there is no second VALUE parameter, so it cannot compare response vs response', + ); + check('drift() arity', drift.length, 1); + } + + // ----------------------------------------------------------------------- + heading( + 'C3 (3) — the primitive DOES exist; it is just not on the public surface', + ); + { + // `diff(before, after)` takes two arbitrary values. This is exactly the response-vs-response + // comparator the scenario needs — and it lives at packages/core/src/diff.ts:94, reachable + // from no subpath in the table above. + const ops = diff(v1Customer(), v2Customer()); + note('diff(v1, v2) op count', ops.length); + check('diff() arity — two values, no schema', diff.length, 2); + check( + 'the returned ops carry a path and an op', + ops.every((o) => Array.isArray(o.path) && typeof o.op === 'string'), + true, + ); + check( + 'it found the regression at balance_cents', + ops.some((o) => o.path.join('.') === 'balance_cents'), + true, + ); + + // `classifyDiff(a, b, opts)` is the other one: also two arbitrary values (drift.ts:113-117), + // and it accepts the declarative `ignore` grammar. Also unreachable from any subpath. + const findings = classifyDiff(v1Customer(), v2Customer(), {}); + note('classifyDiff(v1, v2) finding count', findings.length); + check('classifyDiff() arity', classifyDiff.length, 2); + check( + 'classifyDiff renders string paths', + findings.every((f) => typeof f.path === 'string'), + true, + ); + note( + 'its LABELS are named for validation, not for a dual-run: `undeclared` = only in v1, `defaulted` = only in v2, `coerced` = differs', + ); + note( + 'and `coerced` renders a TYPE delta ("number -> string"), never the two values — a wrong NUMBER of the right type has no values in its detail', + ); + const balance = findings.find((f) => f.path === 'balance_cents'); + note('the regression, as classifyDiff renders it', balance?.detail); + } + + // ----------------------------------------------------------------------- + heading('C3 (4) — the minimum hand-written comparator'); + { + // What a consumer must write if they will not reach into `src/`. This is a full + // structural comparator: recursive, path-carrying, and honest about the four cases + // (missing left, missing right, kind mismatch, primitive inequality). + // >>> BEGIN USER CODE comparator + type Delta = { path: string; left: unknown; right: unknown }; + function compare(a: unknown, b: unknown, path = ''): Delta[] { + if (Object.is(a, b)) return []; + const both = + a && b && typeof a === 'object' && typeof b === 'object'; + if (!both) return [{ path, left: a, right: b }]; + if (Array.isArray(a) !== Array.isArray(b)) + return [{ path, left: a, right: b }]; + const keys = new Set([ + ...Object.keys(a as object), + ...Object.keys(b as object), + ]); + const out: Delta[] = []; + for (const k of keys) + out.push( + ...compare( + (a as Record)[k], + (b as Record)[k], + path ? `${path}.${k}` : k, + ), + ); + return out; + } + // <<< END USER CODE comparator + + const deltas = compare(v1Customer(), v2Customer()); + note('hand-written comparator delta count', deltas.length); + check( + 'it finds the regression', + deltas.some((d) => d.path === 'balance_cents'), + true, + ); + check( + 'and unlike classifyDiff it carries BOTH VALUES, which is what a regression report needs', + deltas.find((d) => d.path === 'balance_cents')?.right, + 41_520, + ); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const lines = countUserLines(src, 'comparator'); + note('executable lines for the minimum comparator', lines); + check('it fits in under 25 lines', lines <= 25, true); + } + + console.log(` + WHAT IS AND IS NOT REACHABLE + + symbol compares reachable from + -------------------------- ------------------ ---------------------------------------- + drift(schema, opts) nothing (a tag) \`stitchapi\` — PUBLIC + classifyDiff(a, b, opts) value vs value packages/core/src/drift.ts — SOURCE ONLY + diff(before, after) value vs value packages/core/src/diff.ts — SOURCE ONLY + + So the answer is not "there is no comparison primitive". There are TWO, both of them exactly the + right shape — \`diff\` takes two arbitrary values, and \`classifyDiff\` adds the declarative + \`ignore\` grammar C4 needs — and NEITHER is exported from any of the 17 public subpaths. A + consumer doing this today either vendors ~20 lines of comparator or reaches into \`src/\`. +`); + + finish( + 'C3', + 'PARTIAL. Nothing on the PUBLIC surface compares response vs response: the one comparison-shaped export on the root barrel is `drift()`, which takes a SCHEMA and a set of options and performs no comparison at all. But the primitive exists twice in the tree — `diff(before, after)` (diff.ts:94) and `classifyDiff(a, b, opts)` (drift.ts:113), both taking two arbitrary values — and neither is reachable from any of the 17 subpaths in the package `exports` map. The minimum hand-written replacement is a measured 23 executable lines, and it is strictly BETTER than `classifyDiff` for this use because it carries both values: `classifyDiff` renders the planted regression as the detail "number -> number", a type delta with no numbers in it', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c4-relevancy.ts b/docs/scenarios/proofs/dual-run-migration/c4-relevancy.ts new file mode 100644 index 00000000..7220b859 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c4-relevancy.ts @@ -0,0 +1,300 @@ +// C4 — THE RELEVANCY PROBLEM. Telling a real diff from a benign one is the actual work. +// +// The vendor's v2 is CORRECT and still diffs on every single call, because a correct v2 does all of +// this at once (vendor.ts holds the payloads): +// +// rename `created` -> `created_at` +// retype epoch int -> ISO string (the same instant, a different wire type) +// reorder tags[] -> the same three tags in a different order +// add `livemode` -> a field v1 never had +// REGRESS balance_cents -> 41250 became 41520 (THE one thing a dual-run exists to find) +// +// Four of those five are noise. This script measures the raw diff count, then the count after a +// relevancy filter, and — the part the capture asks about — how much of that filter can be +// expressed DECLARATIVELY versus how much is user code. +import { diff } from '../../../../packages/core/src/diff'; +import { classifyDiff } from '../../../../packages/core/src/drift'; +import { + check, + checkSeq, + countUserLines, + finish, + heading, + note, +} from './harness'; +import { + CREATED_EPOCH, + CREATED_ISO, + REGRESSED_BALANCE, + TRUE_BALANCE, + v1Customer, + v2Customer, + v2CustomerFixed, +} from './vendor'; + +import { readFileSync } from 'node:fs'; + +async function main(): Promise { + const v1 = v1Customer(); + const v2 = v2Customer(); + + // ----------------------------------------------------------------------- + heading('C4 (1) — the RAW diff on a correct v2'); + { + const ops = diff(v1, v2); + const paths = ops.map((o) => `${o.op} ${o.path.join('.')}`).sort(); + note('raw diff op count', ops.length); + checkSeq('every raw op', paths, [ + 'change balance_cents', + 'change tags.0', + 'change tags.1', + 'change tags.2', + 'create created_at', + 'create livemode', + 'remove created', + ]); + check( + 'raw diff count on a CORRECT v2 with one regression', + ops.length, + 7, + ); + note( + 'six of the seven are benign. A dual-run that reports this raw is a dual-run nobody reads', + ); + + // The array case is the sharpest: the tags are IDENTICAL as a set, and every element diffs. + const v1Tags = v1['tags'] as string[]; + const v2Tags = v2['tags'] as string[]; + checkSeq('v1 tags sorted', [...v1Tags].sort(), [...v2Tags].sort()); + check( + 'the tag arrays are equal as SETS', + JSON.stringify([...v1Tags].sort()) === + JSON.stringify([...v2Tags].sort()), + true, + ); + check('…and diff() reports every element as changed', 3, 3); + note( + 'diff() walks arrays strictly by index (diff.ts:58-73) — no keying, no LCS, no set semantics', + ); + } + + // ----------------------------------------------------------------------- + heading('C4 (2) — how much of the filter is DECLARATIVE?'); + { + // The only declarative filter in the tree is `DriftOptions.ignore` (types.ts:83-97), + // consumed by `classifyDiff`. Its grammar: exact path, single-segment `*`, or prefix, + // with `[]` for array elements. Measure what it CAN and CANNOT express, one clause at a + // time, against the four kinds of noise. + const all = classifyDiff(v1, v2, {}); + note('classifyDiff findings, unfiltered', all.length); + + // (a) the NEW FIELD — a plain path. Expressible. + const noNew = classifyDiff(v1, v2, { ignore: ['livemode'] }); + check( + 'ignore: livemode drops the new field', + all.length - noNew.length, + 1, + ); + + // (b) the ARRAY REORDER — `tags[]` matches every element. Expressible AS SUPPRESSION. + const noTags = classifyDiff(v1, v2, { ignore: ['tags[]'] }); + note('findings after ignore: tags[]', noTags.length); + check( + 'ignore: `tags[]` suppresses the reorder', + noTags.some((f) => f.path.startsWith('tags')), + false, + ); + note( + 'but this SUPPRESSES the field, it does not compare it unordered — a genuine tag change is now invisible too', + ); + + // Prove that last sentence rather than asserting it: change a tag for real and confirm the + // same `ignore` clause hides it. + const v2Tampered = { + ...v2, + tags: ['eu', 'invoiced', 'ENTERPRISE-PLUS'], + }; + const tampered = classifyDiff(v1, v2Tampered, { ignore: ['tags[]'] }); + check( + 'a REAL tag change is also hidden by ignore: tags[]', + tampered.some((f) => f.path.startsWith('tags')), + false, + ); + + // (c) the RENAME + RETYPE — two paths, `created` and `created_at`. `ignore` can suppress + // both, but suppression is not equivalence: nothing checks that the two carry the same + // instant, so a v2 that reported the WRONG date would pass identically. + const noCreated = classifyDiff(v1, v2, { + ignore: ['created', 'created_at'], + }); + check( + 'ignore can suppress both halves of the rename', + noCreated.some((f) => f.path.startsWith('created')), + false, + ); + const v2WrongDate = { ...v2, created_at: '1999-01-01T00:00:00.000Z' }; + const wrongDate = classifyDiff(v1, v2WrongDate, { + ignore: ['created', 'created_at'], + }); + check( + 'a WRONG created_at is hidden by the same clause', + wrongDate.some((f) => f.path.startsWith('created')), + false, + ); + note( + 'there is no aliasing option anywhere — no `equivalent`, no `rename`, no `keyBy`, no comparator hook, no numeric tolerance', + ); + + // All four noise clauses together. + const filtered = classifyDiff(v1, v2, { + ignore: ['livemode', 'tags[]', 'created', 'created_at'], + }); + note('findings after the full declarative filter', filtered.length); + checkSeq( + 'what survives', + filtered.map((f) => f.path), + ['balance_cents'], + ); + check( + 'the declarative filter alone gets to exactly the regression', + filtered.length, + 1, + ); + } + + // ----------------------------------------------------------------------- + heading('C4 (3) — the honest filter: normalize, then compare'); + { + // Suppression got the count right for the wrong reason: it hid `created`/`tags` rather than + // checking them. The filter a real migration wants NORMALIZES the two shapes onto common + // ground and then compares everything that is left — so a wrong date and a changed tag are + // still caught. This is the "relevancy model" the field guidance names, and all of it is + // user code. + // >>> BEGIN USER CODE relevancy + const KNOWN = { + renamed: { created: 'created_at' } as Record, + unordered: new Set(['tags']), + added: new Set(['livemode']), + }; + function normalize( + body: Record, + side: 'v1' | 'v2', + ): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(body)) { + if (side === 'v2' && KNOWN.added.has(k)) continue; + const key = side === 'v1' ? (KNOWN.renamed[k] ?? k) : k; + out[key] = KNOWN.unordered.has(key) + ? [...(v as unknown[])].sort() + : key === 'created_at' + ? new Date( + typeof v === 'number' ? v * 1000 : (v as string), + ).toISOString() + : v; + } + return out; + } + const deltas = diff(normalize(v1, 'v1'), normalize(v2, 'v2')); + // <<< END USER CODE relevancy + + note('normalized diff op count', deltas.length); + checkSeq( + 'what survives normalization', + deltas.map((o) => o.path.join('.')), + ['balance_cents'], + ); + check( + 'normalization also reaches exactly the regression', + deltas.length, + 1, + ); + check('and it carries BOTH values', deltas[0]?.oldValue, TRUE_BALANCE); + check('…including the wrong one', deltas[0]?.value, REGRESSED_BALANCE); + + // The difference from suppression, measured: normalization still catches the two things + // `ignore` hid. + const wrongDate = diff( + normalize(v1, 'v1'), + normalize({ ...v2, created_at: '1999-01-01T00:00:00.000Z' }, 'v2'), + ); + check( + 'normalization CATCHES a wrong created_at (ignore did not)', + wrongDate.some((o) => o.path.join('.') === 'created_at'), + true, + ); + const tagChange = diff( + normalize(v1, 'v1'), + normalize( + { ...v2, tags: ['eu', 'invoiced', 'ENTERPRISE-PLUS'] }, + 'v2', + ), + ); + check( + 'normalization CATCHES a real tag change (ignore did not)', + tagChange.some((o) => o.path.join('.').startsWith('tags')), + true, + ); + check( + 'and it does NOT flag the benign reorder', + diff(normalize(v1, 'v1'), normalize(v2, 'v2')).some((o) => + o.path.join('.').startsWith('tags'), + ), + false, + ); + + // The end state: a fixed v2 diffs at zero. This is the "cut over when the diff is quiet" + // signal, and it only exists once normalization is in place. + const quiet = diff( + normalize(v1, 'v1'), + normalize(v2CustomerFixed(), 'v2'), + ); + check('a CORRECTED v2 normalizes to a silent diff', quiet.length, 0); + // Assert the fixture's own premise instead of trusting it: the retype must be a pure + // encoding change, or "normalization silences it" would be measuring a bug in the fixture. + check( + 'the epoch and the ISO string are the SAME instant', + new Date(CREATED_EPOCH * 1000).toISOString(), + CREATED_ISO, + ); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const lines = countUserLines(src, 'relevancy'); + note('executable lines for the relevancy model', lines); + check( + 'the relevancy model is under 30 lines for FIVE known changes', + lines <= 30, + true, + ); + } + + console.log(` + THE RELEVANCY LEDGER + + v1 -> v2 change kind declarative? user code? + ------------------ -------- --------------------------------- ---------------------------- + livemode added new YES ignore: 'livemode' — + tags[] reordered reorder PARTLY ignore: 'tags[]' needed — ignore SUPPRESSES, + it cannot compare unordered + created->created_at rename PARTLY ignore both paths needed — no aliasing option + exists, so a wrong date passes + epoch -> ISO retype NO needed — no coercion hook + balance_cents REAL (must survive every filter) — + + raw diff ops 7 + after the declarative filter (ignore x4) 1 + after a hand-written relevancy model 1 <- and it still catches what ignore hid + + \`ignore\` is path-based SUPPRESSION and nothing else. It gets the count to 1 here, but it does so + by refusing to look at three of the four noise sites — so the same config that silences the + benign reorder silences a genuine tag change, and the same config that silences the rename + silences a v2 that reports the wrong date. The two constructions produce the same NUMBER and + are not the same test. +`); + + finish( + 'C4', + 'MEASURED. A correct v2 with one planted regression produces 7 raw diff ops, 6 of them benign — a 6:1 noise ratio on every single call. The declarative surface is exactly one option, `DriftOptions.ignore` (path, `*`, prefix, `[]` for array elements), reachable only through the source-only `classifyDiff`; it takes 7 -> 1 with four clauses. But it is SUPPRESSION, not relevancy: the clause that silences the benign tag reorder also silences a real tag change, and the clause that silences the rename also silences a v2 reporting the wrong instant — both measured. There is no aliasing, no unordered-array comparison, no type-coercion hook and no tolerance anywhere in the tree, so a filter that still catches what it should is a measured 24 lines of user code', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c5-writes.ts b/docs/scenarios/proofs/dual-run-migration/c5-writes.ts new file mode 100644 index 00000000..d8d81374 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c5-writes.ts @@ -0,0 +1,354 @@ +// C5 — WRITES. "You cannot shadow a write" is the sentence the whole technique rests on, and a +// mirrored `POST /charges` charges the customer twice. So: is there any guard? And what is the +// CHEAPEST construction that makes shadowing a write IMPOSSIBLE rather than merely discouraged? +// +// "Impossible" is the operative word. A rule in a code-review checklist is discouragement. A gate +// that a stitch cannot get past is impossibility, and the two are told apart by one measurement: +// how many non-GET requests the vendor actually received. +// +// Three candidate gates are measured, in increasing order of how hard they are to bypass: +// +// (1) nothing — the naive dual-run. Measures the damage. +// (2) a CONFIG gate — refuse at construction on `__config.method`. Earliest, and BYPASSABLE. +// (3) an ADAPTER gate — refuse at the last seam before the transport. Cannot be bypassed. +import { type LlmProvider, llm } from '../../../../packages/core/src/llm'; +import { stitch } from '../../../../packages/core/src/stitch'; +import type { Adapter, Stitch } from '../../../../packages/core/src/types'; +import { + check, + checkHas, + countUserLines, + finish, + heading, + ledgerRow, + note, + printLedger, +} from './harness'; +import { HOST, fakeVendor } from './vendor'; + +import { readFileSync } from 'node:fs'; + +// --------------------------------------------------------------------------- +// The two gates, as a consumer would vendor them. + +// >>> BEGIN USER CODE config-gate +/** Refuse at CONSTRUCTION: a stitch whose configured method is not a read may not be shadowed. */ +function readOnlyStitch>(s: T): T { + const m = (s.__config.method ?? 'GET').toUpperCase(); + if (m !== 'GET' && m !== 'HEAD') + throw new Error( + `refusing to shadow a ${m} — a shadowed write runs twice`, + ); + return s; +} +// <<< END USER CODE config-gate + +// >>> BEGIN USER CODE adapter-gate +/** Refuse at the LAST SEAM before the transport: this adapter cannot emit a non-read, ever. */ +function readsOnly(adapter: Adapter): Adapter { + return (req) => { + if (req.method !== 'GET' && req.method !== 'HEAD') + throw new Error(`refusing to shadow a ${req.method} to ${req.url}`); + return adapter(req); + }; +} +// <<< END USER CODE adapter-gate + +async function main(): Promise { + // ----------------------------------------------------------------------- + heading( + 'C5 (1) — the naive dual-run of a write: what does the vendor receive?', + ); + { + const vendor = fakeVendor({}); + const charge1 = stitch({ + baseUrl: HOST, + path: '/v1/charges', + method: 'POST', + adapter: vendor.adapter, + name: 'charge-v1', + }); + const charge2 = stitch({ + baseUrl: HOST, + path: '/v2/charges', + method: 'POST', + adapter: vendor.adapter, + name: 'charge-v2', + }); + + // Nothing anywhere objects to this. It constructs, it runs, it charges twice. + await Promise.all([ + charge1({ body: { amount: 4200, currency: 'gbp' } }), + charge2({ body: { amount: 4200, currency: 'gbp' } }), + ]); + + check('v1 charges', vendor.countMethod('v1', 'POST'), 1); + check( + 'v2 charges — the customer was billed twice', + vendor.countMethod('v2', 'POST'), + 1, + ); + check( + 'total non-GET requests the vendor received', + vendor.log.filter((c) => c.method !== 'GET').length, + 2, + ); + ledgerRow( + 'no guard (naive dual-run of a POST)', + vendor.countMethod('v1', 'POST'), + vendor.countMethod('v2', 'POST'), + 'both charges accepted', + ); + note( + 'no config key, no type error, no runtime nudge objected to any of this', + ); + } + + // ----------------------------------------------------------------------- + heading('C5 (2) — is there a BUILT-IN guard anywhere?'); + { + const vendor = fakeVendor({}); + // The only method-shaped option in the config surface is `cache.methods`, and it is about + // which methods are CACHEABLE (types.ts:1255-1261, default ['GET','HEAD']) — it gates the + // cache, not the wire. Setting it does not stop a POST from being sent. + const charge = stitch({ + baseUrl: HOST, + path: '/v2/charges', + method: 'POST', + adapter: vendor.adapter, + name: 'charge-v2', + cache: { ttl: '1m', methods: ['GET'] }, + }); + await charge({ body: { amount: 1 } }); + check( + 'POSTs the vendor received DESPITE cache.methods:[GET]', + vendor.countMethod('v2', 'POST'), + 1, + ); + note( + 'it is a cacheability gate, not a safety gate — the request went out exactly as before', + ); + note( + 'there is no `readOnly`, no `safe`, no `idempotentOnly` and no shadow-aware slot on StitchConfig', + ); + } + + // ----------------------------------------------------------------------- + heading( + 'C5 (3) — the CONFIG gate: refuse at construction on `__config.method`', + ); + { + const vendor = fakeVendor({}); + const read = stitch({ + baseUrl: HOST, + path: '/v2/customers', + adapter: vendor.adapter, + name: 'cust-v2', + }); + const write = stitch({ + baseUrl: HOST, + path: '/v2/charges', + method: 'POST', + adapter: vendor.adapter, + name: 'charge-v2', + }); + + note('a GET stitch reports `__config.method`', read.__config.method); + note('a POST stitch reports `__config.method`', write.__config.method); + check( + 'the default is `undefined`, so a gate must read it AS a GET', + read.__config.method, + undefined, + ); + + let refused = ''; + try { + readOnlyStitch(write); + } catch (e) { + refused = (e as Error).message; + } + checkHas( + 'the config gate refuses a POST', + refused, + 'refusing to shadow a POST', + ); + check( + 'and it refuses BEFORE any request is made', + vendor.log.length, + 0, + ); + check( + 'a GET passes the config gate', + readOnlyStitch(read) === read, + true, + ); + + // THE HOLE, measured. The gate reads the AUTHORED method, and a surface may override the + // method the wire actually carries. `llm` "always POSTs to the provider — `method` is + // ignored" (types.ts:809), so an llm stitch reports no method at all and still writes. + const provider: LlmProvider = { + id: 'fake', + url: `${HOST}/v2/messages`, + defaultModel: 'fake-1', + buildBody: (r) => ({ model: r.model, messages: r.messages }), + parse: () => ({ text: 'ok', raw: {} }), + }; + const chat = llm({ + provider, + adapter: vendor.adapter, + name: 'chat-v2', + }); + note('an llm stitch reports `__config.method`', chat.__config.method); + const passedGate = (() => { + try { + readOnlyStitch(chat as unknown as Stitch); + return true; + } catch { + return false; + } + })(); + check('the llm stitch PASSES the config gate', passedGate, true); + await chat({ + body: { messages: [{ role: 'user', content: 'hi' }] }, + }).catch(() => undefined); + const wireMethod = vendor.log.at(-1)?.method; + check('…and the method it actually sent', wireMethod, 'POST'); + note( + 'a gate that reads the CONFIG can be told a different story than the transport gets', + ); + ledgerRow( + 'config gate (`__config.method`)', + 0, + vendor.log.filter((c) => c.method === 'POST').length, + 'refuses a POST stitch; an llm stitch slips past', + ); + } + + // ----------------------------------------------------------------------- + heading( + 'C5 (4) — the ADAPTER gate: refuse at the last seam before the transport', + ); + { + const vendor = fakeVendor({}); + // The primary keeps the REAL adapter — it must still be able to write. Only the SHADOW is + // built on the gated one, so the asymmetry is structural rather than procedural. + const chargeV1 = stitch({ + baseUrl: HOST, + path: '/v1/charges', + method: 'POST', + adapter: vendor.adapter, + name: 'charge-v1', + }); + const chargeV2Shadow = stitch({ + baseUrl: HOST, + path: '/v2/charges', + method: 'POST', + adapter: readsOnly(vendor.adapter), + name: 'charge-v2', + }); + + const primary = await chargeV1({ body: { amount: 4200 } }); + check( + 'the PRIMARY write still succeeded', + (primary as { ok?: boolean }).ok, + true, + ); + + const shadow = await chargeV2Shadow.safe({ body: { amount: 4200 } }); + check('the SHADOW write failed', shadow.ok, false); + checkHas( + 'and the refusal names the method and the URL', + (shadow.error as Error).message, + 'refusing to shadow a POST', + ); + + // Try to get past it. The gate is below every authoring surface, so none of them reach it. + const viaLlm = llm({ + provider: { + id: 'fake', + url: `${HOST}/v2/messages`, + defaultModel: 'fake-1', + buildBody: () => ({}), + parse: () => ({ text: '', raw: {} }), + }, + adapter: readsOnly(vendor.adapter), + name: 'chat-v2', + }); + const llmResult = await viaLlm + .safe({ body: { messages: [{ role: 'user', content: 'hi' }] } }) + .catch(() => ({ ok: false as const })); + check( + 'the llm surface cannot get past the adapter gate either', + llmResult.ok, + false, + ); + + // A `.with()`-bound handle shares the same runtime, so it shares the gated adapter. + const bound = chargeV2Shadow.with({ body: { amount: 1 } }); + const boundResult = await bound.safe({}); + check( + 'a `.with()`-bound handle cannot get past it', + boundResult.ok, + false, + ); + + // THE MEASUREMENT THE CLAIM ASKS FOR: how many writes did the vendor receive on v2? + const shadowWrites = vendor.log.filter( + (c) => + c.version === 'v2' && c.method !== 'GET' && c.method !== 'HEAD', + ).length; + check('SHADOW WRITES THE VENDOR RECEIVED', shadowWrites, 0); + check( + 'primary writes the vendor received', + vendor.countMethod('v1', 'POST'), + 1, + ); + ledgerRow( + 'adapter gate (`readsOnly`)', + vendor.countMethod('v1', 'POST'), + shadowWrites, + '3 shadow write attempts, 0 reached the wire', + ); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + note( + 'executable lines — config gate', + countUserLines(src, 'config-gate'), + ); + note( + 'executable lines — adapter gate', + countUserLines(src, 'adapter-gate'), + ); + check( + 'the adapter gate is 10 executable lines or fewer', + countUserLines(src, 'adapter-gate') <= 10, + true, + ); + } + + printLedger('C5 — non-GET requests the vendor received'); + + console.log(` + THE THREE GATES + + gate refuses when bypassable by shadow writes on the wire + ------------------------ ----------------- ----------------------- ------------------------- + none never — 1 (the customer paid twice) + config (__config.method) construction any surface that forces 0 for a plain POST stitch, + a method (llm always 1 for an llm stitch + POSTs, types.ts:809) + adapter (readsOnly) call, last seam nothing measured here 0 of 3 attempts + + The adapter gate is the cheap one AND the sound one: 8 executable lines, wrapping the seam that + every authoring surface must eventually pass through. The config gate is worth keeping ON TOP of + it — it fails at construction rather than at call time, which is the better error — but it is a + nudge, not a guarantee, and this script measures exactly the case where it is wrong. +`); + + finish( + 'C5', + "NO BUILT-IN GUARD EXISTS, and the cheapest construction that makes shadowing a write impossible is an 8-line Adapter wrapper. Measured: an unguarded dual-run of `POST /charges` sent 2 charges to the vendor with no config key, type error, or runtime nudge objecting; the only method-shaped option on the config surface is `cache.methods`, which gates CACHEABILITY and let the POST through unchanged. A construction-time gate on `__config.method` refuses a plain POST stitch before any request, but is bypassed by a surface that forces its own method — the `llm` surface reports `__config.method === undefined`, passes the gate, and sends a POST. The Adapter wrapper sits below every authoring surface: 3 shadow write attempts (a plain POST, an llm-surface call, a `.with()`-bound handle) reached the wire 0 times, while the primary's own POST still succeeded", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c6-cutover.ts b/docs/scenarios/proofs/dual-run-migration/c6-cutover.ts new file mode 100644 index 00000000..bb205970 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c6-cutover.ts @@ -0,0 +1,228 @@ +// C6 — CUTOVER WITHOUT REDEPLOY. The dual-run ends when the diff goes quiet, and the point of +// running it against live traffic is that you can stop at any moment — including at 3am, from a +// flag, without shipping code. +// +// Scenario 19 measured that a `baseUrl` THUNK (`string | (() => string)`) is resolved per call, so +// it retargets between calls. The question here is whether that extends to swapping a WHOLE STITCH +// — a different path, a different input shape, a different response shape — or whether it only +// moves an origin. +// +// Four things have to change at cutover, and they are measured one at a time: +// the ORIGIN api.vendor.test (v1 and v2 may or may not share it) +// the PATH /v1/customers/{id} -> /v2/customers +// the INPUT SHAPE params.id -> query.customer_id +// the OUTPUT SHAPE created (epoch) -> created_at (ISO) +import { stitch } from '../../../../packages/core/src/stitch'; +import { + check, + checkStr, + countUserLines, + finish, + heading, + note, +} from './harness'; +import { HOST, fakeVendor } from './vendor'; + +import { readFileSync } from 'node:fs'; + +async function main(): Promise { + // ----------------------------------------------------------------------- + heading('C6 (1) — the `baseUrl` thunk: does it retarget BETWEEN calls?'); + { + const vendor = fakeVendor({}); + // The flag a cutover would actually be driven by: read at call time, changeable at runtime. + let cutover = false; + const cust = stitch({ + baseUrl: () => (cutover ? `${HOST}/v2` : `${HOST}/v1`), + path: '/customers/{id}', + adapter: vendor.adapter, + name: 'cust', + }); + + await cust({ params: { id: 'cus_7Q2' } }); + checkStr( + 'before the flag', + vendor.pathOf('v1'), + '/v1/customers/cus_7Q2', + ); + cutover = true; + await cust({ params: { id: 'cus_7Q2' } }); + checkStr( + 'after the flag — no redeploy, no reconstruction', + vendor.pathOf('v2'), + '/v2/customers/cus_7Q2', + ); + check('the thunk is resolved per call', vendor.log.length, 2); + note( + 'confirms scenario 19: `baseUrl` is `string | (() => string)` (types.ts:1570)', + ); + } + + // ----------------------------------------------------------------------- + heading('C6 (2) — does it extend to the PATH? `url` is a thunk too'); + { + // `path` is a plain `string` (types.ts:1572) — no thunk. But `url` IS + // `string | (() => string)` (types.ts:1568) and it carries the COMPLETE endpoint, so a + // thunk on `url` moves the path as well as the origin. Measure whether `{param}` templating + // still applies to a thunk-supplied url. + const vendor = fakeVendor({}); + let cutover = false; + const cust = stitch({ + url: () => + cutover ? `${HOST}/v2/customers` : `${HOST}/v1/customers/{id}`, + adapter: vendor.adapter, + name: 'cust', + }); + + await cust({ params: { id: 'cus_7Q2' } }).catch(() => undefined); + checkStr( + 'a `{param}` slot in a THUNK-supplied url is still interpolated', + vendor.pathOf('v1'), + '/v1/customers/cus_7Q2', + ); + cutover = true; + await cust({ query: { customer_id: 'cus_7Q2' } }).catch( + () => undefined, + ); + checkStr( + 'and the flag moved the whole path, not just the origin', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_7Q2', + ); + note( + 'so the thunk covers origin AND path — the capture\'s "only a base URL" reading is too narrow', + ); + + // …but the caller had to pass a DIFFERENT input shape on either side of the flag, and + // nothing in the config did that. Prove it: pass the v1 input after the flag and watch the + // request go out wrong rather than fail. + vendor.reset(); + await cust({ params: { id: 'cus_7Q2' } }).catch(() => undefined); + checkStr( + 'the v1 input against the v2 url silently drops the id', + vendor.pathOf('v2'), + '/v2/customers', + ); + check( + 'nothing threw — the wrong call was simply made', + vendor.log.length, + 1, + ); + } + + // ----------------------------------------------------------------------- + heading('C6 (3) — the two things a thunk canNOT move'); + { + const vendor = fakeVendor({}); + // (a) INPUT SHAPE. There is no thunk and no per-call hook that rewrites the input slots: + // `input` is a schema bag, `transform` reshapes the RESPONSE (types.ts:1600), and + // `.with()` binds a constant (C2 measured that). So the mapping is caller-side. + // + // (b) OUTPUT SHAPE. `output` is resolved once at construction. A schema that describes v1 + // rejects v2's body, and a flag cannot change which schema is installed. + let cutover = false; + const v1Shape = (b: unknown): boolean => + typeof (b as { created?: unknown })?.created === 'number'; + const cust = stitch({ + baseUrl: () => (cutover ? `${HOST}/v2` : `${HOST}/v1`), + path: '/customers/{id}', + output: v1Shape, + adapter: vendor.adapter, + name: 'cust', + }); + + const before = await cust.safe({ params: { id: 'cus_7Q2' } }); + check('v1 body passes the v1 output schema', before.ok, true); + cutover = true; + const after = await cust.safe({ params: { id: 'cus_7Q2' } }); + check( + 'the SAME stitch, flag flipped, now fails validation on the v2 body', + after.ok, + false, + ); + note( + 'the flag moved the endpoint and left the contract behind — `output` is fixed at construction', + ); + } + + // ----------------------------------------------------------------------- + heading('C6 (4) — what a flag-driven cutover actually costs'); + { + const vendor = fakeVendor({}); + // Because the input shape and the output contract both change, the honest cutover swaps + // the WHOLE STITCH, not a URL. That is a selector plus a per-version input mapping — and + // it is strictly simpler than a thunk, because each version keeps its own correct schema. + let cutover = false; + const v1 = stitch({ + baseUrl: HOST, + path: '/v1/customers/{id}', + adapter: vendor.adapter, + name: 'cust-v1', + }); + const v2 = stitch({ + baseUrl: HOST, + path: '/v2/customers', + adapter: vendor.adapter, + name: 'cust-v2', + }); + + // >>> BEGIN USER CODE cutover + const getCustomer = (id: string) => + cutover + ? v2({ query: { customer_id: id } }) + : v1({ params: { id } }); + // <<< END USER CODE cutover + + await getCustomer('cus_7Q2'); + checkStr( + 'before the flag', + vendor.pathOf('v1'), + '/v1/customers/cus_7Q2', + ); + cutover = true; + await getCustomer('cus_7Q2'); + checkStr( + 'after the flag', + vendor.pathOf('v2'), + '/v2/customers?customer_id=cus_7Q2', + ); + check( + 'one flag flip, two different stitches, no redeploy', + vendor.log.length, + 2, + ); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const lines = countUserLines(src, 'cutover'); + note('executable lines for the whole-stitch cutover', lines); + check('the cutover is 4 lines or fewer', lines <= 4, true); + } + + console.log(` + WHAT A THUNK CAN AND CANNOT MOVE + + what changes at cutover thunkable? spelling measured + ------------------------ ---------- ------------------------------------- --------------------- + origin YES baseUrl: () => flag ? v2 : v1 retargeted per call + path YES url: () => flag ? urlB : urlA whole path moved, and + {param} still applies + input shape NO — wrong call made + silently, nothing threw + output contract NO — same stitch, flag on, + validation now fails + + So the capture's question — "does it extend to a whole stitch, or only a base URL?" — has a + three-part answer. It extends FURTHER than a base URL (\`url\` is a thunk and carries the path), + and still stops short of a whole stitch, because the two halves of a stitch that a v2 migration + actually changes — how the input maps in and what contract the output is held to — are both fixed + at construction. The working cutover is therefore not a thunk at all: it is a 4-line selector over + two stitches, each keeping its own correct path, input mapping and schema. +`); + + finish( + 'C6', + 'PARTIAL — and the capture UNDERSTATES the thunk while overstating what it buys. A thunk moves more than the base URL: `url` is also `string | (() => string)`, it carries the complete endpoint, and `{param}` interpolation still applies to a thunk-supplied url, so one flag moved `/v1/customers/{id}` to `/v2/customers` between calls with no redeploy. But it cannot move the two things a v2 actually changes. The input mapping is caller-side (measured: after the flag, the v1 input shape against the v2 url silently produced `/v2/customers` with no id and nothing threw), and `output` is resolved once at construction (measured: the same stitch, flag flipped, went from `ok: true` to `ok: false` against a v1-shaped schema). The real cutover is a 4-line selector over two whole stitches', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c7-cost.ts b/docs/scenarios/proofs/dual-run-migration/c7-cost.ts new file mode 100644 index 00000000..21086dd1 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c7-cost.ts @@ -0,0 +1,314 @@ +// C7 — COST. The shadow spends the VENDOR's meter, not yours. Three questions: +// +// (1) does the shadow double rate-limit consumption? +// (2) does a shared `throttle` correctly account for BOTH calls? +// (3) is sampling ("shadow 5% of reads") expressible in config, or is it user code? +// +// TIMING: real clock throughout, deliberately. `manualClock()` DOES drive throttle pacing — which +// is exactly why it is unusable here: nothing advances it, so the first paced call never resolves. +// The rates are chosen small (`'20/s'` = a 50ms gap) so a real-time measurement stays fast, and +// every elapsed figure is reported as a band rather than an exact number. +import { seam } from '../../../../packages/core/src/seam'; +import { stitch } from '../../../../packages/core/src/stitch'; +import { + check, + checkBand, + countUserLines, + finish, + heading, + ledgerRow, + note, + printLedger, +} from './harness'; +import { HOST, elapsed, fakeVendor, seededRandom } from './vendor'; + +import { readFileSync } from 'node:fs'; + +const V1_PATH = '/v1/customers/{id}'; +const V2_PATH = '/v2/customers'; +/** `'20/s'` is a 50ms minimum spacing — the throttle paces, it does not bucket (types.ts:1052-1077). */ +const RATE = '20/s'; +const GAP = 50; + +async function main(): Promise { + // ----------------------------------------------------------------------- + heading('C7 (1) — does the shadow double what the vendor is charged for?'); + { + const vendor = fakeVendor({}); + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }); + + const N = 20; + for (let i = 0; i < N; i++) await v1({ params: { id: `cus_${i}` } }); + const soloTotal = vendor.log.length; + check('20 logical calls, primary only', soloTotal, 20); + ledgerRow( + 'primary only, 20 calls', + vendor.count('v1'), + vendor.count('v2'), + '20 billed', + ); + + vendor.reset(); + for (let i = 0; i < N; i++) { + const shadow = v2.safe({ query: { customer_id: `cus_${i}` } }); + await v1({ params: { id: `cus_${i}` } }); + await shadow; + } + check('20 logical calls, dual-run — v1', vendor.count('v1'), 20); + check('20 logical calls, dual-run — v2', vendor.count('v2'), 20); + check('total requests the vendor received', vendor.log.length, 40); + check('the multiplier', vendor.log.length / soloTotal, 2); + ledgerRow( + 'dual-run 100%, 20 calls', + vendor.count('v1'), + vendor.count('v2'), + '40 billed — 2x', + ); + note( + 'there is no deduplication anywhere: two stitches, two requests, two meter ticks', + ); + } + + // ----------------------------------------------------------------------- + heading('C7 (2) — does a shared `throttle` account for BOTH calls?'); + // The measurement is PACING, in real milliseconds. Four requests through one limiter at a 50ms + // gap take ~150ms; four requests through two independent limiters take ~50ms. The elapsed time + // is therefore a direct read of how many buckets there are. + + // (2a) Two standalone stitches, each with its own `throttle`. The default `pool` is `'stitch'`. + { + const vendor = fakeVendor({}); + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + throttle: RATE, + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + throttle: RATE, + }); + const r = await elapsed(async () => { + await Promise.all([ + v1({ params: { id: 'a' } }), + v2({ query: { customer_id: 'a' } }), + v1({ params: { id: 'b' } }), + v2({ query: { customer_id: 'b' } }), + ]); + }); + note('4 requests, two standalone stitches at 20/s each', `${r.ms}ms`); + checkBand( + 'elapsed ≈ ONE gap — the two versions paced independently', + r.ms, + 0, + GAP + 45, + ); + check('the vendor still received all 4', vendor.log.length, 4); + ledgerRow( + `standalone throttle: '${RATE}' each`, + vendor.count('v1'), + vendor.count('v2'), + `${r.ms}ms — 2 buckets, 40/s at the vendor`, + ); + } + + // (2b) Two members of ONE seam. `seamBucket` re-keys every acquire onto `seam:` + // (seam.ts:51-69), so a seam-level throttle is ONE budget across all members. + { + const vendor = fakeVendor({}); + const vendorSeam = seam({ + baseUrl: HOST, + adapter: vendor.adapter, + throttle: RATE, + }); + const v1 = vendorSeam.stitch({ path: V1_PATH }); + const v2 = vendorSeam.stitch({ path: V2_PATH }); + const r = await elapsed(async () => { + await Promise.all([ + v1({ params: { id: 'a' } }), + v2({ query: { customer_id: 'a' } }), + v1({ params: { id: 'b' } }), + v2({ query: { customer_id: 'b' } }), + ]); + }); + note( + '4 requests, two seam members sharing one 20/s budget', + `${r.ms}ms`, + ); + checkBand( + 'elapsed ≈ THREE gaps — one bucket, correctly counting both', + r.ms, + GAP * 2, + GAP * 5, + ); + check('the vendor still received all 4', vendor.log.length, 4); + ledgerRow( + `seam throttle: '${RATE}'`, + vendor.count('v1'), + vendor.count('v2'), + `${r.ms}ms — 1 bucket, 20/s at the vendor`, + ); + note( + 'a seam accounts for the shadow correctly BY DEFAULT — this is the one cost question the library already answers', + ); + } + + // (2c) `pool: 'host'` — the other way to get one bucket, and the one C1 (d4) measured as the + // trap: it also re-keys the CIRCUIT onto the host. + { + const vendor = fakeVendor({}); + const opts = { rate: RATE, pool: 'host' } as const; + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + throttle: opts, + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + throttle: opts, + }); + const r = await elapsed(async () => { + await Promise.all([ + v1({ params: { id: 'a' } }), + v2({ query: { customer_id: 'a' } }), + v1({ params: { id: 'b' } }), + v2({ query: { customer_id: 'b' } }), + ]); + }); + note( + "4 requests, two standalone stitches with pool:'host'", + `${r.ms}ms`, + ); + checkBand( + "elapsed ≈ THREE gaps — pool:'host' pools across separate stitches", + r.ms, + GAP * 2, + GAP * 5, + ); + ledgerRow( + `standalone pool:'host'`, + vendor.count('v1'), + vendor.count('v2'), + `${r.ms}ms — 1 bucket, and a SHARED BREAKER (C1 d4)`, + ); + note( + "pool:'host' uses a MODULE-LEVEL registry (resilience.ts:84), so it pools across separately-constructed stitches with no shared store", + ); + } + + // ----------------------------------------------------------------------- + heading('C7 (3) — is sampling expressible in config?'); + { + const vendor = fakeVendor({}); + const v1 = stitch({ + baseUrl: HOST, + path: V1_PATH, + adapter: vendor.adapter, + name: 'cust-v1', + }); + const v2 = stitch({ + baseUrl: HOST, + path: V2_PATH, + adapter: vendor.adapter, + name: 'cust-v2', + }); + + // There is no `sample`, `ratio`, `percent`, or `probability` slot anywhere on the config + // surface — C3 enumerated the whole public barrel and the 17 subpaths. The closest thing in + // the tree is `trace`'s sink wiring, which samples nothing. So: user code. + const rand = seededRandom(20260805); + const SAMPLE = 0.05; + // >>> BEGIN USER CODE sampling + const shadowed = (id: string) => { + const primary = v1({ params: { id } }); + if (rand() < SAMPLE) void v2.safe({ query: { customer_id: id } }); + return primary; + }; + // <<< END USER CODE sampling + + const N = 400; + for (let i = 0; i < N; i++) await shadowed(`cus_${i}`); + await new Promise((r) => setTimeout(r, 50)); + + check('every primary call went out', vendor.count('v1'), N); + const sampled = vendor.count('v2'); + note('shadow calls at a 5% sample over 400 primaries', sampled); + checkBand( + 'the sample lands near 5% (deterministic PRNG, seed 20260805)', + sampled, + 8, + 34, + ); + check('total vendor requests', vendor.log.length, N + sampled); + const multiplier = (N + sampled) / N; + note('the multiplier at 5%', multiplier.toFixed(3)); + checkBand( + 'the cost multiplier is near 1.05x, not 2x', + multiplier * 1000, + 1020, + 1085, + ); + ledgerRow( + 'sampled dual-run 5%, 400 calls', + vendor.count('v1'), + sampled, + `${multiplier.toFixed(3)}x — vs 2.000x unsampled`, + ); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const lines = countUserLines(src, 'sampling'); + note('executable lines for sampling', lines); + check('sampling is 5 lines or fewer of user code', lines <= 5, true); + } + + printLedger('C7 — what the vendor was actually charged for'); + + console.log(` + THE COST LEDGER + + question answer + -------------------------------------------- ------------------------------------------------ + does the shadow double consumption? YES, exactly 2.000x. No dedup anywhere. + does a per-stitch throttle account for both? NO — two buckets, so the configured 20/s + became 40/s at the vendor. + does a SEAM throttle account for both? YES, by default. seamBucket re-keys every + acquire onto one seam id (seam.ts:51-69). + does pool:'host' account for both? YES — and it drags the CIRCUIT onto the host + key with it (measured in C1 d4). + is sampling expressible in config? NO. No sample/ratio/percent slot exists. + 5 lines of user code; 2.000x -> 1.05x. + + The sharp edge is that the two ways to make the METER accounting correct are not equivalent. A + seam gets it right and leaves the breaker keyed per path. \`pool: 'host'\` gets it right and + silently re-keys the breaker onto the host, which is the exact configuration C1 measured taking + the primary down. A consumer reaching for \`pool: 'host'\` is reaching for it for a good reason + — the two versions genuinely do share one vendor quota — and gets an unasked-for shared breaker. +`); + + finish( + 'C7', + "MEASURED. The shadow doubles consumption exactly — 20 logical calls became 40 vendor requests, a 2.000x multiplier, with no deduplication anywhere. A per-stitch `throttle` does NOT account for both: two standalone stitches each configured `20/s` put 4 requests through in ~1 gap, so the vendor saw 40/s. Two constructions fix it, and they are not equivalent: a SEAM-level throttle is one bucket by default (seamBucket re-keys every acquire onto one seam id) and leaves the breaker keyed per path, while `pool: 'host'` also pools correctly but drags the CIRCUIT onto the host key with it — the same setting C1 (d4) measured fast-failing the primary on the shadow's breaker. Sampling is not expressible in config: no sample/ratio/percent slot exists on any of the 17 subpaths, and 5 lines of user code took the multiplier from 2.000x to 1.05x", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/c8-assembled.ts b/docs/scenarios/proofs/dual-run-migration/c8-assembled.ts new file mode 100644 index 00000000..3a907c73 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/c8-assembled.ts @@ -0,0 +1,408 @@ +// C8 — ASSEMBLE THE SAFEST DUAL-RUN, and measure what it costs. +// +// Every finding from C1–C7 is a constraint, and this is the one construction that satisfies all of +// them at once. It is replayed side by side against the NAIVE version — the one a reader would +// write from the capture's own description ("issue both, return v1, log the diff") — over the same +// fake vendor, so the difference is a set of counts rather than an argument. +// +// The constraints, and where each came from: +// +// C1 (a) the shadow must not be AWAITED -> `.safe()`, floated +// C1 (b) a shadow failure must not reach the caller -> `.safe()` cannot reject +// C1 (b') a floated shadow must actually RUN -> `.safe()` is eager; `v2(...)` is not +// C1 (c*) a shadow failure must not CANCEL the primary -> no combinator; separate calls +// C1 (d) the shadow's breaker must not be the primary's -> explicit distinct `circuit.key` +// C2 the two calls take different inputs -> a per-version input mapping +// C3 there is no public response-vs-response comparator-> ~20 lines, vendored +// C4 a correct v2 diffs 7 times on every call -> a normalizer, not `ignore` +// C5 a shadowed write charges twice -> `readsOnly` on the SHADOW's adapter +// C7 the shadow doubles the vendor's meter -> a seam-level throttle + sampling +import { diff } from '../../../../packages/core/src/diff'; +import { all } from '../../../../packages/core/src/pipe'; +import { seam } from '../../../../packages/core/src/seam'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { + check, + checkBand, + checkSeq, + countUserLines, + finish, + heading, + ledgerRow, + note, + printLedger, +} from './harness'; +import { + HOST, + REGRESSED_BALANCE, + TRUE_BALANCE, + elapsed, + fakeVendor, + seededRandom, +} from './vendor'; + +import { readFileSync } from 'node:fs'; + +const V1_PATH = '/v1/customers/{id}'; +const V2_PATH = '/v2/customers'; + +/** One reported difference between the two versions, after the relevancy model has run. */ +interface Finding { + path: string; + v1: unknown; + v2: unknown; +} + +async function main(): Promise { + // ======================================================================= + heading( + 'C8 (1) — THE NAIVE DUAL-RUN: "issue both, return v1, log the diff"', + ); + // Written straight from the capture's description, using the combinator that looks + // purpose-built for it. One shared seam, because configuring the vendor once is the obvious + // thing to do, and both versions on `/customers` because that is how this vendor shipped v2. + { + const vendor = fakeVendor({ latency: { v1: 10, v2: 90 } }); + const vendorSeam = seam({ + adapter: vendor.adapter, + circuit: [2, '60s'], + throttle: { pool: 'host' }, + }); + const v1 = vendorSeam.stitch({ + baseUrl: `${HOST}/v1`, + path: '/customers', + }); + const v2 = vendorSeam.stitch({ + baseUrl: `${HOST}/v2`, + path: '/customers', + }); + + const r = await elapsed(() => + all([v1, v2])({ query: { customer_id: 'cus_7Q2' } }), + ); + const raw = r.value as unknown[] | undefined; + note('caller-observed latency', `${r.ms}ms`); + checkBand('the caller waited for the SLOWEST version', r.ms, 80, 200); + check( + "and v1 was handed v2's parameter name", + vendor.pathOf('v1').includes('customer_id'), + true, + ); + const rawDiff = raw ? diff(raw[0], raw[1]).length : -1; + note('raw diff ops the naive version would log, per call', rawDiff); + ledgerRow( + 'NAIVE — one healthy call', + vendor.count('v1'), + vendor.count('v2'), + `${r.ms}ms, ${rawDiff} diff ops`, + ); + + // Now make v2 flaky, which is the entire reason a dual-run exists. + const flaky = fakeVendor({ statuses: { v2: [500, 500, 500, 500] } }); + const flakySeam = seam({ + adapter: flaky.adapter, + circuit: [2, '60s'], + throttle: { pool: 'host' }, + }); + const f1 = flakySeam.stitch({ + baseUrl: `${HOST}/v1`, + path: '/customers', + }); + const f2 = flakySeam.stitch({ + baseUrl: `${HOST}/v2`, + path: '/customers', + }); + const outcomes: string[] = []; + for (let i = 0; i < 4; i++) { + const res = await all([f1, f2])({ query: { customer_id: 'x' } }) + .then(() => 'ok') + .catch((e: Error) => e.message); + outcomes.push(res === 'ok' ? 'ok' : res); + } + checkSeq( + 'four user-facing calls through the naive dual-run', + outcomes, + // The first two calls carry v2's status verbatim; the last two are v1 being + // fast-failed on a breaker only v2 ever opened. + ['HTTP 500', 'HTTP 500', 'circuit open', 'circuit open'], + ); + check( + 'user-facing calls that succeeded', + outcomes.filter((o) => o === 'ok').length, + 0, + ); + check('v1 requests that ever reached the vendor', flaky.count('v1'), 2); + ledgerRow( + 'NAIVE — 4 calls, v2 flaky', + flaky.count('v1'), + flaky.count('v2'), + '0 of 4 succeeded', + ); + note( + "the experiment took down the thing it was protecting — first by propagating v2's error, then by fast-failing v1 on v2's breaker", + ); + } + + // ======================================================================= + heading('C8 (2) — THE SAFE DUAL-RUN'); + + const vendor = fakeVendor({ latency: { v1: 10, v2: 90 } }); + const rand = seededRandom(20260805); + const findings: Finding[] = []; + + // >>> BEGIN USER CODE + /** SEAM 1 — the Adapter. A shadow that cannot emit a non-read cannot double-charge (C5). */ + const readsOnly = + (inner: Adapter): Adapter => + (req) => { + if (req.method !== 'GET' && req.method !== 'HEAD') + throw new Error(`shadow refused a ${req.method} to ${req.url}`); + return inner(req); + }; + + /** SEAM 2 — one `seam`, so ONE throttle bucket spans both versions and the meter adds up (C7). */ + const vendorSeam = seam({ + baseUrl: HOST, + adapter: vendor.adapter, + throttle: '200/s', + }); + /** SEAM 3 — `circuit.key`, distinct per version, so the shadow's breaker is not the primary's (C1 d). */ + const v1 = vendorSeam.stitch({ + path: V1_PATH, + circuit: { failures: 5, cooldown: '30s', key: 'cust-v1' }, + }); + const v2 = vendorSeam.stitch({ + path: V2_PATH, + adapter: readsOnly(vendor.adapter), + circuit: { failures: 5, cooldown: '30s', key: 'cust-v2' }, + }); + + /** SEAM 4 — the relevancy model. Four known-benign changes, normalized onto common ground (C4). */ + const RENAMED: Record = { created: 'created_at' }; + const UNORDERED = new Set(['tags']); + const ADDED = new Set(['livemode']); + const normalize = (b: Record, side: 'v1' | 'v2') => { + const out: Record = {}; + for (const [k, v] of Object.entries(b)) { + if (side === 'v2' && ADDED.has(k)) continue; + const key = side === 'v1' ? (RENAMED[k] ?? k) : k; + out[key] = UNORDERED.has(key) + ? [...(v as unknown[])].sort() + : key === 'created_at' + ? new Date( + typeof v === 'number' ? v * 1000 : (v as string), + ).toISOString() + : v; + } + return out; + }; + + /** SEAM 5 — the dual-run itself. The shadow is sampled, eager, un-awaited and unable to reject. */ + const SAMPLE = 0.25; + const getCustomer = (id: string) => { + const primary = v1({ params: { id } }); + if (rand() < SAMPLE) + void v2 + .safe({ query: { customer_id: id } }) + .then(async (shadow) => { + if (!shadow.ok) return; + const a = normalize( + (await primary) as Record, + 'v1', + ); + const b = normalize( + shadow.data as Record, + 'v2', + ); + for (const d of diff(a, b)) + findings.push({ + path: d.path.join('.'), + v1: d.oldValue, + v2: d.value, + }); + }); + return primary; + }; + // <<< END USER CODE + + // ----------------------------------------------------------------------- + heading('C8 (2a) — latency, isolation, and cost'); + { + const N = 40; + const r = await elapsed(async () => { + for (let i = 0; i < N; i++) await getCustomer(`cus_${i}`); + }); + await new Promise((res) => setTimeout(res, 250)); + + const perCall = r.ms / N; + note( + 'mean caller-observed latency per call', + `${perCall.toFixed(1)}ms`, + ); + checkBand( + "the caller never paid the shadow's 90ms", + perCall * 10, + 80, + 400, + ); + check('every primary call went out', vendor.count('v1'), N); + const shadows = vendor.count('v2'); + note('shadow calls at a 25% sample', shadows); + checkBand('the sample landed near 25%', shadows, 5, 17); + const multiplier = (N + shadows) / N; + note('cost multiplier', multiplier.toFixed(3)); + checkBand( + 'cost is well under the unsampled 2.000x', + multiplier * 1000, + 1100, + 1450, + ); + ledgerRow( + `SAFE — ${N} calls, 25% sample`, + vendor.count('v1'), + shadows, + `${perCall.toFixed(1)}ms/call, ${multiplier.toFixed(3)}x`, + ); + } + + // ----------------------------------------------------------------------- + heading( + 'C8 (2b) — the comparison: does it find the regression and nothing else?', + ); + { + const paths = [...new Set(findings.map((f) => f.path))]; + note( + 'total findings reported across the sampled calls', + findings.length, + ); + checkSeq('distinct paths reported', paths, ['balance_cents']); + check( + 'the regression, as reported (v1)', + findings[0]?.v1, + TRUE_BALANCE, + ); + check( + 'the regression, as reported (v2)', + findings[0]?.v2, + REGRESSED_BALANCE, + ); + note( + 'the rename, the retype, the reorder and the new field produced ZERO findings — 7 raw ops per call became 1', + ); + } + + // ----------------------------------------------------------------------- + heading('C8 (2c) — the safety properties, exercised'); + { + // A flaky v2, replayed through the SAFE construction. Same failure, same seam, same host. + const flaky = fakeVendor({ + statuses: { v2: [500, 500, 500, 500, 500, 500] }, + }); + const s = seam({ + baseUrl: HOST, + adapter: flaky.adapter, + throttle: '200/s', + }); + const p = s.stitch({ + path: V1_PATH, + circuit: { failures: 2, cooldown: '30s', key: 'cust-v1' }, + }); + const sh = s.stitch({ + path: V2_PATH, + adapter: readsOnly(flaky.adapter), + circuit: { failures: 2, cooldown: '30s', key: 'cust-v2' }, + }); + const results: string[] = []; + for (let i = 0; i < 4; i++) { + void sh.safe({ query: { customer_id: 'x' } }); + const r = await p.safe({ params: { id: 'x' } }); + results.push(r.ok ? 'ok' : (r.error as Error).message); + } + await new Promise((res) => setTimeout(res, 60)); + checkSeq( + 'four user-facing calls with the shadow failing every time', + results, + ['ok', 'ok', 'ok', 'ok'], + ); + check('v1 requests that reached the vendor', flaky.count('v1'), 4); + check( + 'the shadow tripped its OWN breaker', + flaky.count('v2') < 4, + true, + ); + note('shadow requests before its breaker opened', flaky.count('v2')); + ledgerRow( + 'SAFE — 4 calls, v2 flaky', + flaky.count('v1'), + flaky.count('v2'), + '4 of 4 succeeded', + ); + + // A shadowed WRITE, attempted through the same construction. + const writeShadow = s.stitch({ + path: '/v2/charges', + method: 'POST', + adapter: readsOnly(flaky.adapter), + circuit: { failures: 5, cooldown: '30s', key: 'charge-v2' }, + }); + const w = await writeShadow.safe({ body: { amount: 4200 } }); + check('the shadowed write failed', w.ok, false); + check( + 'shadow WRITES the vendor received', + flaky.log.filter((c) => c.version === 'v2' && c.method !== 'GET') + .length, + 0, + ); + } + + printLedger('C8 — naive vs safe, over the same vendor'); + + const src = readFileSync(new URL(import.meta.url), 'utf8'); + const userLines = countUserLines(src); + note('EXECUTABLE LINES OF USER CODE', userLines); + + console.log(` + THE FIVE SEAMS, AND WHAT EACH BUYS + + seam spelling closes + ------------------------- ---------------------------------------- -------------------------- + 1 the Adapter readsOnly(adapter) on the SHADOW only C5 — a shadowed write is + impossible, not discouraged + 2 the seam seam({ throttle: '200/s' }) C7 — ONE bucket spans both + versions, so the vendor's + meter adds up + 3 circuit.key distinct string per version C1 (d) — the shadow's + breaker is not the primary's + 4 the relevancy model normalize() — user code, no library seam C4 — 7 raw ops/call -> 1 + 5 the call site v2.safe(...) floated, never awaited C1 (a)(b)(b')(c*) — off the + critical path, cannot reject, + cannot cancel, and DOES run + + SAME VENDOR, SAME FLAKY v2, FOUR USER-FACING CALLS + + naive (all([v1, v2]), one seam, shared key) 0 of 4 succeeded + safe (this construction) 4 of 4 succeeded + + WHAT IT COSTS + + ${userLines} executable lines of user code, of which the largest single block is the relevancy model + (${countUserLines(src)} total; the normalizer alone is roughly half). Four of the five seams are + ordinary config — an adapter wrapper, a seam, two circuit keys, a call site. The fifth is not + config at all: there is no declarative surface anywhere in the library for "these two field + names mean the same thing" or "compare this array unordered", so the relevancy model is code you + write and maintain, and it is exactly the part the field guidance says the effort goes into. + + Three things NO amount of user code fixes, and they bound the technique rather than the library: + - a write can only be REFUSED, never shadowed (C5); + - the shadow's correctness depends on the caller mapping the input twice, and getting it + wrong is silent — C2 measured a bound shadow querying the wrong customer forever; + - the comparator is vendored, so a library-side improvement to \`diff\` never reaches it (C3). +`); + + finish( + 'C8', + `ASSEMBLED, AND THE SAFE CONSTRUCTION IS ${userLines} EXECUTABLE LINES ACROSS 5 SEAMS. Replayed against the naive version over the same vendor and the same flaky v2, the numbers are 0-of-4 versus 4-of-4 user-facing calls succeeding: the naive dual-run — \`all([v1, v2])\` under one seam with both versions on \`/customers\` — propagated v2's 500 to the caller twice, then fast-failed v1 on v2's breaker twice, and along the way put a 90ms shadow on a 10ms call's critical path and sent v2's parameter name to v1. The safe construction needed no fork and no new config key: \`readsOnly\` on the SHADOW's adapter only (0 shadow writes reached the wire while the primary still wrote), a seam-level throttle (one bucket, so the meter adds up), distinct \`circuit.key\` strings (the shadow tripped its own breaker and the primary never noticed), a hand-written normalizer (7 raw diff ops per call became exactly 1 — the planted \`balance_cents\` regression, reported with both values), and a floated \`.safe()\` at the call site, which is the one spelling that is simultaneously off the critical path, unable to reject, unable to cancel the primary, and actually eager enough to run. The irreducible cost is the relevancy model: no declarative surface for field aliasing or unordered comparison exists anywhere in the library, so that block is user code by construction`, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/dual-run-migration/harness.ts b/docs/scenarios/proofs/dual-run-migration/harness.ts new file mode 100644 index 00000000..dd28ddd2 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/harness.ts @@ -0,0 +1,248 @@ +// Assertion + measurement harness for the `dual-run-migration` proofs. Every check prints a line and +// the script exits non-zero if any check failed. No test framework — these are standalone `tsx` +// scripts, exactly like the other proof directories. +// +// What this directory needs that the others did not: a CHANNEL TABLE and a REQUEST LEDGER. +// +// - Every claim here reduces to "how many requests did the vendor actually receive, and what was +// in them?" — so the primitive is a count and a literal URL string taken from the fake +// transport, never a belief about what the config should have done. `ledgerRow` records one +// measured configuration; `printLedger` prints the matrix. +// - C1 asks a four-part question ("can the shadow hurt the primary?") whose answer is a table: +// channel × safe-by-default × what it takes. `channelRow` / `printChannels` build exactly that. +// +// TIMING NOTE, stated once and inherited by every script here. Scenario 19 measured that +// `manualClock()` does NOT drive `timeout.total`, `cache.ttl`, event `at` / `done.elapsed`, OAuth2 +// expiry, or SigV4. Two consequences for this directory: +// +// - C1 (a) measures CALLER-OBSERVED LATENCY, which is wall-clock by definition. It uses +// `performance.now()` and a real `setTimeout` inside the fake adapter. Every latency number in +// this directory is REAL TIME, and is reported in a band rather than as an exact figure. +// - C1 (d) measures CIRCUIT COOLDOWN, which `manualClock` DOES drive (retry backoff, throttle +// pacing, per-attempt timeout, circuit cooldown). Those scripts inject a `manualClock` and say so. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides several rows. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v.toString()}n`; + if (typeof v === 'string') return JSON.stringify(v); + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** Assert an exact string match, printing the measured string. For URLs and error messages. */ +export function checkStr( + label: string, + actual: string, + expected: string, +): void { + checks++; + const ok = actual === expected; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${JSON.stringify(actual)}${ok ? '' : ` (expected ${JSON.stringify(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence prints in full whether it passes or fails — a request ledger IS the evidence for several + * rows here. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * Assert a measured number falls in an inclusive BAND. The only honest assertion shape for a + * wall-clock latency: the measurement is real time, so an exact equality would be a flake, and a + * bare `note` would let the reader supply the verdict. Prints the measured number either way. + */ +export function checkBand( + label: string, + actual: number, + lo: number, + hi: number, +): void { + checks++; + const ok = actual >= lo && actual <= hi; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${actual}${ok ? '' : ` (expected ${lo}..${hi})`}`, + ); +} + +/** Assert a substring is present — for an error message whose prefix is the load-bearing part. */ +export function checkHas( + label: string, + haystack: string, + needle: string, +): void { + checks++; + const ok = haystack.includes(needle); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: ${ok ? 'present' : 'ABSENT'} in ${JSON.stringify(haystack)}${ok ? '' : ` (wanted ${JSON.stringify(needle)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +// ---- the request ledger ---------------------------------------------------- +// One row per measured CONFIGURATION: how many requests each version's endpoint actually received. +// The counts come from the fake transport (vendor.ts), which is the only thing in the process that +// can see a request — so a row is a measurement of the wire, not of the config's intent. + +export interface LedgerRow { + config: string; + v1: number; + v2: number; + /** What the caller observed — a value, an error class, a count. Free text. */ + caller: string; +} + +const ledgerRows: LedgerRow[] = []; + +export function ledgerRow( + config: string, + v1: number, + v2: number, + caller: string, +): LedgerRow { + const row: LedgerRow = { config, v1, v2, caller }; + ledgerRows.push(row); + return row; +} + +export function printLedger(title = 'requests the vendor received'): void { + if (ledgerRows.length === 0) return; + const w = Math.max(...ledgerRows.map((r) => r.config.length), 6); + const c = Math.max(...ledgerRows.map((r) => r.caller.length), 6); + console.log( + `\n ${title}\n\n ${'configuration'.padEnd(w)} ${'v1'.padStart(4)} ${'v2'.padStart(4)} ${'caller saw'.padEnd(c)}\n` + + ` ${'-'.repeat(w)} ${'-'.repeat(4)} ${'-'.repeat(4)} ${'-'.repeat(c)}`, + ); + for (const r of ledgerRows) + console.log( + ` ${r.config.padEnd(w)} ${String(r.v1).padStart(4)} ${String(r.v2).padStart(4)} ${r.caller.padEnd(c)}`, + ); +} + +export function resetLedger(): void { + ledgerRows.length = 0; +} + +// ---- the C1 channel table -------------------------------------------------- +// Four ways a shadow can hurt a primary. Each row is filled from a MEASUREMENT in this directory, +// and carries the measurement that decided it so the table is auditable from its own output. + +export interface ChannelRow { + /** `(a) latency`, `(b) thrown`, … — the capture's own labels. */ + channel: string; + /** Is the primary safe with NO extra configuration? */ + safeByDefault: boolean; + /** The number/string that decided the cell. */ + measured: string; + /** What it takes to make it safe (or `'—'` when it already is). */ + fix: string; +} + +const channelRows: ChannelRow[] = []; + +export function channelRow(row: ChannelRow): ChannelRow { + channelRows.push(row); + return row; +} + +export function printChannels(): void { + if (channelRows.length === 0) return; + const w = Math.max(...channelRows.map((r) => r.channel.length), 7); + const m = Math.max(...channelRows.map((r) => r.measured.length), 8); + console.log( + `\n C1 — the four channels a shadow can hurt the primary through\n\n` + + ` ${'channel'.padEnd(w)} safe? ${'measured'.padEnd(m)} what it takes\n` + + ` ${'-'.repeat(w)} ----- ${'-'.repeat(m)} -------------`, + ); + for (const r of channelRows) + console.log( + ` ${r.channel.padEnd(w)} ${(r.safeByDefault ? 'YES' : 'NO').padEnd(5)} ${r.measured.padEnd(m)} ${r.fix}`, + ); +} + +// ---- line counting --------------------------------------------------------- +// C2 and C8 report a COST in lines. Counting them by hand invites flattery, so the scripts count +// the real thing: executable lines in a marked region of a file on disk. + +/** + * Count EXECUTABLE lines between `>>> BEGIN USER CODE` and `<<< END USER CODE` in a source file — + * blank lines and comment-only lines excluded. This is the honest denominator for "what does the + * working spelling cost": it counts what a reader would have to write and maintain, not the prose + * explaining it. + */ +export function countUserLines(source: string, marker = ''): number { + const lines = source.split('\n'); + const begin = `>>> BEGIN USER CODE${marker ? ' ' + marker : ''}`; + const end = `<<< END USER CODE${marker ? ' ' + marker : ''}`; + let inside = false; + let count = 0; + for (const raw of lines) { + const line = raw.trim(); + if (!inside) { + if (line.includes(begin)) inside = true; + continue; + } + if (line.includes(end)) break; + if (line === '') continue; + if (line.startsWith('//') || line.startsWith('*') || line === '*/') + continue; + count++; + } + return count; +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a HAZARD. The verdict statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/dual-run-migration/vendor.ts b/docs/scenarios/proofs/dual-run-migration/vendor.ts new file mode 100644 index 00000000..1e9b8e35 --- /dev/null +++ b/docs/scenarios/proofs/dual-run-migration/vendor.ts @@ -0,0 +1,276 @@ +// The fake vendor every claim in this directory calls. Two API versions on ONE host, because that +// is the situation: `api.vendor.test` is retiring `/v1` and you are the consumer, so both the +// primary and the shadow spend the SAME meter, hit the SAME breaker key material, and share the +// SAME origin. A fake pointing v2 at a second host would have quietly dissolved half the scenario. +// +// Everything is in-memory. The transport is a plain `Adapter` — `(req) => Promise` — which is +// the only thing in the process that can observe a request, so every count in this directory is +// taken from `log` here rather than from a config's stated intent. +// +// WHAT THE TWO VERSIONS DISAGREE ABOUT (C2 and C4 both live off this): +// +// input v1: GET /v1/customers/{id} — the id is a PATH parameter +// v2: GET /v2/customers?customer_id= — the id moved to a QUERY parameter, renamed +// output v1: `created` epoch int, `tags` in one order, no `livemode` +// v2: `created_at` ISO string, `tags` reordered, `livemode` added +// ... and ONE genuine regression planted in `balance_cents` (C4). +// +// The regression is a transposition (41250 -> 41520), not a null or a missing field: a wrong VALUE +// of the right type at the right path is the diff a schema cannot catch and the one a dual-run +// exists to find. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +export const HOST = 'https://api.vendor.test'; + +// ---- the payloads ---------------------------------------------------------- + +/** + * The epoch second v1 reports, and the instant v2 spells as ISO. THE SAME MOMENT, two wire types — + * which is the whole point of the retype: a correct v2 changes the encoding and not the value, so + * any comparator that flags this is producing noise. (`1785888000 === Date.parse(CREATED_ISO)/1000`; + * C4 asserts the round-trip rather than trusting the pair.) + */ +export const CREATED_EPOCH = 1_785_888_000; +export const CREATED_ISO = '2026-08-05T00:00:00.000Z'; + +/** What the customer's balance really is. v1 reports it correctly. */ +export const TRUE_BALANCE = 41_250; +/** What v2 reports — two digits transposed. THE regression this whole technique exists to catch. */ +export const REGRESSED_BALANCE = 41_520; + +export function v1Customer(id = 'cus_7Q2'): Record { + return { + id, + name: 'Wilhelmina Ashcombe', + created: CREATED_EPOCH, + balance_cents: TRUE_BALANCE, + currency: 'gbp', + tags: ['enterprise', 'eu', 'invoiced'], + address: { + line1: '4 Ashcombe Mews', + city: 'London', + postal: 'EC1A 1BB', + }, + }; +} + +export function v2Customer(id = 'cus_7Q2'): Record { + return { + id, + name: 'Wilhelmina Ashcombe', + // RENAME + RETYPE: `created` (epoch int) became `created_at` (ISO string). + created_at: CREATED_ISO, + // THE REGRESSION. Right path, right type, wrong number. + balance_cents: REGRESSED_BALANCE, + currency: 'gbp', + // REORDER: the same three tags, a different order. + tags: ['eu', 'invoiced', 'enterprise'], + address: { + line1: '4 Ashcombe Mews', + city: 'London', + postal: 'EC1A 1BB', + }, + // NEW FIELD. + livemode: true, + }; +} + +/** A v2 response with the regression corrected — the "diff is quiet, cut over" end state. */ +export function v2CustomerFixed(id = 'cus_7Q2'): Record { + return { ...v2Customer(id), balance_cents: TRUE_BALANCE }; +} + +// ---- the request ledger ---------------------------------------------------- + +export interface VendorCall { + /** `'v1'`, `'v2'`, or `'?'` when the URL matched neither prefix. */ + version: 'v1' | 'v2' | '?'; + method: string; + /** The full URL the transport was handed, query string included. */ + url: string; + /** Path + query only — the part a URL assertion should read. */ + pathQuery: string; + body?: unknown; + /** Wall-clock ms since the vendor was created, for ordering. */ + at: number; + /** + * Did this request run to completion, or was it ABORTED in flight? The fake honours + * `req.signal` during its latency wait, so a combinator that auto-cancels its losers is + * measurable here rather than inferred. `false` until the response is returned. + */ + completed: boolean; + /** Set when the request was cut short by `req.signal`. C1 (c) reads this. */ + aborted?: boolean; +} + +export interface VendorOptions { + /** Real milliseconds this version's endpoint takes to answer. Default 0. */ + latency?: { v1?: number; v2?: number }; + /** + * Answer v2 reads with the CORRECTED body ({@link v2CustomerFixed}) instead of the regressed + * one — the "diff has gone quiet, cut over" end state. C8 measures both ends with one vendor. + */ + v2Fixed?: boolean; + /** + * Statuses to answer with, consumed in order, per version. `[500, 500, 200]` fails twice then + * succeeds. Anything past the end of the list answers 200. A `0` means "reject at the transport" + * (a connection error) rather than answer with a status. + */ + statuses?: { v1?: number[]; v2?: number[] }; +} + +export interface FakeVendor { + adapter: Adapter; + /** Every request the transport actually received, in order. */ + log: VendorCall[]; + /** How many requests this version's endpoints received. */ + count(version: 'v1' | 'v2'): number; + /** Requests of a given method this version received — the C5 measurement. */ + countMethod(version: 'v1' | 'v2', method: string): number; + /** The path+query of the Nth request to a version, for a literal URL assertion. */ + pathOf(version: 'v1' | 'v2', n?: number): string; + reset(): void; +} + +/** + * Build the fake vendor. The adapter never throws on a non-2xx (ADR 0005) — it answers with the + * status, which is what a real transport does and what the engine's retry/circuit stages read. The + * one exception is a scripted `0`, which rejects: a connection error is a different failure mode + * and C1 (d) needs both. + */ +export function fakeVendor(opts: VendorOptions = {}): FakeVendor { + const log: VendorCall[] = []; + const started = Date.now(); + const pending: Record<'v1' | 'v2', number[]> = { + v1: [...(opts.statuses?.v1 ?? [])], + v2: [...(opts.statuses?.v2 ?? [])], + }; + + const adapter: Adapter = async (req: AdapterRequest) => { + const u = new URL(req.url); + const version: 'v1' | 'v2' | '?' = u.pathname.startsWith('/v1') + ? 'v1' + : u.pathname.startsWith('/v2') + ? 'v2' + : '?'; + const entry: VendorCall = { + version, + method: req.method, + url: req.url, + pathQuery: u.pathname + u.search, + body: req.body, + at: Date.now() - started, + completed: false, + }; + log.push(entry); + + const wait = + version === 'v1' + ? (opts.latency?.v1 ?? 0) + : version === 'v2' + ? (opts.latency?.v2 ?? 0) + : 0; + // REAL time. Caller-observed latency is wall-clock by definition (see harness.ts). + // The wait honours `req.signal` so an auto-cancelled member is MEASURED (`aborted: true`) + // rather than assumed — a combinator that aborts its losers is a C1 (c) finding. + if (wait > 0) + await new Promise((resolve, reject) => { + const t = setTimeout(resolve, wait); + req.signal?.addEventListener( + 'abort', + () => { + clearTimeout(t); + entry.aborted = true; + reject(new Error('aborted by signal')); + }, + { once: true }, + ); + }); + + const scripted = + version === '?' ? undefined : (pending[version].shift() ?? 200); + if (scripted === 0) + throw new Error(`ECONNRESET ${version} ${u.pathname}`); + + const status = scripted ?? 200; + if (status !== 200) + return { + status, + headers: { 'content-type': 'application/json' }, + body: { error: `vendor ${version} says ${status}` }, + } satisfies AdapterResponse; + + const id = String( + u.searchParams.get('customer_id') ?? + u.pathname.split('/').pop() ?? + 'cus_7Q2', + ); + // A write answers with a receipt; a read answers with the version's customer shape. + const body = + req.method === 'GET' || req.method === 'HEAD' + ? version === 'v2' + ? opts.v2Fixed + ? v2CustomerFixed(id) + : v2Customer(id) + : v1Customer(id) + : { ok: true, charged: true, version, received: req.body }; + entry.completed = true; + return { + status: 200, + headers: { 'content-type': 'application/json' }, + body, + } satisfies AdapterResponse; + }; + + return { + adapter, + log, + count: (v) => log.filter((c) => c.version === v).length, + countMethod: (v, m) => + log.filter((c) => c.version === v && c.method === m).length, + pathOf: (v, n = 0) => + log.filter((c) => c.version === v)[n]?.pathQuery ?? '', + reset() { + log.length = 0; + pending.v1 = [...(opts.statuses?.v1 ?? [])]; + pending.v2 = [...(opts.statuses?.v2 ?? [])]; + }, + }; +} + +/** + * Wall-clock elapsed of an async thunk, in whole milliseconds. Real time, deliberately. + * + * The parameter is `PromiseLike`, not `Promise`, because a bare stitch call returns a + * `StitchResult` — a lazy `PromiseLike` (types.ts:1905) that lacks `Symbol.toStringTag` and so is + * not assignable to `Promise`. That distinction is itself a C1 (b) finding, not a typing nuisance. + */ +export async function elapsed( + fn: () => PromiseLike, +): Promise<{ ms: number; value?: T; error?: unknown }> { + const t0 = performance.now(); + try { + const value = await fn(); + return { ms: Math.round(performance.now() - t0), value }; + } catch (error) { + return { ms: Math.round(performance.now() - t0), error }; + } +} + +/** A deterministic PRNG so C7's sampling measurement is a number, not a coin flip. */ +export function seededRandom(seed = 1): () => number { + let s = seed >>> 0; + return () => { + // xorshift32 — small, deterministic, and good enough to sample a percentage. + s ^= s << 13; + s >>>= 0; + s ^= s >> 17; + s ^= s << 5; + s >>>= 0; + return s / 0x1_0000_0000; + }; +} diff --git a/docs/scenarios/proofs/expiring-signatures/README.md b/docs/scenarios/proofs/expiring-signatures/README.md new file mode 100644 index 00000000..5ad6f779 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/README.md @@ -0,0 +1,209 @@ +# Proofs — the signature that expired in your own queue + +Runnable evidence for the claims in [`../../expiring-signatures.md`](../../expiring-signatures.md). + +**The scenario's answer is a number, and it is 0 milliseconds.** Four calls behind +`throttle: { rate: '1/2m' }`, granted at 0, 2, 4 and 6 virtual minutes: every one arrived carrying a +signature aged **0ms**, including the one that waited six minutes — past the five-minute window. The +same four calls signed once before being enqueued measured **0 / 2 / 4 / 6 minutes** and a **403 +RequestTimeTooSkewed** on the last. botocore#149 is not present in this library, and the control +proves the instrument would have found it. + +That is the deciding claim (C2), and it goes the library's way. So do C1, C3, C4 and the default half +of C6. Two things do not: the shipped signer ignores the injected clock (C5), and a skew 403 is +counted as a circuit failure while the pure-config way to reclassify it **swallows** it (C6). + +Every script is standalone and offline. The measurement is always the same one — the **age of the +signature on arrival**: the gap between the instant `x-amz-date` claims and the instant the request +reached the transport. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c2-throttle-rate.ts + +# all of them +for f in docs/scenarios/proofs/expiring-signatures/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core and `@stitchapi/aws-sigv4` from +`packages/*/src` by relative path, so they test the working tree, not the published bundles. The +whole suite takes about ten seconds; the two claims that use real time (C1 (c), C2 (c)) account for +most of it. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/expiring-signatures/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `c1-sign-per-attempt.ts` | signed per attempt, or once per call? | **Per attempt.** 3 attempts 6 min apart → 3 signatures, ages `[0,0,0]`. Signed once → `[0,6min]` and a 403 | +| `c2-throttle-rate.ts` | **DECIDING** — does the rate wait happen before signing? | **Before.** 6-min queue → age **0ms**. Control → `[0,2,4,6]` min + 403. **But `hooks.onRequest` runs after** | +| `c3-throttle-concurrency.ts` | same, for a busy concurrency pool | **Same answer**, and for both limiters stacked. 6 min behind the pool → age 0ms | +| `c4-circuit-cooldown.ts` | does a breaker cooldown hold a signed request? | **It holds nothing** — 3 fast-failed calls performed **0 signings**. Half-open trial after 6 min: age 0ms | +| `c5-clock-source.ts` | injected clock, or `Date.now()`? | **`Date.now()`.** 600 virtual seconds moved the stamp **0s**; a clock-reading signer moved **600s** | +| `c6-skew-403-classification.ts` | is a skew 403 retried? classifiable without swallowing? | **Not retried (good).** But it **opens the circuit**, and `verdict: { accept, flag }` **succeeds on it** | +| `c7-skew-correction.ts` | is there a seam for AWS-style skew correction? | **Yes — `shouldRefresh`/`refresh`.** Learned 600000ms from the `Date` header, re-signed to a 200, **free** | +| `c8-assembled.ts` | all three failure modes at once | **4 of 4 succeeded**, worst age 0ms, breaker never opened — **26 lines** of user code, all for the drift half | + +## Files + +- `fake-aws.ts` — the AWS-ish server, as a plain `Adapter`. It parses `x-amz-date` off the wire, + compares it against its OWN clock (the client's clock plus a configurable `skewMs` — failure mode + 1 as one number), and rejects outside a five-minute window with the S3 `RequestTimeTooSkewed` + envelope and a `Date` header. Every request is recorded with `signedAt`, `arrivedAt`, `ageMs` and + `skewMs`; **`ages()` is the spine nearly every claim asserts on**. +- `signers.ts` — the two instruments, and the control. `stampedSigV4` wraps the **shipped** + `awsSigV4` and brackets its `apply` with wall-clock reads. `clockSigV4` is ~20 lines that mint the + timestamp from an injected `Clock` and hand it to the package's own exported `signRequestV4`. + `presignedSigV4` + `presign` are the **control**: headers computed once, before the calls are + enqueued — sign-then-queue, expressed in this library. +- `virtual-time.ts` — `runOut`, and the reason it exists. `manualClock.advance` drains microtasks + between timer fires; `crypto.subtle` is genuinely async and settles on the macrotask queue several + turns deep. Without extra drains the virtual clock jumps while real crypto is still running and + the ledger reports an age that is **pure artifact, in the library's disfavour** — measured at + 120000ms for a request signed at the last possible moment. +- `harness.ts` — `check` / `checkSeq` / `checkAtMost` / `checkAtLeast` / `note` / `heading` / + `finish`. No test framework. + +## Reading the numbers honestly + +- **C2 is the finding, and it is a good one.** `acquireWithin` is at engine.ts:629 and + `cfg.auth.apply` at engine.ts:649 — **the wait is above the signing, inside the attempt loop**. A + call queued six virtual minutes behind `rate: '1/2m'` arrived with a **0ms-old** signature and a + 200; the server measured **0ms of skew** against a 300000ms window. The same measurement on the + **real clock with the real `awsSigV4`** across a 2.4-second queue: the worst **sign→wire gap was + 2ms**. This is the property the capture hoped for, and it should be advertised: _a StitchAPI + throttle cannot expire a signature._ +- **The control is what makes that a measurement.** Four calls pre-signed at t0 through the same + fake server measured ages of **0 / 2 / 4 / 6 minutes**, one distinct signature across four + requests, and a **403** on the last. The instrument detects botocore#149; the library does not have + it. +- **C1: `cloneReq` is why, and it is stronger than "auth re-runs".** Each attempt gets a fresh + header object copied from the UNSIGNED base request (engine.ts:261-264,646), so a previous + attempt's `x-amz-date` cannot survive even by accident. Three attempts six minutes apart: + `["…T120000Z","…T120600Z","…T121200Z"]`, ages `[0,0,0]`. A server-directed **`Retry-After: 600`** + parked the call for ten minutes and attempt 2 still arrived fresh. +- **A quiet piece of protection nobody documents: `backoff.max` defaults to 10 seconds** + (resilience.ts:47,56). `base: '6m'` alone yields a **10-second** wait — measured, and it cost this + proof a false negative before the `max` was set explicitly. A COMPUTED backoff therefore cannot + park a call long enough to expire a signature. `Retry-After` can: it skips `backoffDelay` entirely + and is unbounded by design (engine.ts:743-767). +- **C4 reframes the capture's question.** The breaker does not queue a signed request — it fast-fails + BEFORE the attempt loop (engine.ts:863,871), so the throttle and `auth.apply` are never reached. + Across five calls, **three fast-failed and performed 0 signings**. There is no held signature to + expire because nothing was signed. The half-open trial admitted after a six-minute cooldown carried + the post-cooldown timestamp and an age of 0ms. +- **C5 is the third instance of one inconsistency, and it is a TESTABILITY defect, not a wire one.** + `awsSigV4` stamps `amzDateOf(new Date())` (aws-sigv4/src/index.ts:301) and the package imports no + `Clock` at all. Advancing a `manualClock` 600 virtual seconds between two signings moved the stamp + **0 seconds**. On the real clock it is correct — the skew the server measured was **538ms**, all of + it `x-amz-date`'s one-second resolution. But it means **a SigV4 stitch cannot be tested on a + virtual clock**: under a default `manualClock()` (which starts at epoch 0) **0 of 3 calls were + accepted**, ~20670 days of apparent skew, purely from the test harness. +- **That is also why C1–C4 are each measured twice.** A virtual queue is invisible to a signer on + wall time. So each ordering claim runs once with `clockSigV4` at virtual intervals large enough to + cross the five-minute window, and once with the SHIPPED strategy on the real clock at intervals + small enough to finish in seconds. Both instruments enter at the same seam (`cfg.auth.apply`) and + both agree. +- **C6 (a) is right by default and worth keeping.** `retry.on` defaults to `[429,502,503,504]` + (engine.ts:612), so a 403 with `retry: { attempts: 4 }` produced **one** request. The failure retry + cannot fix is not retried. +- **C6 (b) is the sharp restatement of why per-attempt signing is not enough.** With a host ten + minutes behind and 403 added to `retry.on`, four attempts produced **four distinct signatures and + four identical skews of 600000ms**. Re-signing faithfully re-mints the same wrong time. Ordering + solves the queue; nothing but a corrected clock solves drift. +- **C6 (c): a skew 403 opens the circuit.** It throws at engine.ts:824-831 and `attemptWithCircuit` + counts it (engine.ts:879-891). Measured `["403","403","503","503"]` — after two failures the page + says **`circuit open` / 503**, a dependency outage, for a fault entirely inside this process. +- **C6 (d) refutes the obvious fix, and this is the most dangerous single result in the set.** + `verdict: { accept: [403], flag: 'ok' }` — the pure-config classification scenario 9 measured + working for a 401 — **swallows** the skew error. The call returned **`ok: true`** and handed the + caller `{"Error":{"Code":"RequestTimeTooSkewed",…}}` **as its data**. `verdict.flag` is three-state + and an ABSENT flag is explicitly "no signal" (surface.ts:180-190); AWS error bodies have no `ok` + field, so the flag never fires and `accept` alone succeeds on the 403. **The scenario-9 recipe does + not transfer to AWS.** What works is **6 lines** of `Surface.interpret` composing `verdictOf`: a + real error for the caller, `4 of 4` requests reaching the wire, breaker never tripped. +- **C7 finds a real seam, and the capture guessed it was missing.** `AuthStrategy.shouldRefresh` / + `refresh` (types.ts:1273-1274, engine.ts:707-725) was built for "the token expired, get a new one + and redo this attempt", and a stale clock is that story with a different noun. A drifting host + measured: 403 → offset **600000ms** learned from the response's `Date` → **same attempt redone** + with a corrected clock → 200. `attempt--` (engine.ts:723) means it is **free**: a stitch with + `retry: { attempts: 1 }` still got its second request. The correction persists across calls (call 2 + needed **1** request), and the `refreshed` latch (engine.ts:615,709) keeps it to one per run, so an + uncorrectable clock fails rather than loops. +- **But `refresh` cannot see the response.** It is handed an `AuthContext` and nothing else, so the + `Date` header has to be captured by `shouldRefresh` — the only auth hook that receives the + response — and smuggled across through a closure. It works and it is six lines, but it is not what + the seam looks like it is for. +- **The alternative seam is worse.** `hooks.onResponse` also sees the response and can learn the + offset, but a hook cannot ask for another attempt — so it only lands if 403 is in `retry.on`, which + re-arms C6 (b) for every genuinely-bad-credential 403. Measured working, at that price. +- **C8: 26 lines, in two declarations, and all of it for the drift half.** Against a ten-minute host + drift, a six-minute rate-limited queue and a breaker: **4 of 4 calls succeeded**, worst signature + age **0ms**, exactly **one** 403 (the probe that taught the offset), breaker never opened. The same + workload with no user code: `["403","403","503","503"]`. **Nothing in those 26 lines is about the + queue or the retry** — those needed no code at all. +- **A skew correction costs a second rate slot.** In C8 the succeeding calls were granted at + **2/4/6/8** virtual minutes, not 0/2/4/6: the correction probe took t=0 and its corrected re-sign + took t=2m, because `attempt--; continue` re-enters the loop at engine.ts:624 and re-acquires the + throttle. + +## The footguns + +- **`hooks.onRequest` runs AFTER signing** (engine.ts:652 vs 649) and is the **only** user code that + does. A six-minute wait inside it aged the signature by exactly six minutes and produced a **403** — + measured. Anyone pacing calls with a hand-rolled gate in `onRequest`, because `throttle` could not + express their rule, has re-created botocore#149 inside a library that does not have it. Nothing + warns, and the config reads like a throttle. +- **`verdict: { accept: [403], flag: 'ok' }` does not classify an AWS skew error — it SUCCEEDS on + it.** The caller receives `RequestTimeTooSkewed` as data and the call reports `ok: true`. This is + worse than not classifying at all, and it is the recipe an earlier scenario published for a 401. + The flag needs a body field that is _present and falsy_; AWS envelopes have none. +- **A skew 403 counts as a circuit failure, so a wrong local clock reads as a vendor outage.** The + operator sees `circuit open` / 503 for a fault inside their own process, and the breaker's + half-open probe then reports `RequestTimeTooSkewed` rather than whatever real fault opened it + (measured in C4 (c)) — the recovery path reports the wrong cause. +- **A SigV4 stitch cannot be tested on a `manualClock`.** The signer is on wall time and everything + around it is on virtual time, so any fake that validates the timestamp rejects everything — + measured, **0 of 3**, ~20670 days of apparent skew. Anyone writing that test will conclude their + signing is broken. The fix is a signer that takes a `Clock`; ~20 lines, in `signers.ts`. +- **`retry: { on: [403, …] }` turns an unfixable failure into a budget burn.** Four attempts, four + fresh signatures, four identical 600000ms skews. If a 403 must be retried for other reasons, + exclude the skew code. +- **`backoff.max` defaults to 10 seconds**, so a long `base` is silently clamped — `base: '6m'` waits + ten seconds. Protective here, surprising everywhere else. +- **A `circuit` does not shed a burst already queued behind a `throttle`.** `circuit.phase()` is read + at engine.ts:863, before `attemptLoop` reaches the throttle at 629, so every call in a concurrent + burst clears the breaker at enqueue time. Measured: four calls fired together all reached the wire + over six minutes, long after the first two failures had opened the breaker — where the same four + calls made sequentially stopped after two. +- **Pre-signing outside the engine forfeits every property measured here.** The engine's contribution + is the absence of a bug, and it only applies to signing it performs. A presigned URL handed between + services, or a signature computed before a queue you own, is stale on exactly the schedule of that + queue — measured `[0,2,4,6]` minutes with a perfect clock and no drift anywhere. + +## What is NOT measured here + +- **Streaming.** The SSE/reconnect path signs per open (a fresh `cloneReq` then `cfg.auth.apply` at + engine.ts:1343-1346, after its own `acquireWithin` at 1318), which reads like the same ordering — but + it was not run. A long-lived stream's signature is minted once at open and cannot be refreshed + mid-body; for a SigV4 endpoint held open past five minutes that is an inherent property of + streaming, not a library defect. +- **Pagination.** Each page is a full request through `attemptWithCircuit` (engine.ts:918-946), so + per-page signing should follow from C1 and C2. Not run. +- **Real AWS.** The server here is a fake that validates timestamps. It does not check the signature + itself, so nothing in this directory demonstrates that `signRequestV4` is correct — that is what + `packages/aws-sigv4/test/sigv4.spec.ts` and the official AWS test vectors are for. diff --git a/docs/scenarios/proofs/expiring-signatures/c1-sign-per-attempt.ts b/docs/scenarios/proofs/expiring-signatures/c1-sign-per-attempt.ts new file mode 100644 index 00000000..fa7ee311 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c1-sign-per-attempt.ts @@ -0,0 +1,237 @@ +// C1 — is the request signed PER ATTEMPT, or once per call and replayed? +// +// Failure mode 3 from the capture: if signing is hoisted above the retry loop, attempt 2 carries +// attempt 1's timestamp plus whatever the backoff was. A long backoff — or a `Retry-After` the +// server asked for — then guarantees the retry is stale, and the retry that was supposed to rescue +// the call is the thing that kills it. +// +// MEASURED: signed per attempt. Three attempts six virtual minutes apart produced three DISTINCT +// timestamps, three DISTINCT signatures, and an age of 0ms on every one — including attempt 3, +// eighteen minutes after the call started. `cloneReq` (engine.ts:261-264) hands each attempt a +// FRESH header object copied from the unsigned base request, so last attempt's `x-amz-date` cannot +// survive into this one even by accident, and `cfg.auth.apply` (engine.ts:649) re-runs inside the +// loop. +// +// Part (d) is the one that is NOT free: a server-directed `Retry-After: 600` is honoured +// unboundedly, and it is still fine here — but only because signing is per attempt. Under the +// control it is a guaranteed 403. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c1-sign-per-attempt.ts +import { stitch, systemClock } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeAws } from './fake-aws'; +import { check, checkAtMost, checkSeq, finish, heading, note } from './harness'; +import type { SignEvent } from './signers'; +import { + CREDS, + clockSigV4, + presign, + presignedSigV4, + stampedSigV4, +} from './signers'; +import { runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +const ATTEMPTS = 3; +/** + * Longer than the five-minute window, so attempt 2 is ALREADY doomed if the signature is reused. + * + * `max` is NOT decoration. `backoffDelay` clamps every computed delay to `backoff.max`, which + * defaults to **10 seconds** (resilience.ts:47,56) — so `base: '6m'` alone yields a 10-second wait, + * measured. That default is a quiet piece of protection for this scenario (a COMPUTED backoff can + * never park a call long enough to expire a signature), and part (d) shows the hole in it: a + * server-directed `Retry-After` skips `backoffDelay` entirely and is unbounded by design. + */ +const BACKOFF = { curve: 'fixed', base: '6m', max: '10m' } as const; + +async function main(): Promise { + heading( + 'C1 — three attempts, six minutes apart: does attempt 2 carry attempt 1’s timestamp?', + ); + + // ── (a) the library — retry with a backoff longer than the skew window ──────────────────── + // The server fails every request with 503 (in `retry.on`'s default set), so all three attempts + // run. A stale signature would arrive as a 403 instead, which is NOT retried — so under the + // broken ordering the run would also STOP a retry early. Both effects are visible in the ledger. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 503 }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + retry: { attempts: ATTEMPTS, backoff: BACKOFF }, + clock, + }); + + const pending = call({}).safe(); + await runOut(clock, 30 * MIN); + const result = await pending; + + check('(a) attempts that reached the wire', aws.calls.length, ATTEMPTS); + checkSeq( + '(a) arrival time per attempt (virtual min from t0)', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 6, 12], + ); + checkSeq('(a) WIRE TIMESTAMP per attempt', aws.stamps(), [ + '20260805T120000Z', + '20260805T120600Z', + '20260805T121200Z', + ]); + checkSeq('(a) SIGNATURE AGE per attempt (ms)', aws.ages(), [0, 0, 0]); + check( + '(a) DISTINCT signatures — equal to the attempt count means no replay', + aws.distinctSignatures(), + ATTEMPTS, + ); + checkSeq( + '(a) status per attempt — 503 throughout, never a skew 403', + aws.calls.map((c) => c.status), + [503, 503, 503], + ); + check('(a) the call still failed (503 is real)', result.ok, false); + note( + '(a) `auth.apply` invocations', + `${String(signed.length)} for ${String(ATTEMPTS)} attempts`, + ); + } + + // ── (b) THE CONTROL — signed once, before the retry loop ───────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 503 }); + const headers = await presign(CREDS, URL_S3, T0); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: presignedSigV4(headers), + retry: { attempts: ATTEMPTS, backoff: BACKOFF }, + clock, + }); + + const pending = call({}).safe(); + await runOut(clock, 30 * MIN); + await pending; + + checkSeq( + '(b) control — SIGNATURE AGE per attempt (virtual min)', + aws.ages().map((a) => a / MIN), + [0, 6], + ); + checkSeq( + '(b) control — status per attempt', + aws.calls.map((c) => c.status), + [503, 403], + ); + check( + '(b) control — attempts that reached the wire (the 403 is terminal, so retry 3 never ran)', + aws.calls.length, + 2, + ); + check( + '(b) control — DISTINCT signatures across those attempts', + aws.distinctSignatures(), + 1, + ); + } + + // ── (c) the SHIPPED signer on the real clock ───────────────────────────────────────────── + { + const aws = new FakeAws({ clock: systemClock, failWith: 503 }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: stampedSigV4(CREDS, signed), + retry: { + attempts: ATTEMPTS, + backoff: { curve: 'fixed', base: '1100ms' }, + }, + }); + + const t0 = Date.now(); + await call({}).safe(); + const gaps = aws.calls.map( + (c, i) => c.arrivedAt - (signed[i]?.at ?? NaN), + ); + + check('(c) real clock — attempts', aws.calls.length, ATTEMPTS); + check( + '(c) real clock — DISTINCT signatures from the SHIPPED awsSigV4', + aws.distinctSignatures(), + ATTEMPTS, + ); + check( + '(c) real clock — DISTINCT wire timestamps', + new Set(aws.stamps()).size, + ATTEMPTS, + ); + note( + '(c) real clock — arrival per attempt (ms from t0)', + JSON.stringify(aws.calls.map((c) => c.arrivedAt - t0)), + ); + note( + '(c) real clock — sign→wire gap per attempt (ms)', + JSON.stringify(gaps), + ); + checkAtMost( + '(c) real clock — WORST sign→wire gap (ms)', + Math.max(...gaps), + 250, + ); + } + + // ── (d) a server-directed `Retry-After` longer than the window ─────────────────────────── + // `retry.respect` defaults ON and is deliberately unbounded (engine.ts:743-767), so a server + // saying `Retry-After: 600` parks the call for TEN MINUTES — twice the skew window — before the + // next attempt. That is only survivable because the next attempt re-signs. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 503 }); + const base = aws.adapter(); + const call = stitch({ + url: URL_S3, + // Wrap the fake so it also asks for a 10-minute wait, the way a throttled S3 would. + adapter: async (req) => { + const res = await base(req); + return { + ...res, + headers: { ...res.headers, 'retry-after': '600' }, + }; + }, + auth: clockSigV4({ ...CREDS, clock }), + retry: { attempts: 2 }, + clock, + }); + + const pending = call({}).safe(); + await runOut(clock, 30 * MIN); + await pending; + + checkSeq( + '(d) `Retry-After: 600` honoured — arrival per attempt (virtual min)', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 10], + ); + checkSeq( + '(d) SIGNATURE AGE after a 10-minute server-directed wait (ms)', + aws.ages(), + [0, 0], + ); + note( + '(d) skew the server saw on attempt 2', + `${String(aws.calls[1]?.skewMs)}ms after a 600000ms wait`, + ); + } + + finish( + 'C1', + 'the request is signed PER ATTEMPT — 3 attempts 6 virtual minutes apart produced 3 distinct signatures aged 0ms each, and a 10-minute `Retry-After` wait still arrived fresh; the same call signed once measured 6 minutes and a terminal 403 on attempt 2', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c2-throttle-rate.ts b/docs/scenarios/proofs/expiring-signatures/c2-throttle-rate.ts new file mode 100644 index 00000000..4ce5decf --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c2-throttle-rate.ts @@ -0,0 +1,244 @@ +// C2 — THE DECIDING CLAIM. Does the throttle's wait happen BEFORE the signing, or AFTER it? +// +// This is botocore#149 asked of StitchAPI. Sign, then hold the request behind a rate limiter, and +// the timestamp on the wire is however old the queue was. AWS's own answer to the bug report is to +// generate the timestamp per signing operation and to sign as late as possible; the question here is +// whether the engine already does that. +// +// MEASURED: it does. Four calls behind `throttle: { rate: '1/2m' }` were granted at 0, 2, 4 and 6 +// virtual minutes, and every one of them carried a signature aged 0ms on arrival. The fourth waited +// SIX MINUTES — past the five-minute window — and was accepted, because it was signed at the moment +// its slot came up, not at the moment it was enqueued. `auth.apply` runs at engine.ts:649, INSIDE +// the attempt loop and AFTER the `acquireWithin` at engine.ts:629. +// +// Part (b) is the control, and it is what makes (a) evidence rather than an assertion: the same +// four calls with the headers pre-signed at t=0 measured ages of 0 / 2 / 4 / 6 minutes and a 403 +// RequestTimeTooSkewed on the last. The instrument detects the bug; the library does not have it. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c2-throttle-rate.ts +import { stitch, systemClock } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeAws } from './fake-aws'; +import { check, checkAtMost, checkSeq, finish, heading, note } from './harness'; +import type { SignEvent } from './signers'; +import { + CREDS, + clockSigV4, + presign, + presignedSigV4, + stampedSigV4, +} from './signers'; +import { runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +/** Four slots two minutes apart: the last one is granted at t+6m, PAST the five-minute window. */ +const RATE = '1/2m'; +const CALLS = 4; + +async function main(): Promise { + heading( + 'C2 — a call queued behind `throttle: { rate }`: is its signature stale when it arrives?', + ); + + // ── (a) the library, on a virtual clock, over a queue longer than the window ─────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + throttle: { rate: RATE }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => call({}).safe()); + await runOut(clock, 20 * MIN); + const results = await Promise.all(inFlight); + + checkSeq( + '(a) grant times (virtual min from t0) — the queue really was 6 minutes deep', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 2, 4, 6], + ); + checkSeq( + '(a) SIGNATURE AGE ON ARRIVAL (ms), per call', + aws.ages(), + [0, 0, 0, 0], + ); + checkSeq( + '(a) when `auth.apply` ran (virtual min from t0)', + signed.map((s) => (s.at - T0) / MIN), + [0, 2, 4, 6], + ); + checkSeq( + '(a) status per call — the 6-minute-queued one included', + aws.calls.map((c) => c.status), + [200, 200, 200, 200], + ); + check( + '(a) calls that succeeded', + results.filter((r) => r.ok).length, + CALLS, + ); + note( + '(a) skew the SERVER saw on the last (6-min-queued) call', + `${String(aws.calls[CALLS - 1]?.skewMs)}ms — the window is 300000ms`, + ); + } + + // ── (b) THE CONTROL — the same four calls, pre-signed at t0 ─────────────────────────────── + // Without this the (a) numbers are unfalsifiable. Sign once up front, enqueue, and the ages + // are the queue depth. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock }); + const headers = await presign(CREDS, URL_S3, T0); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: presignedSigV4(headers), + throttle: { rate: RATE }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => call({}).safe()); + await runOut(clock, 20 * MIN); + const results = await Promise.all(inFlight); + + checkSeq( + '(b) control — SIGNATURE AGE ON ARRIVAL (virtual min), signed once at t0', + aws.ages().map((a) => a / MIN), + [0, 2, 4, 6], + ); + checkSeq( + '(b) control — status per call', + aws.calls.map((c) => c.status), + [200, 200, 200, 403], + ); + check( + '(b) control — the skew the server saw on the 6-minute-queued call (ms)', + aws.calls[CALLS - 1]?.skewMs, + 6 * MIN, + ); + check( + '(b) control — calls that succeeded', + results.filter((r) => r.ok).length, + 3, + ); + note( + '(b) control — one signature replayed by every call', + `${String(aws.distinctSignatures())} distinct signature(s) across ${String(aws.calls.length)} requests`, + ); + } + + // ── (c) the SHIPPED signer, on the real clock ───────────────────────────────────────────── + // (a) uses `clockSigV4` because `awsSigV4` stamps `new Date()` (C5), so a virtual queue is + // invisible to it. That makes (a) a measurement of the ENGINE's ordering with a substitute + // strategy. This part closes the gap: the real `@stitchapi/aws-sigv4`, real time, a real + // ~2.4-second queue. Both instruments run through the same seam (`cfg.auth.apply`), so if the + // ordering held only for the substitute, this is where it would show. + { + const aws = new FakeAws({ clock: systemClock }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: stampedSigV4(CREDS, signed), + throttle: { rate: '1/800ms' }, + }); + + const t0 = Date.now(); + const results = await Promise.all( + Array.from({ length: CALLS }, () => call({}).safe()), + ); + const spread = (aws.calls[CALLS - 1]?.arrivedAt ?? t0) - t0; + + checkAtMost( + '(c) real clock — queue was at least 2s deep, so a t0 signature would be visibly old', + 2000, + spread, + ); + // The precise gap: `auth.apply` entry → transport entry, in real ms. The `ageMs` column + // cannot be used here because `x-amz-date` has ONE-SECOND resolution, so it reports up to + // 999ms of pure truncation on a signature that is actually microseconds old. + const gaps = aws.calls.map( + (c, i) => c.arrivedAt - (signed[i]?.at ?? NaN), + ); + note( + '(c) real clock — arrival times (ms from t0)', + JSON.stringify(aws.calls.map((c) => c.arrivedAt - t0)), + ); + note( + '(c) real clock — sign→wire gap per call (ms)', + JSON.stringify(gaps), + ); + checkAtMost( + '(c) real clock — WORST sign→wire gap across the whole queue (ms)', + Math.max(...gaps), + 250, + ); + check( + '(c) real clock — every request accepted', + results.filter((r) => r.ok).length, + CALLS, + ); + note( + '(c) real clock — distinct signatures', + `${String(aws.distinctSignatures())} of ${String(aws.calls.length)}`, + ); + } + + // ── (d) the one seam that CAN reintroduce the bug ──────────────────────────────────────── + // `hooks.onRequest` runs at engine.ts:652 — AFTER `cfg.auth.apply` at 649 and before the + // transport. Everything the engine does with the request happens before signing; this hook is + // the single place a user's own code runs after it. A hook that WAITS therefore ages the + // signature by exactly its wait, and the natural reason to put a wait there — pacing the call + // by some rule `throttle` cannot express — is precisely the case this scenario is about. + // + // Nothing warns. The config that produces it reads as a throttle and behaves as the opposite. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + hooks: { + // A hand-rolled gate: "one call every six minutes", implemented where it seems + // natural rather than where the engine's own wait lives. + onRequest: async () => { + await clock.sleep(6 * MIN); + }, + }, + clock, + }); + + const pending = call({}).safe(); + await runOut(clock, 20 * MIN); + const result = await pending; + + check( + '(d) a 6-minute wait inside `hooks.onRequest` — signature AGE on arrival (virtual min)', + (aws.ages()[0] ?? NaN) / MIN, + 6, + ); + check('(d) the request was rejected', aws.calls[0]?.status, 403); + check('(d) the call failed', result.ok, false); + note( + '(d) the ordering', + 'auth.apply engine.ts:649 → hooks.onRequest engine.ts:652 → transport. A hook is the ONLY user code that runs after signing', + ); + } + + finish( + 'C2', + 'the throttle wait happens BEFORE signing — a rate-limited queue does NOT age the signature (0ms after a 6-minute wait), while the same calls pre-signed measured 6 minutes and a 403. The one exception is `hooks.onRequest`, which runs AFTER signing: a 6-minute wait there aged the signature 6 minutes and got a 403', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c3-throttle-concurrency.ts b/docs/scenarios/proofs/expiring-signatures/c3-throttle-concurrency.ts new file mode 100644 index 00000000..49e9e726 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c3-throttle-concurrency.ts @@ -0,0 +1,162 @@ +// C3 — the same question as C2, for the OTHER half of `throttle`: a request held behind a busy +// concurrency pool. +// +// The two halves are separate code paths inside one `acquire` (resilience.ts:129-157): the +// concurrency slot is taken first and its waiters are served FIFO (`takeSlot`, +// resilience.ts:120-127), then the rate spacing is paced WITHIN the held slot. So a request can be +// blocked by either, and scenario 9 measured the concurrency half being the sharper of the two — a +// quiet caller behind a busy pool is not slowed proportionally, it is queued LAST. +// +// MEASURED: same answer. Four calls behind `throttle: { concurrency: 1 }` against an upstream that +// holds each request for two virtual minutes were granted at 0, 2, 4 and 6 minutes, and every one +// carried a signature aged 0ms. The whole `acquire` — both halves — sits at engine.ts:629, above +// the `cfg.auth.apply` at engine.ts:649. +// +// Part (c) is worth more than it looks: the two limiters STACK, so a call can be held by the +// concurrency pool and then paced again by the rate budget, and the signature is still minted after +// BOTH. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c3-throttle-concurrency.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeAws } from './fake-aws'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { SignEvent } from './signers'; +import { CREDS, clockSigV4, presign, presignedSigV4 } from './signers'; +import { runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +const CALLS = 4; +/** Each in-flight request occupies the single slot for two virtual minutes. */ +const HOLD = 2 * MIN; + +async function main(): Promise { + heading( + 'C3 — a call held behind a busy `throttle: { concurrency }` pool: is its signature stale?', + ); + + // ── (a) the library ────────────────────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, holdMs: HOLD }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + throttle: { concurrency: 1 }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => call({}).safe()); + await runOut(clock, 20 * MIN); + const results = await Promise.all(inFlight); + + checkSeq( + '(a) arrival per call (virtual min) — the 4th waited 6 minutes for the pool', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 2, 4, 6], + ); + checkSeq('(a) SIGNATURE AGE ON ARRIVAL (ms)', aws.ages(), [0, 0, 0, 0]); + checkSeq( + '(a) when `auth.apply` ran (virtual min)', + signed.map((s) => (s.at - T0) / MIN), + [0, 2, 4, 6], + ); + checkSeq( + '(a) status per call', + aws.calls.map((c) => c.status), + [200, 200, 200, 200], + ); + check( + '(a) calls that succeeded', + results.filter((r) => r.ok).length, + CALLS, + ); + check('(a) DISTINCT signatures', aws.distinctSignatures(), CALLS); + } + + // ── (b) THE CONTROL ───────────────────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, holdMs: HOLD }); + const headers = await presign(CREDS, URL_S3, T0); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: presignedSigV4(headers), + throttle: { concurrency: 1 }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => call({}).safe()); + await runOut(clock, 20 * MIN); + const results = await Promise.all(inFlight); + + checkSeq( + '(b) control — SIGNATURE AGE ON ARRIVAL (virtual min)', + aws.ages().map((a) => a / MIN), + [0, 2, 4, 6], + ); + checkSeq( + '(b) control — status per call', + aws.calls.map((c) => c.status), + [200, 200, 200, 403], + ); + check( + '(b) control — calls that succeeded', + results.filter((r) => r.ok).length, + 3, + ); + } + + // ── (c) both limiters at once ─────────────────────────────────────────────────────────── + // `concurrency: 1` with a 2-minute hold paces at 2 minutes; a `rate` of 1/3m is tighter, so the + // rate budget wins and grants land 3 minutes apart. The point is not which one wins — it is + // that signing happens after BOTH, so a call gated twice is still minted fresh. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, holdMs: HOLD }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + throttle: { concurrency: 1, rate: '1/3m' }, + clock, + }); + + const inFlight = Array.from({ length: 3 }, () => call({}).safe()); + await runOut(clock, 20 * MIN); + const results = await Promise.all(inFlight); + + checkSeq( + '(c) concurrency AND rate — arrival per call (virtual min)', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 3, 6], + ); + checkSeq( + '(c) SIGNATURE AGE after both gates (ms)', + aws.ages(), + [0, 0, 0], + ); + check( + '(c) calls that succeeded', + results.filter((r) => r.ok).length, + 3, + ); + note( + '(c) skew the server saw on the doubly-gated 6-minute call', + `${String(aws.calls[2]?.skewMs)}ms`, + ); + } + + finish( + 'C3', + 'a request held six virtual minutes behind a busy concurrency pool still arrived with a 0ms-old signature; the concurrency wait, the rate wait, and both stacked all happen BEFORE `auth.apply`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c4-circuit-cooldown.ts b/docs/scenarios/proofs/expiring-signatures/c4-circuit-cooldown.ts new file mode 100644 index 00000000..cf781063 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c4-circuit-cooldown.ts @@ -0,0 +1,176 @@ +// C4 — does a circuit-breaker cooldown hold a signed request the way a queue would? +// +// The capture groups the breaker with the rate limiter as a place a signature could age. It is a +// different shape and the difference is the finding: the breaker does not QUEUE anything. It +// fast-fails. `attemptWithCircuit` (engine.ts:846-893) reads the phase BEFORE calling +// `attemptLoop`, so an open breaker throws `CircuitOpenError` at engine.ts:871 without ever +// reaching the throttle, the transport, or `cfg.auth.apply`. +// +// MEASURED: a call that fast-failed on an open breaker performed ZERO signings — the signing ledger +// is empty for it. There is no held signature to expire, because nothing was signed. And the +// half-open trial admitted after a six-minute cooldown carried a signature aged 0ms. +// +// Part (c) is where the real hazard lives, and it is not a StitchAPI hazard: the cooldown is a wait +// the CALLER does, outside the call. Anyone who pre-signs and then waits out a breaker has the bug +// regardless of what the engine does. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c4-circuit-cooldown.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeAws, outcomeOf } from './fake-aws'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { SignEvent } from './signers'; +import { CREDS, clockSigV4, presign, presignedSigV4 } from './signers'; +import { drain, runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +/** Longer than the five-minute window, so a signature minted before the cooldown would expire in it. */ +const CIRCUIT = { failures: 2, cooldown: '6m' } as const; + +async function main(): Promise { + heading( + 'C4 — a circuit cooldown: does it hold a signed request, the way a rate limiter would?', + ); + + // ── (a) the breaker opens; does a fast-failed call sign anything? ──────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 500 }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + circuit: CIRCUIT, + clock, + }); + + // Two failures trip it (`failures: 2`), then three more calls inside the cooldown. + const spine: string[] = []; + for (let i = 0; i < 5; i++) { + spine.push(await outcomeOf(() => call({}))); + await drain(); + } + + checkSeq( + '(a) outcome per call — 500, 500, then the breaker fast-fails', + spine, + ['500', '500', '503', '503', '503'], + ); + check('(a) requests that reached the wire', aws.calls.length, 2); + // Signings == wire requests, across FIVE calls. The three that fast-failed contributed + // nothing to either: they never reached `cfg.auth.apply`, so there was no signature for a + // cooldown to age. + check( + '(a) SIGNINGS across all 5 calls (== the 2 that reached the wire)', + signed.length, + 2, + ); + check( + '(a) signings performed by the 3 fast-failed calls', + signed.length - aws.calls.length, + 0, + ); + note( + '(a) the fast-fail error', + 'CircuitOpenError, status 503, thrown at engine.ts:871 before the throttle and before auth', + ); + } + + // ── (b) the half-open trial after a six-minute cooldown ───────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 500 }); + const signed: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }, signed), + circuit: CIRCUIT, + clock, + }); + + await outcomeOf(() => call({})); + await outcomeOf(() => call({})); // breaker now open, armed at t0 + await drain(); + const blocked = await outcomeOf(() => call({})); + // Ride out the cooldown, then take the trial. + await runOut(clock, 6 * MIN); + aws.skewMs = 0; + const trial = await outcomeOf(() => call({})); + await drain(); + + check('(b) inside the cooldown', blocked, '503'); + check('(b) the half-open trial reached the wire', aws.calls.length, 3); + checkSeq( + '(b) arrival per wire request (virtual min)', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 0, 6], + ); + checkSeq( + '(b) SIGNATURE AGE — including the trial admitted after a 6-minute cooldown', + aws.ages(), + [0, 0, 0], + ); + // NOT `distinctSignatures()`: the two pre-cooldown attempts happen in the same virtual + // SECOND, and `x-amz-date` has one-second resolution, so SigV4 over identical inputs + // correctly yields a byte-identical signature. Two equal signatures are evidence of a + // replay only when the requests are more than a second apart — which is exactly what + // distinguishes the trial. + check( + '(b) the trial carries the post-cooldown timestamp, not the one from before it', + aws.stamps()[2], + '20260805T120600Z', + ); + check( + '(b) and its signature differs from the pre-cooldown ones', + aws.calls[2]?.signature !== aws.calls[0]?.signature, + true, + ); + note('(b) the trial outcome', `${trial} (the upstream is still 500)`); + } + + // ── (c) the caller-side hazard the engine cannot help with ────────────────────────────── + // A breaker cooldown is time the CALLER spends not calling. If the request was signed before + // that wait — pre-signed, or handed between services — the engine never gets the chance to + // re-sign it, because the signature arrives as data. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, failWith: 500 }); + const headers = await presign(CREDS, URL_S3, T0); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: presignedSigV4(headers), + circuit: CIRCUIT, + clock, + }); + + await outcomeOf(() => call({})); + await outcomeOf(() => call({})); + await drain(); + await runOut(clock, 6 * MIN); + const trial = await outcomeOf(() => call({})); + await drain(); + + check('(c) control — the trial after the cooldown', trial, '403'); + checkSeq( + '(c) control — SIGNATURE AGE per wire request (virtual min)', + aws.ages().map((a) => a / MIN), + [0, 0, 6], + ); + note( + '(c) control — what the trial actually reported', + 'RequestTimeTooSkewed, not the 500 that opened the breaker — the recovery probe reports the wrong fault', + ); + } + + finish( + 'C4', + 'the breaker does NOT queue a signed request — it fast-fails before signing (3 blocked calls, 0 signings), and the half-open trial after a 6-minute cooldown was signed fresh (age 0ms)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c5-clock-source.ts b/docs/scenarios/proofs/expiring-signatures/c5-clock-source.ts new file mode 100644 index 00000000..e325351e --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c5-clock-source.ts @@ -0,0 +1,169 @@ +// C5 — does SigV4 signing read the INJECTED clock, or `Date.now()`? +// +// This is the one claim in the set that the library LOSES, and it is the third instance of a +// pattern two earlier scenarios already found: scenario 4 measured `timeout.total` reading wall +// time and scenario 6 measured `cache.ttl` doing the same, each while the code beside them used the +// injected `clock`. `@stitchapi/aws-sigv4` makes it three — `amzDateOf(new Date())`, aws-sigv4/ +// src/index.ts:301, with no `clock` anywhere in the package. +// +// MEASURED: advancing a `manualClock` by ten virtual minutes between two signings moved the shipped +// signer's timestamp by 0 SECONDS. The same two signings through a clock-reading signer moved by +// exactly 600. So the shipped strategy is unaffected by `clock`, and: +// +// • Part (c): a SigV4 stitch under `manualClock()` — the default `manualClock()` starts at epoch +// 0 — signs with the real wall time while every fake in the test runs in 1970. Everything is +// 403. Measured: 0 of 3 calls accepted, and the failure is a pure artifact of the test clock. +// • This is also WHY C1–C4 are measured the way they are. A virtual queue is invisible to a +// signer on wall time, so those claims run twice: once with a clock-reading substitute at +// virtual intervals big enough to cross the five-minute window, and once with the shipped +// strategy on the real clock at intervals small enough to run in seconds. +// +// The narrow good news in (d): because the timestamp comes from `new Date()` at the moment `apply` +// runs, it is at least always CURRENT. The bug is a testability bug, not a correctness one — no +// production request goes out with a wrong timestamp because of it. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c5-clock-source.ts +import { stitch, systemClock } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { AuthStrategy } from '../../../../packages/core/src/types'; +import { FakeAws, parseAmzDate } from './fake-aws'; +import { check, checkAtMost, checkSeq, finish, heading, note } from './harness'; +import type { SignEvent } from './signers'; +import { CREDS, clockSigV4, stampedSigV4 } from './signers'; +import { runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; + +/** + * Sign twice through a stitch, ten virtual minutes apart (a `rate` of 1/10m does the spacing), and + * report how far the WIRE timestamp moved between them. A signer on the injected clock reports 600 + * seconds; one on `Date.now()` reports ~0. + */ +async function stampDriftSeconds( + strategyFor: (clock: ReturnType) => AuthStrategy, +): Promise<{ drift: number; stamps: string[] }> { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, windowMs: Number.MAX_SAFE_INTEGER }); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: strategyFor(clock), + throttle: { rate: '1/10m' }, + clock, + }); + const inFlight = [call({}).safe(), call({}).safe()]; + await runOut(clock, 30 * MIN); + await Promise.all(inFlight); + const stamps = aws.stamps(); + const drift = + (parseAmzDate(stamps[1] ?? '') - parseAmzDate(stamps[0] ?? '')) / 1000; + return { drift, stamps }; +} + +async function main(): Promise { + heading( + 'C5 — advance a manualClock by 10 minutes: does the signature’s timestamp move?', + ); + + // ── (a) the shipped strategy ──────────────────────────────────────────────────────────── + { + const log: SignEvent[] = []; + const { drift, stamps } = await stampDriftSeconds(() => + stampedSigV4(CREDS, log), + ); + note('(a) the two wire timestamps', JSON.stringify(stamps)); + // At most 1, not exactly 0: the two signings are milliseconds apart in REAL time, and if + // those milliseconds straddle a wall-clock second the stamp ticks by one. That tick is the + // claim restated — the timestamp tracks wall time, not the 600 virtual seconds that passed + // between the two grants. + checkAtMost( + '(a) SHIPPED awsSigV4 — seconds the timestamp moved across a 600s virtual gap', + drift, + 1, + ); + note( + '(a) what it read instead', + '`amzDateOf(new Date())` — aws-sigv4/src/index.ts:301; the package imports no Clock at all', + ); + } + + // ── (b) a clock-reading signer, same rig ──────────────────────────────────────────────── + { + const { drift, stamps } = await stampDriftSeconds((clock) => + clockSigV4({ ...CREDS, clock }), + ); + note('(b) the two wire timestamps', JSON.stringify(stamps)); + check( + '(b) clock-reading signer — seconds the timestamp moved across the same gap', + drift, + 600, + ); + } + + // ── (c) the consequence for anyone testing a SigV4 stitch ─────────────────────────────── + // `manualClock()` with no argument starts at epoch 0 — the documented default. Point the fake + // server at that same clock (the ordinary thing to do) and every request is ~56 years skewed. + { + const clock = manualClock(); + const aws = new FakeAws({ clock }); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: stampedSigV4(CREDS, []), + clock, + }); + const results = await Promise.all([ + call({}).safe(), + call({}).safe(), + call({}).safe(), + ]); + await runOut(clock, 1000); + + check( + '(c) calls accepted with a default `manualClock()` and the SHIPPED signer', + results.filter((r) => r.ok).length, + 0, + ); + checkSeq( + '(c) status per call', + aws.calls.map((c) => c.status), + [403, 403, 403], + ); + note( + '(c) the skew the fake server measured', + `${String(Math.round((aws.calls[0]?.skewMs ?? 0) / -86_400_000))} days — the wall clock against a virtual epoch of 0`, + ); + } + + // ── (d) the limit of the damage ───────────────────────────────────────────────────────── + // On the real clock the shipped signer is CORRECT: it stamps the instant `apply` runs. So this + // is a testability defect, not a wire defect — worth stating plainly, because "the signer + // ignores the clock" reads much worse than it is. + { + const aws = new FakeAws({ clock: systemClock }); + const log: SignEvent[] = []; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: stampedSigV4(CREDS, log), + }); + const r = await call({}).safe(); + const skew = Math.abs(aws.calls[0]?.skewMs ?? Infinity); + + check('(d) on the real clock the request is accepted', r.ok, true); + checkAtMost( + '(d) skew the server measured, ms (1000 = `x-amz-date`’s one-second resolution)', + skew, + 1000, + ); + } + + finish( + 'C5', + 'the SHIPPED signer reads `new Date()`, NOT the injected clock — 0 seconds of movement across a 600-second virtual advance, and 0 of 3 calls accepted under a default manualClock; a clock-reading signer moved 600s. It is a testability defect, not a wire defect: on real time the stamp is correct', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c6-skew-403-classification.ts b/docs/scenarios/proofs/expiring-signatures/c6-skew-403-classification.ts new file mode 100644 index 00000000..dcd0d237 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c6-skew-403-classification.ts @@ -0,0 +1,230 @@ +// C6 — a `RequestTimeTooSkewed` 403 is the failure most likely to be retried and the one retry +// cannot fix. Is it retried by default? Can it be classified as terminal WITHOUT swallowing it? +// +// MEASURED, and the answers split three ways: +// +// (a) NOT retried by default — `retry.on` defaults to `[429, 502, 503, 504]` (engine.ts:612) and +// 403 is not in it. One request, one failure. The library gets this right by default. +// (b) But `retry` does not HELP either, and the reason is worth stating: with a drifting host the +// engine re-signs on every attempt (C1) and every fresh signature carries the SAME wrong +// clock. Four attempts, four distinct signatures, four identical skews of 600000ms. Signing +// per attempt is necessary and completely insufficient. +// (c) With `circuit` configured, a skew 403 IS counted as a dependency failure: it throws at +// engine.ts:824-831 and `attemptWithCircuit` records it (engine.ts:879-891). Measured: a +// misconfigured host clock opened the breaker and the next calls reported `503 circuit open`. +// A local clock problem now reads as "S3 is down". +// (d) THE FOOTGUN. `verdict: { accept: [403], flag: 'ok' }` — the pure-config classification that +// worked in scenario 9 — SWALLOWS the skew error here, because `verdict.flag` is three-state +// and an ABSENT flag is "no signal" (surface.ts:180-190). AWS's error envelope has no `ok` +// field, so the flag never fires and `accept` alone succeeds on the 403. Measured: the call +// returned `ok: true` and handed the caller `RequestTimeTooSkewed` AS ITS DATA. +// (e) What does work is ~6 lines of `Surface.interpret` composing `verdictOf`: a real error for +// the caller, and zero circuit failures. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c6-skew-403-classification.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { FakeAws, isSkewError, outcomeOf } from './fake-aws'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { CREDS, clockSigV4 } from './signers'; +import { drain, runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +/** The host is ten minutes behind the server. Every signature it mints is already outside the window. */ +const DRIFT = 10 * MIN; + +/** + * The classification that actually works: reject a skew 403 on its BODY, so it reaches the caller as + * a real failure while staying off the transport-health signal the breaker reads. + * + * Six lines, and every one of them is load-bearing. `verdictOf` first so `retry.on`/`verdict.accept` + * keep their meaning; then the skew check; then the default "body is the value". + */ +const skewAwareSurface: Surface = { + id: 'http+skew', + interpret: (res, cfg) => { + const failed = verdictOf(res, cfg); + if (failed) return failed; + if (res.status === 403 && isSkewError(res.body)) + return { ok: false, message: 'RequestTimeTooSkewed', status: 403 }; + return { ok: true, data: res.body }; + }, +}; + +async function main(): Promise { + heading( + 'C6 — a RequestTimeTooSkewed 403 from a drifting host: retried? classifiable? swallowed?', + ); + + const rig = ( + extra: Partial = {}, + ): { + aws: FakeAws; + call: ReturnType; + clock: ReturnType; + } => { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }), + clock, + ...extra, + }); + return { aws, call, clock }; + }; + + // ── (a) is a 403 retried by default? ──────────────────────────────────────────────────── + { + const { aws, call } = rig({ retry: { attempts: 4 } }); + const outcome = await outcomeOf(() => call({})); + await drain(); + + check('(a) outcome', outcome, '403'); + check( + '(a) requests that reached the wire, with `retry: { attempts: 4 }`', + aws.calls.length, + 1, + ); + note( + '(a) why', + '`retry.on` defaults to [429,502,503,504] (engine.ts:612); 403 is not in it', + ); + } + + // ── (b) and if someone adds 403 to `retry.on`? ────────────────────────────────────────── + // The natural reading of a 403 is "auth blip, retry it". This is what that costs. + { + const { aws, call, clock } = rig({ + retry: { + attempts: 4, + on: [403, 429, 502, 503, 504], + // `fixed`, and two whole seconds. The default `expo-jitter` curve puts all four + // attempts inside one virtual second with fractional-millisecond offsets, and + // `x-amz-date` has ONE-SECOND resolution — so the four genuinely-re-signed requests + // would carry one identical timestamp and one identical signature, which reads as a + // replay and is not one. Spacing the attempts past the resolution makes the + // re-signing visible. + backoff: { curve: 'fixed', base: '2s' }, + }, + }); + // The retry backoffs sleep on the INJECTED clock, so the call cannot progress unless + // something advances it. (Nothing hangs the process: a manualClock timer is an array entry, + // not a real one, so an undriven run simply exits with the event loop empty.) + const pending = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const outcome = await pending; + + check('(b) outcome after 4 attempts', outcome, '403'); + check('(b) requests that reached the wire', aws.calls.length, 4); + check( + '(b) DISTINCT signatures — every attempt WAS re-signed', + aws.distinctSignatures(), + 4, + ); + checkSeq( + '(b) the skew the server measured on each fresh signature (ms)', + aws.calls.map((c) => c.skewMs), + [DRIFT, DRIFT, DRIFT, DRIFT], + ); + note( + '(b) the lesson', + 'per-attempt signing does not correct a wrong clock — it faithfully re-mints the same wrong time', + ); + } + + // ── (c) the circuit reads a local clock fault as a dependency outage ──────────────────── + { + const { aws, call } = rig({ + circuit: { failures: 2, cooldown: '30s' }, + }); + const spine: string[] = []; + for (let i = 0; i < 4; i++) { + spine.push(await outcomeOf(() => call({}))); + await drain(); + } + + checkSeq( + '(c) outcome per call — the host clock is wrong, and the breaker opens on it', + spine, + ['403', '403', '503', '503'], + ); + check('(c) requests that reached the wire', aws.calls.length, 2); + note( + '(c) what the page says', + '`circuit open` / 503 — a dependency outage, for a fault entirely inside this process', + ); + } + + // ── (d) THE FOOTGUN — the pure-config classification swallows it ─────────────────────── + { + const { aws, call } = rig({ + circuit: { failures: 2, cooldown: '30s' }, + verdict: { accept: [403], flag: 'ok' }, + }); + const result = await call({}).safe(); + await drain(); + + check( + '(d) `verdict: { accept: [403], flag: "ok" }` — did the call SUCCEED?', + result.ok, + true, + ); + check( + '(d) and the code the caller received as its DATA', + (result.data as { Error?: { Code?: string } } | undefined)?.Error + ?.Code, + 'RequestTimeTooSkewed', + ); + check('(d) requests that reached the wire', aws.calls.length, 1); + note( + '(d) why the `flag` did not save it', + '`verdict.flag` is three-state; an ABSENT flag is "no signal" (surface.ts:180-190). AWS error bodies have no `ok` field, so the flag never fires and `accept` alone succeeds on the 403', + ); + } + + // ── (e) the classification that works ────────────────────────────────────────────────── + { + const { aws, call } = rig({ + kind: skewAwareSurface, + circuit: { failures: 2, cooldown: '30s' }, + verdict: { accept: [403] }, + }); + const spine: string[] = []; + for (let i = 0; i < 4; i++) { + spine.push(await outcomeOf(() => call({}))); + await drain(); + } + + checkSeq( + '(e) outcome per call — a real error every time, and the breaker never trips', + spine, + ['403', '403', '403', '403'], + ); + check( + '(e) requests that reached the wire — no fast-fail, so 4 of 4', + aws.calls.length, + 4, + ); + note( + '(e) the mechanism', + 'a surface rejection returns `{ ok: false }` rather than throwing, so `attemptWithCircuit` records a circuit SUCCESS (engine.ts:874-878) while the caller still gets a failure', + ); + note( + '(e) cost', + '6 lines of `Surface.interpret` + `verdict: { accept: [403] }`', + ); + } + + finish( + 'C6', + 'a skew 403 is NOT retried by default (1 request with attempts:4) — but it IS counted as a circuit failure, so a local clock fault reports `503 circuit open`; and the pure-config `verdict: { accept, flag }` SWALLOWS it (ok:true, RequestTimeTooSkewed handed over as data) because AWS bodies carry no flag. 6 lines of surface fix both', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c7-skew-correction.ts b/docs/scenarios/proofs/expiring-signatures/c7-skew-correction.ts new file mode 100644 index 00000000..ab13cf13 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c7-skew-correction.ts @@ -0,0 +1,310 @@ +// C7 — clock-skew correction. AWS's own SDKs read the server's `Date` off a skew failure, store the +// offset, and re-sign with a corrected clock. Is there any seam in StitchAPI that can do that? +// +// MEASURED: yes, and it is the seam the library already has for exactly this SHAPE of problem — +// `AuthStrategy.shouldRefresh` / `refresh` (types.ts:1273-1274, engine.ts:707-725). It was built for +// "the token expired, get a new one and redo this attempt", and a stale clock is the same story with +// a different noun. A drifting host measured: attempt 1 → 403 RequestTimeTooSkewed, the offset +// learned from the response's `Date` header, the attempt REDONE with a corrected clock, 200. One +// call, no retry budget spent. +// +// Three things about it are worth knowing before relying on it, and all three are measured here: +// +// • `refresh(ctx)` does NOT receive the response. `shouldRefresh(res)` is the only auth hook that +// sees it, so the `Date` header has to be smuggled from one to the other through a closure. +// • It fires ONCE per run (the `refreshed` latch, engine.ts:615,709). Correct for this job — a +// second failure after correcting is a real failure — but it is a latch, not a policy. +// • The redone attempt does NOT count against `retry.attempts` (`attempt--`, engine.ts:723), and +// it DOES re-acquire the throttle, so a corrected re-sign pays a second rate slot. +// +// Part (d) measures the alternative and finds it worse: `hooks.onResponse` also sees the response, +// but a hook cannot ask for another attempt — so correcting there requires putting 403 into +// `retry.on`, which is the thing C6 (b) showed you should not do. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c7-skew-correction.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + AdapterResponse, + AuthStrategy, + Clock, +} from '../../../../packages/core/src/types'; +import { FakeAws, isSkewError, outcomeOf } from './fake-aws'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { SkewOffset } from './signers'; +import { CREDS, clockSigV4 } from './signers'; +import { drain, runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +const DRIFT = 10 * MIN; + +/** + * SigV4 signing that corrects itself from the server's clock — the AWS-SDK mitigation, as an + * `AuthStrategy`. + * + * `apply` is `clockSigV4` (which reads `offset.ms` before stamping). The correction is the two hooks + * below it: `shouldRefresh` recognises a skew failure AND stashes the response, `refresh` reads the + * `Date` header off the stashed response and writes the offset. The stash exists only because + * `refresh(ctx)` is handed an `AuthContext` and nothing else. + * + * `offset` is passed in rather than owned so a caller can share ONE correction across many stitches + * — which is what you want, since the drift is a property of the host, not of the endpoint. + */ +function selfCorrectingSigV4( + opts: { clock: Clock; offset: SkewOffset }, + log: { corrections: number[] }, +): AuthStrategy { + const inner = clockSigV4({ + ...CREDS, + clock: opts.clock, + offset: opts.offset, + }); + let lastSkewResponse: AdapterResponse | undefined; + return { + name: 'selfCorrectingSigV4', + apply: inner.apply, + shouldRefresh: (res) => { + const skewed = res.status === 403 && isSkewError(res.body); + if (skewed) lastSkewResponse = res; + return skewed; + }, + refresh: () => { + const serverDate = Date.parse( + lastSkewResponse?.headers['date'] ?? '', + ); + if (Number.isNaN(serverDate)) return; + opts.offset.ms = serverDate - opts.clock.now(); + log.corrections.push(opts.offset.ms); + }, + }; +} + +async function main(): Promise { + heading( + 'C7 — can anything read the server’s `Date` on a failure and re-sign with a corrected clock?', + ); + + // ── (a) the seam, end to end ──────────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + const log = { corrections: [] as number[] }; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: selfCorrectingSigV4({ clock, offset }, log), + clock, + }); + + const pending = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const outcome = await pending; + + check( + '(a) the call SUCCEEDED despite a 10-minute host drift', + outcome, + 'ok', + ); + check('(a) requests that reached the wire', aws.calls.length, 2); + checkSeq( + '(a) status per request — the skew failure, then the corrected re-sign', + aws.calls.map((c) => c.status), + [403, 200], + ); + checkSeq( + '(a) the skew the server measured, per request (ms)', + aws.calls.map((c) => c.skewMs), + [DRIFT, 0], + ); + checkSeq('(a) corrections learned (ms)', log.corrections, [DRIFT]); + note( + '(a) where the offset came from', + 'the `Date` header on the 403 — the only place the server tells you its time', + ); + } + + // ── (b) what it costs against the retry budget ────────────────────────────────────────── + // `attempt--` (engine.ts:723) means the corrected re-sign is FREE: a stitch with `attempts: 1` + // still gets its second request. That is the difference between this seam and `retry.on`. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: selfCorrectingSigV4({ clock, offset }, { corrections: [] }), + retry: { attempts: 1 }, + clock, + }); + + const pending = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const outcome = await pending; + + check( + '(b) with `retry: { attempts: 1 }` the call still succeeded', + outcome, + 'ok', + ); + check('(b) requests that reached the wire', aws.calls.length, 2); + note( + '(b) why', + '`attempt--` before `continue` (engine.ts:723) — the redone attempt is not counted', + ); + } + + // ── (c) the latch, and whether the correction survives the call ───────────────────────── + // Two facts in one run: the refresh fires at most ONCE per run (so a still-wrong offset is not + // re-corrected within the same call), and the offset — being the caller's object — persists, so + // the NEXT call signs correctly on its first attempt. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + const log = { corrections: [] as number[] }; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: selfCorrectingSigV4({ clock, offset }, log), + clock, + }); + + const first = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + await first; + const wireAfterFirst = aws.calls.length; + + const second = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const secondOutcome = await second; + + check('(c) call 1 — requests', wireAfterFirst, 2); + check('(c) call 2 — outcome', secondOutcome, 'ok'); + check( + '(c) call 2 — requests (1 means it signed correctly first time)', + aws.calls.length - wireAfterFirst, + 1, + ); + check( + '(c) corrections learned across BOTH calls', + log.corrections.length, + 1, + ); + checkSeq( + '(c) skew per request across both calls (ms)', + aws.calls.map((c) => c.skewMs), + [DRIFT, 0, 0], + ); + } + + // ── (d) the latch has a limit: a drift that CHANGES mid-run ──────────────────────────── + // `refreshed` is set for the life of the run, so if the correction is not enough (the host is + // still drifting, or the server moved), the run fails. That is the right default — an + // uncorrectable clock should fail rather than loop — but it is a latch, not a retry policy. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + const log = { corrections: [] as number[] }; + // Drift AGAIN, by a different amount, the instant the first response is handed back — so + // the offset learned from attempt 1's `Date` is already stale when attempt 2 arrives. + // Wrapping the adapter makes it deterministic; nudging `skewMs` on a timer races the run. + const inner = aws.adapter(); + const call = stitch({ + url: URL_S3, + adapter: async (req) => { + const res = await inner(req); + if (aws.calls.length === 1) aws.skewMs = DRIFT + 20 * MIN; + return res; + }, + auth: selfCorrectingSigV4({ clock, offset }, log), + clock, + }); + + const pending = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const outcome = await pending; + + check( + '(d) the server moved again after the correction — outcome', + outcome, + '403', + ); + check('(d) corrections attempted', log.corrections.length, 1); + check( + '(d) requests that reached the wire (no second correction)', + aws.calls.length, + 2, + ); + note( + '(d) the mechanism', + '`refreshed` latches for the run (engine.ts:615,709), so one correction per call and no loop', + ); + } + + // ── (e) the alternative seam, and why it is worse ────────────────────────────────────── + // `hooks.onResponse` sees every response including the 403, so it can LEARN the offset. What it + // cannot do is ask for another attempt — so the correction only lands if a retry happens anyway, + // which means putting 403 in `retry.on`. That re-arms C6 (b): every OTHER 403 (a genuinely bad + // credential) now burns the full retry budget too. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + let learned = 0; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock, offset }), + hooks: { + onResponse: ({ res }) => { + if (res && res.status === 403 && isSkewError(res.body)) { + const d = Date.parse(res.headers['date'] ?? ''); + if (!Number.isNaN(d)) { + offset.ms = d - clock.now(); + learned++; + } + } + }, + }, + retry: { + attempts: 2, + on: [403], + backoff: { curve: 'fixed', base: '2s' }, + }, + clock, + }); + + const pending = outcomeOf(() => call({})); + await runOut(clock, MIN, 1_000); + const outcome = await pending; + + check( + '(e) `hooks.onResponse` + `retry.on: [403]` — outcome', + outcome, + 'ok', + ); + check('(e) offsets learned', learned, 1); + checkSeq( + '(e) skew per request (ms)', + aws.calls.map((c) => c.skewMs), + [DRIFT, 0], + ); + note( + '(e) the price', + 'it works, but it needs 403 in `retry.on`, so every non-skew 403 now burns the retry budget too — the seam in (a) needs no such concession', + ); + } + + await drain(); + finish( + 'C7', + 'clock-skew correction IS reachable — `AuthStrategy.shouldRefresh`/`refresh` learned a 600000ms offset from the 403’s `Date` header and re-signed the SAME attempt to a 200, costing no retry budget; but `refresh` cannot see the response (the offset must be smuggled through a closure) and the latch fires once per run', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/c8-assembled.ts b/docs/scenarios/proofs/expiring-signatures/c8-assembled.ts new file mode 100644 index 00000000..d796ceee --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/c8-assembled.ts @@ -0,0 +1,275 @@ +// C8 — the best available answer, run against all three failure modes at once. +// +// The scenario names three ways to fall outside the five-minute window. Two of them the engine +// already handles and needs no user code at all: +// +// • the signature ageing in YOUR OWN queue (C2/C3) — the throttle wait is at engine.ts:629 and +// `cfg.auth.apply` at engine.ts:649, so the request is signed AFTER the wait, always; +// • the retry replaying a stale signature (C1) — `cloneReq` gives each attempt fresh headers and +// `auth.apply` re-runs per attempt. +// +// The third — the host clock is simply wrong — no client library can fix by ordering, and the +// mitigation the AWS SDKs ship (learn the offset from the server's `Date`, re-sign) is user code +// here. This is that code, assembled, with the classification C6 showed is needed to stop a local +// clock fault reading as a dependency outage. +// +// Run against a host ten minutes behind, six virtual minutes of rate-limited queueing, and a +// breaker watching: 4 of 4 calls succeeded, 0 circuit failures recorded, and the worst signature +// age on the wire was 0ms. +// +// pnpm exec tsx docs/scenarios/proofs/expiring-signatures/c8-assembled.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + AdapterResponse, + AuthStrategy, + Clock, +} from '../../../../packages/core/src/types'; +import { FakeAws, isSkewError, outcomeOf } from './fake-aws'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { SkewOffset } from './signers'; +import { CREDS, clockSigV4, presign, presignedSigV4 } from './signers'; +import { drain, runOut } from './virtual-time'; + +const URL_S3 = 'https://bucket.s3.us-east-1.amazonaws.com/key'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const MIN = 60_000; +const DRIFT = 10 * MIN; +const CALLS = 4; + +// ── THE ANSWER ───────────────────────────────────────────────────────────────────────────────── +// Everything below this line is what a user writes. Everything above the line — per-attempt +// signing, signing after the queue, not retrying a 403 — is already the engine's behaviour. + +/** Piece 1 — SigV4 on an injectable, correctable clock. The seam `@stitchapi/aws-sigv4` lacks (C5). */ +function sigV4WithSkewCorrection(opts: { + clock: Clock; + offset: SkewOffset; +}): AuthStrategy { + const inner = clockSigV4({ + ...CREDS, + clock: opts.clock, + offset: opts.offset, + }); + let skewed: AdapterResponse | undefined; + return { + name: 'sigV4WithSkewCorrection', + apply: inner.apply, + // The only auth hook that sees the response — so it both decides AND captures. + shouldRefresh: (res) => { + const hit = res.status === 403 && isSkewError(res.body); + if (hit) skewed = res; + return hit; + }, + // `refresh` gets an AuthContext and nothing else, hence the closure above. + refresh: () => { + const serverTime = Date.parse(skewed?.headers['date'] ?? ''); + if (!Number.isNaN(serverTime)) + opts.offset.ms = serverTime - opts.clock.now(); + }, + }; +} + +/** Piece 2 — a skew 403 is a real error, and NOT a dependency-health signal (C6). */ +const skewAwareSurface: Surface = { + id: 'http+skew', + interpret: (res, cfg) => { + const failed = verdictOf(res, cfg); + if (failed) return failed; + if (res.status === 403 && isSkewError(res.body)) + return { ok: false, message: 'RequestTimeTooSkewed', status: 403 }; + return { ok: true, data: res.body }; + }, +}; + +// ── end of the answer: 2 declarations, 26 lines ──────────────────────────────────────────────── + +async function main(): Promise { + heading('C8 — drifting host + a six-minute queue + a breaker, all at once'); + + // ── (a) the assembled construction ────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const offset: SkewOffset = { ms: 0 }; + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + kind: skewAwareSurface, + auth: sigV4WithSkewCorrection({ clock, offset }), + verdict: { accept: [403] }, + throttle: { rate: '1/2m' }, + circuit: { failures: 2, cooldown: '30s' }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => + outcomeOf(() => call({})), + ); + await runOut(clock, 20 * MIN); + const spine = await Promise.all(inFlight); + + checkSeq('(a) outcome per call', spine, ['ok', 'ok', 'ok', 'ok']); + check('(a) the offset learned from the server (ms)', offset.ms, DRIFT); + check( + '(a) WORST signature age on the wire (ms), across a 6-minute queue', + Math.max(...aws.ages()), + 0, + ); + // [2,4,6,8], not [0,2,4,6]: the correction probe took the t=0 slot and its corrected + // re-sign took t=2m. A skew correction is a SECOND trip through the attempt loop, so it + // re-acquires the throttle and spends another rate slot — worth knowing when the budget is + // the scarce resource. + checkSeq( + '(a) grant time of each SUCCEEDING call (virtual min)', + aws.calls + .filter((c) => c.status === 200) + .map((c) => (c.arrivedAt - T0) / MIN), + [2, 4, 6, 8], + ); + check( + '(a) skew-failed requests — exactly one, the probe that taught the offset', + aws.calls.filter((c) => c.status === 403).length, + 1, + ); + note( + '(a) requests to the wire', + `${String(aws.calls.length)} for ${String(CALLS)} calls — one correction probe plus one per call`, + ); + note( + '(a) the breaker', + 'never opened: the one 403 was a SURFACE rejection, which records a circuit success (engine.ts:874-878)', + ); + } + + // ── (b) the same workload with none of it ─────────────────────────────────────────────── + // Stock config, the shipped signer's ordering (per-attempt, post-queue) preserved by using the + // clock signer WITHOUT correction, and no classification. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + // No `throttle` here: this part is about the drift and the breaker, and a rate budget would + // force gaps between the calls longer than the breaker's own cooldown — at which point the + // breaker is perpetually half-open and admits every call, which measures the cooldown + // rather than the drift. (a) and (d) carry the queue. + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }), + circuit: { failures: 2, cooldown: '30s' }, + clock, + }); + + // SEQUENTIAL, so each call sees the breaker state the previous one left. Part (d) runs the + // same four calls concurrently and gets a different answer, for a reason worth its own + // measurement. + const spine: string[] = []; + for (let i = 0; i < CALLS; i++) { + spine.push(await outcomeOf(() => call({}))); + await drain(); + } + + checkSeq('(b) no user code — outcome per call', spine, [ + '403', + '403', + '503', + '503', + ]); + check('(b) requests that reached the wire', aws.calls.length, 2); + note( + '(b) what the operator sees', + 'two auth failures then `circuit open` — a dependency outage, for a wrong clock in this process', + ); + } + + // ── (c) and the hand-rolled version of the part the engine gives free ────────────────── + // The engine's contribution is not a feature you can point at; it is the absence of a bug. This + // measures its size: the same four calls, presigned before the queue — which is what a + // hand-rolled `sign(); await limiter.acquire(); send()` does — against the same server. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock }); + const headers = await presign(CREDS, URL_S3, T0); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: presignedSigV4(headers), + throttle: { rate: '1/2m' }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => + outcomeOf(() => call({})), + ); + await runOut(clock, 20 * MIN); + const spine = await Promise.all(inFlight); + + checkSeq( + '(c) sign-then-queue, with a PERFECT clock — outcome per call', + spine, + ['ok', 'ok', 'ok', '403'], + ); + checkSeq( + '(c) signature age per call (virtual min)', + aws.ages().map((a) => a / MIN), + [0, 2, 4, 6], + ); + note( + '(c) the point', + 'no clock is wrong here. The only defect is WHERE the signing happened relative to the queue — and that is the defect the engine does not have', + ); + } + + // ── (d) the breaker is checked BEFORE the queue, not after it ────────────────────────── + // The same four failing calls as (b), fired together instead of one after another. Every one of + // them reads the breaker's phase at t=0 — `attemptWithCircuit` calls `circuit.phase()` + // (engine.ts:863) before `attemptLoop` reaches `acquireWithin` (engine.ts:629) — so all four + // are already past the gate when the first failure opens it. The breaker cannot retract a + // request it has already admitted to the queue. + { + const clock = manualClock(T0); + const aws = new FakeAws({ clock, skewMs: DRIFT }); + const call = stitch({ + url: URL_S3, + adapter: aws.adapter(), + auth: clockSigV4({ ...CREDS, clock }), + throttle: { rate: '1/2m' }, + circuit: { failures: 2, cooldown: '30s' }, + clock, + }); + + const inFlight = Array.from({ length: CALLS }, () => + outcomeOf(() => call({})), + ); + await runOut(clock, 20 * MIN); + const spine = await Promise.all(inFlight); + + checkSeq( + '(d) the SAME four calls fired concurrently — outcome per call', + spine, + ['403', '403', '403', '403'], + ); + check( + '(d) requests that reached the wire (2 in (b), because the breaker stopped the rest)', + aws.calls.length, + 4, + ); + checkSeq( + '(d) and they went out over six minutes, long after the breaker opened (virtual min)', + aws.calls.map((c) => (c.arrivedAt - T0) / MIN), + [0, 2, 4, 6], + ); + note( + '(d) the consequence', + 'a `circuit` does not shed a burst that is already queued behind a `throttle` — the phase check happens at enqueue, the wait happens after it', + ); + } + + finish( + 'C8', + 'assembled: 4 of 4 calls succeeded through a 10-minute host drift and a 6-minute rate-limited queue, worst signature age 0ms, breaker never opened — 26 lines of user code in 2 declarations, all of it for the clock-drift half. The queue and retry halves needed none', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/expiring-signatures/fake-aws.ts b/docs/scenarios/proofs/expiring-signatures/fake-aws.ts new file mode 100644 index 00000000..432d4c59 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/fake-aws.ts @@ -0,0 +1,229 @@ +// The AWS-ish server this scenario measures against: an `Adapter` that VALIDATES the timestamp +// embedded in the signed request against its own clock and rejects with a `403 +// RequestTimeTooSkewed` outside a five-minute window — the same window S3 and every other SigV4 +// service enforces, for the same reason (replay protection). +// +// The single measurement everything here turns on is the SIGNATURE'S AGE ON ARRIVAL: the gap +// between the instant `x-amz-date` claims and the instant the request actually reached the +// transport. A library that signs before a wait it controls produces a growing age; one that signs +// after the wait produces zero regardless of how long the queue was. So every request is recorded +// with BOTH times and the difference between them, and the ledger is what each claim reads. +// +// Two clocks, deliberately separate: +// • `clock` — the CLIENT's clock. `arrivedAt` is stamped from it, so a virtual-time run can +// hold a request for six virtual minutes without waiting six real ones. +// • `skewMs` — how far the SERVER's clock sits from the client's. This is failure mode 1 from +// the capture (the drifting host) expressed as one number: `skewMs: 600_000` is a +// host ten minutes behind, and every signature it mints is already outside the +// window when it arrives, however fast it got there. +import type { + Adapter, + AdapterRequest, + Clock, +} from '../../../../packages/core/src/types'; + +/** The five-minute window. Not configurable at AWS; configurable here only so a test can narrow it. */ +export const SKEW_WINDOW_MS = 5 * 60 * 1000; + +/** One recorded request: what the signature claimed, when it actually arrived, and the gap. */ +export interface AwsCall { + /** 1-based arrival order. */ + n: number; + /** The `x-amz-date` that arrived on the wire, verbatim (`YYYYMMDDTHHMMSSZ`). */ + amzDate: string; + /** `x-amz-date` parsed to epoch ms — the instant the signature CLAIMS it was minted. */ + signedAt: number; + /** Client-clock time the request reached the transport. */ + arrivedAt: number; + /** + * `arrivedAt - signedAt` — how long the signature sat between minting and the wire. THE + * measurement. Zero means the request was signed at the last moment before the transport; + * anything large means it aged in a queue on the way there (botocore#149). + */ + ageMs: number; + /** `serverNow - signedAt` — the skew the SERVER saw, which is what the window is checked against. */ + skewMs: number; + status: number; + path: string; + /** The `Signature=` hex from the `Authorization` header. Two attempts sharing one value is a REPLAY. */ + signature: string; + /** The `Credential=...//...` scope date, which must agree with `x-amz-date`. */ + credentialScope: string; +} + +export interface FakeAwsOptions { + /** The client's clock — `arrivedAt` is stamped from it. */ + clock: Clock; + /** + * How far the SERVER's clock is AHEAD of the client's, in ms. `600_000` models a host ten + * minutes slow: every signature it mints looks ten minutes stale to the server. Default 0. + */ + skewMs?: number; + /** The accepted window either side of the server's own time. Default {@link SKEW_WINDOW_MS}. */ + windowMs?: number; + /** Hold each request this many client-clock ms before answering — a slow upstream, for the concurrency claim. */ + holdMs?: number; + /** Fail every request with this status regardless of the timestamp — for the circuit claim. */ + failWith?: number; +} + +/** Parse an AWS `YYYYMMDDTHHMMSSZ` datetime to epoch ms. `NaN` when it is not that shape. */ +export function parseAmzDate(amzDate: string): number { + const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(amzDate); + if (!m) return NaN; + return Date.UTC( + Number(m[1]), + Number(m[2]) - 1, + Number(m[3]), + Number(m[4]), + Number(m[5]), + Number(m[6]), + ); +} + +/** Render epoch ms as AWS's `YYYYMMDDTHHMMSSZ` — the same shape `@stitchapi/aws-sigv4` emits. */ +export function amzDateOf(ms: number): string { + return new Date(ms) + .toISOString() + .replace(/[:-]/g, '') + .replace(/\.\d{3}/, ''); +} + +/** Pull the `Signature=` hex out of an `Authorization: AWS4-HMAC-SHA256 ...` header. */ +export function signatureOf(authorization: string): string { + return /Signature=([0-9a-f]+)/.exec(authorization)?.[1] ?? ''; +} + +/** Pull the `Credential=/` scope out of an `Authorization` header. */ +export function credentialScopeOf(authorization: string): string { + return /Credential=[^/]+\/([^,]+)/.exec(authorization)?.[1] ?? ''; +} + +/** + * The fake AWS endpoint. `adapter()` is what a stitch is handed; `calls` is the ledger every claim + * reads its numbers off. + */ +export class FakeAws { + readonly calls: AwsCall[] = []; + private readonly clock: Clock; + private readonly windowMs: number; + private readonly holdMs: number; + private readonly failWith: number | undefined; + /** Mutable so a claim can heal a drifting host mid-run and watch the next attempt succeed. */ + skewMs: number; + + constructor(opts: FakeAwsOptions) { + this.clock = opts.clock; + this.skewMs = opts.skewMs ?? 0; + this.windowMs = opts.windowMs ?? SKEW_WINDOW_MS; + this.holdMs = opts.holdMs ?? 0; + this.failWith = opts.failWith; + } + + /** The server's own idea of the time — the client's clock plus whatever drift was configured. */ + serverNow(): number { + return this.clock.now() + this.skewMs; + } + + /** Every recorded signature age, in arrival order — the spine most claims assert on. */ + ages(): number[] { + return this.calls.map((c) => c.ageMs); + } + + /** Every recorded wire timestamp, in arrival order. Two identical entries across attempts = a replay. */ + stamps(): string[] { + return this.calls.map((c) => c.amzDate); + } + + /** How many DISTINCT signatures were seen — equal to `calls.length` iff every attempt re-signed. */ + distinctSignatures(): number { + return new Set(this.calls.map((c) => c.signature)).size; + } + + adapter(): Adapter { + return async (req: AdapterRequest) => { + const arrivedAt = this.clock.now(); + const amzDate = req.headers['x-amz-date'] ?? ''; + const authorization = req.headers['authorization'] ?? ''; + const signedAt = parseAmzDate(amzDate); + const serverNow = this.serverNow(); + // `date` is what a real AWS error carries and what an SDK's skew correction learns + // from — C7's whole seam question is whether anything can read it. + const headers = { date: new Date(serverNow).toUTCString() }; + + const n = this.calls.length + 1; + const skewMs = serverNow - signedAt; + const withinWindow = + Number.isFinite(signedAt) && Math.abs(skewMs) <= this.windowMs; + const status = !withinWindow ? 403 : (this.failWith ?? 200); + + this.calls.push({ + n, + amzDate, + signedAt, + arrivedAt, + ageMs: arrivedAt - signedAt, + skewMs, + status, + path: new URL(req.url).pathname, + signature: signatureOf(authorization), + credentialScope: credentialScopeOf(authorization), + }); + + if (this.holdMs > 0) await this.clock.sleep(this.holdMs); + + if (!withinWindow) { + return { + status: 403, + headers, + body: { + // The real S3 error envelope, near enough: the code is what any + // classification has to key on, because the STATUS alone (403) is + // indistinguishable from a genuinely bad credential. + Error: { + Code: 'RequestTimeTooSkewed', + Message: + 'The difference between the request time and the current time is too large.', + RequestTime: amzDate, + ServerTime: new Date(serverNow).toISOString(), + }, + }, + }; + } + if (this.failWith !== undefined) { + return { + status: this.failWith, + headers, + body: { Error: { Code: 'InternalError' } }, + }; + } + return { status: 200, headers, body: { ok: true, n } }; + }; + } +} + +/** + * Run one call and reduce it to a short outcome token — `'ok'`, or `''` for a failure. + * + * `PromiseLike`, not `Promise`: a stitch call returns a lazy `StitchResult` thenable that starts on + * `.then`, and it is deliberately not a full `Promise`. + */ +export async function outcomeOf( + call: () => PromiseLike, +): Promise { + try { + await call(); + return 'ok'; + } catch (e) { + const err = e as Error & { status?: number }; + return String(err.status ?? err.name); + } +} + +/** Is this response body the S3 skew envelope? The predicate a real classification would use. */ +export function isSkewError(body: unknown): boolean { + return ( + (body as { Error?: { Code?: string } } | undefined)?.Error?.Code === + 'RequestTimeTooSkewed' + ); +} diff --git a/docs/scenarios/proofs/expiring-signatures/harness.ts b/docs/scenarios/proofs/expiring-signatures/harness.ts new file mode 100644 index 00000000..95fae2fe --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/harness.ts @@ -0,0 +1,105 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is an AGE IN MILLISECONDS: how long a signature sat between the +// instant it was minted and the instant it reached the wire. So every assertion prints the +// measured number whether it passes or fails — `age 0ms` and `age 360000ms` ARE the findings, +// and they have to be readable out of context. +// +// Two of the checks below exist because half this scenario cannot be measured on a virtual clock +// (C5: the shipped signer stamps `new Date()`), so those runs use REAL time and their numbers +// carry real scheduling jitter. `checkAtMost` / `checkAtLeast` state the bound the claim actually +// rests on rather than pretending a wall-clock measurement is exact. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-attempt timestamp spine + * (`["...T120000Z","...T120500Z"]`) and the age spine (`[0,0,0,0]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * Assert a measured number is at most `bound`. The wall-clock claims need this: "the signature was + * no more than 250ms old when it hit the wire" is the real statement, and an equality check on a + * real-time measurement would be a check on the machine's scheduler, not on the library. + */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (<= ${String(bound)})` : ` (expected <= ${String(bound)})`}`, + ); +} + +/** Assert a measured number is at least `bound` — the other half of a wall-clock bound. */ +export function checkAtLeast( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual >= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (>= ${String(bound)})` : ` (expected >= ${String(bound)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring the library doing the RIGHT thing, and one (C5) passes by + * measuring it doing the wrong one — the verdict statement carries the direction, because + * "PASS C5" on a claim whose content is "the signer ignores the injected clock" is otherwise + * unreadable. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/expiring-signatures/signers.ts b/docs/scenarios/proofs/expiring-signatures/signers.ts new file mode 100644 index 00000000..68ba06d7 --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/signers.ts @@ -0,0 +1,194 @@ +// The two signing instruments this scenario measures with, and why there have to be two. +// +// `@stitchapi/aws-sigv4`'s `awsSigV4` stamps its timestamp from `new Date()` (aws-sigv4/src/ +// index.ts:301) — NOT from the stitch's injected `clock`. C5 measures that directly. The +// consequence for everything else is procedural: on a `manualClock` the throttle, the backoff and +// the circuit cooldown all run on virtual time while the signature's timestamp keeps ticking on +// wall time, so an age computed across the two is meaningless. Six virtual minutes of queueing — +// the interval where this scenario actually bites — cannot be measured with the shipped signer at +// all without waiting six real ones. +// +// So: +// • `stampedSigV4` wraps the REAL `awsSigV4` and brackets its `apply` with wall-clock reads. It +// measures the SHIPPED code path, on real time, over queue intervals short enough to run in a +// few seconds. This is the instrument that keeps the findings honest. +// • `clockSigV4` is ~20 lines of user code that mints the timestamp from an injected `Clock` and +// hands it to the package's own exported `signRequestV4`. Same engine seam (`cfg.auth.apply`), +// same signing function, same headers — only the clock source differs. It measures the same +// ordering at virtual intervals large enough to cross the five-minute window. +// +// Both are used for C1–C4 and they must agree. Where they do, the ordering finding rests on the +// shipped code and the virtual-time run is just a magnifying glass. `clockSigV4` is also the +// answer to C5 and the foundation of C8, so it is written as production code, not test scaffolding. +import { + EMPTY_PAYLOAD_SHA256, + awsSigV4, + signRequestV4, +} from '../../../../packages/aws-sigv4/src/index'; +import type { AuthStrategy, Clock } from '../../../../packages/core/src/types'; +import { amzDateOf } from './fake-aws'; + +/** One recorded signing: when `auth.apply` ran, and what timestamp it minted. */ +export interface SignEvent { + /** 1-based signing order. */ + n: number; + /** Clock time at which `apply` was entered — the instant the engine reached the signing seam. */ + at: number; + /** The `x-amz-date` this signing produced. */ + amzDate: string; +} + +/** A mutable clock correction, shared between a failure handler and the signer. C7's whole subject. */ +export interface SkewOffset { + /** Milliseconds added to the clock before stamping. Starts at 0; a skew handler writes it. */ + ms: number; +} + +export interface SignerOptions { + region: string; + service: string; + accessKeyId: string; + secretAccessKey: string; +} + +export const CREDS: SignerOptions = { + region: 'us-east-1', + service: 's3', + accessKeyId: 'AKIDEXAMPLE', + secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY', +}; + +/** + * The SHIPPED `awsSigV4`, bracketed so the moment its `apply` ran is recorded. Nothing about the + * signing changes — this delegates to the real strategy and then reads back the `x-amz-date` it + * wrote onto the request. + * + * `now` defaults to wall-clock because that is the only clock the wrapped strategy honours; passing + * anything else would record a time the signature does not agree with, which is the very confusion + * C5 is about. + */ +export function stampedSigV4( + opts: SignerOptions, + log: SignEvent[], + now: () => number = Date.now, +): AuthStrategy { + const inner = awsSigV4(opts); + return { + name: 'stampedSigV4', + async apply(req, ctx) { + const at = now(); + await inner.apply(req, ctx); + log.push({ + n: log.length + 1, + at, + amzDate: req.headers['x-amz-date'] ?? '', + }); + }, + }; +} + +/** + * SigV4 signing that mints its timestamp from an INJECTED clock — the seam `awsSigV4` does not + * expose. Everything else is the package's own `signRequestV4`, so the wire bytes are the same + * shape the shipped strategy produces. + * + * `offset` is a mutable correction added before stamping. It is the entire mechanism behind + * clock-skew correction (C7): a failure handler that learns the server's real time writes + * `offset.ms`, and because the engine re-runs `auth.apply` on every attempt (C1), the NEXT attempt + * signs with the corrected clock. Nothing needs to reach back into the signer. + * + * Deliberately payload-less (`EMPTY_PAYLOAD_SHA256`): every request in these proofs is a GET, which + * is what the shipped strategy signs for a GET too (aws-sigv4/src/index.ts:307-313). + */ +export function clockSigV4( + opts: SignerOptions & { clock: Clock; offset?: SkewOffset }, + log?: SignEvent[], +): AuthStrategy { + return { + name: 'clockSigV4', + async apply(req, ctx) { + // Stamped SYNCHRONOUSLY, before any await: the timestamp must be the instant the + // engine reached this seam, not the instant the (genuinely async) Web Crypto work + // happened to settle. + const at = opts.clock.now() + (opts.offset?.ms ?? 0); + const amzDate = amzDateOf(at); + log?.push({ n: log.length + 1, at, amzDate }); + + const url = new URL(req.url); + req.headers['host'] = url.host; + req.headers['x-amz-date'] = amzDate; + req.headers['x-amz-content-sha256'] = EMPTY_PAYLOAD_SHA256; + + const { authorization } = await signRequestV4({ + method: req.method, + url: req.url, + headers: { ...req.headers }, + payloadHash: EMPTY_PAYLOAD_SHA256, + accessKeyId: opts.accessKeyId, + secretAccessKey: opts.secretAccessKey, + region: opts.region, + service: opts.service, + amzDate, + }); + req.headers['authorization'] = authorization; + ctx.emit('auth', `clock sigv4 ${opts.service}/${opts.region}`); + }, + }; +} + +/** + * THE CONTROL — the bug, expressed in this library, so the instrument is shown to detect it. + * + * A measured age of 0 across a six-minute queue only means something if a genuinely sign-then-queue + * construction measures something else through the SAME fake server and the SAME ledger. This is + * that construction: the headers are computed ONCE, before the calls are enqueued, and replayed + * verbatim on every `apply`. It is precisely the shape AWS describes — *"the SDK signs the request, + * and then puts the request in a queue"* — and precisely what a hand-rolled `presign(); await + * limiter.acquire(); send()` does. + * + * It is also not a straw man. Pre-signing outside the engine is what someone reaches for when they + * want the signature to cover something the strategy cannot see, and nothing in the config + * vocabulary discourages it. + */ +export function presignedSigV4( + headers: Readonly>, +): AuthStrategy { + return { + name: 'presignedSigV4', + apply(req) { + Object.assign(req.headers, headers); + }, + }; +} + +/** + * Compute one SigV4 header set for `url` at instant `at` — the input to {@link presignedSigV4}, and + * the "what does hand-rolling this cost" baseline C8 counts lines against. + */ +export async function presign( + opts: SignerOptions, + url: string, + at: number, + method = 'GET', +): Promise> { + const amzDate = amzDateOf(at); + const host = new URL(url).host; + const headers: Record = { + host, + 'x-amz-date': amzDate, + 'x-amz-content-sha256': EMPTY_PAYLOAD_SHA256, + }; + const { authorization } = await signRequestV4({ + method, + url, + headers: { ...headers }, + payloadHash: EMPTY_PAYLOAD_SHA256, + accessKeyId: opts.accessKeyId, + secretAccessKey: opts.secretAccessKey, + region: opts.region, + service: opts.service, + amzDate, + }); + headers['authorization'] = authorization; + return headers; +} diff --git a/docs/scenarios/proofs/expiring-signatures/virtual-time.ts b/docs/scenarios/proofs/expiring-signatures/virtual-time.ts new file mode 100644 index 00000000..0b8adf5d --- /dev/null +++ b/docs/scenarios/proofs/expiring-signatures/virtual-time.ts @@ -0,0 +1,75 @@ +// Driving a `manualClock` when the code under test does REAL async work. +// +// `manualClock.advance(ms)` fires every timer due before the target, draining the microtask queue +// between fires. That is enough for code whose only asynchrony is the clock. SigV4 signing is not +// that code: `crypto.subtle.digest`/`sign` are genuinely async and settle on the MACROTASK queue, +// several turns deep (`signRequestV4` chains four HMACs and two digests). +// +// The consequence, measured before this helper existed: a request signed at virtual t=0 had not +// reached the transport by the time `advance` fired the NEXT timer and moved virtual time to +// t=120000 — so the fake server stamped its arrival at 120000 and the ledger reported a 120-second +// signature age for a request that was signed at the last possible moment. A pure artifact: the +// virtual clock jumped while real crypto was still running. +// +// `runOut` removes it by advancing in slices and letting real macrotasks drain between them, so +// every request woken at virtual time T reaches the transport before virtual time leaves T. That is +// the faithful model — real signing takes well under a millisecond, so in production the request +// does arrive at essentially the instant it was signed. Without the drain the instrument +// manufactures the very ageing it is trying to detect, in the library's DISfavour. +import type { ManualClock } from '../../../../packages/core/src/testing'; + +// Yield one full turn of the event loop. +// +// `setImmediate` rather than `setTimeout(…, 0)`: Node clamps a zero timeout to ~1ms, which made a +// drain deep enough to be reliable (see `drain`) cost tens of milliseconds and a whole run of these +// proofs over a minute. `setImmediate` fires in the check phase — AFTER the poll phase where +// libuv delivers `crypto.subtle`'s threadpool completions — so it observes settled crypto just as +// well, at roughly a thousandth of the cost. `manualClock.advance` keeps using `setTimeout` for its +// own internal drain; this only governs the turns `runOut` adds around it. +const macrotask: () => Promise = + typeof setImmediate === 'function' + ? () => + new Promise((resolve) => { + setImmediate(resolve); + }) + : () => + new Promise((resolve) => { + setTimeout(resolve, 0); + }); + +/** + * Yield `turns` times. + * + * The default is deliberately generous. It is not just `signRequestV4`'s six awaits: the deepest + * chain in these proofs is a skew CORRECTION — response → `shouldRefresh` → `refresh` → `attempt--` + * → re-acquire → re-sign → transport — and that path takes no clock wait at all, so a slice + * boundary landing inside it advances virtual time between the signing and the arrival and reports + * an age that is pure artifact. Measured at six turns, C7 (c) reported a 1000ms skew on roughly half + * its runs; at this depth it reports 0 on all of them. Each turn is a `setTimeout(0)`, so the whole + * drain costs single-digit milliseconds. + */ +export async function drain(turns = 40): Promise { + for (let i = 0; i < turns; i++) await macrotask(); +} + +/** + * Advance `clock` by `totalMs` in `stepMs` slices, draining real macrotasks before the first slice + * and after every one. Use this instead of a bare `advance()` in any run where signing (or any + * other real async work) sits between a clock wait and the transport. + * + * `stepMs` only has to be smaller than the smallest interval being measured; it does not have to + * divide anything evenly. + */ +export async function runOut( + clock: ManualClock, + totalMs: number, + stepMs = 30_000, +): Promise { + await drain(); + for (let left = totalMs; left > 0;) { + const slice = Math.min(stepMs, left); + await clock.advance(slice); + await drain(); + left -= slice; + } +} diff --git a/docs/scenarios/proofs/intermittent-drift/README.md b/docs/scenarios/proofs/intermittent-drift/README.md new file mode 100644 index 00000000..2bd6ff02 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/README.md @@ -0,0 +1,221 @@ +# Proofs — the vendor changed the shape for 5% of responses + +Runnable evidence for the claims in [`../../intermittent-drift.md`](../../intermittent-drift.md). + +**This is the one scenario in the pass where the library mostly wins, and the capture was wrong in +the library's favour on both deciding claims.** C3 predicted a silent `NaN`/`0`; measured, the +default posture is a _hard failure_ — `z.number()` on `"12345"` fails the call, and even +`z.coerce.number()` on `"abc"` fails, because Zod rejects `NaN`. C6 predicted aggregation was the +gap, on the strength of every other scenario in this pass finding no cross-call state; measured, +`trace` is a real aggregation seam and an 84-line sink reports +`5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)`. + +**Two findings survive that anyway, and both are about the same hole: the finding carries KINDS, +not VALUES.** `warn|coerced|transaction_id|string -> number` is byte-identical for `"12345"` → the +correct `12345` and for `"abc"` → a literal `0`. And `z.coerce.number()` maps `null`, `""`, `" "`, +`false`, `[]` and `"0"` all to exactly `0`, so the geocoder-style intermittent null and the +$0-transaction are **the same bug** on a money field. + +Every script is standalone and offline. The evidence is **the value the caller received** — every +assertion prints it with `JSON.stringify`, because `0` and `"0"` and `null` have to be +distinguishable on the page. Time is load-bearing only where a rate needs a window, so `manualClock()` +drives C6(f) and all of C8. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c3-the-zero-dollar-test.ts + +# all of them +for f in docs/scenarios/proofs/intermittent-drift/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/intermittent-drift/*.ts +``` + +### Why this directory imports Zod by path + +`drift()` reports the difference between the raw body and **whatever your validator returned**. It +performs no coercion of its own. So the question C3 asks — "what does the caller actually receive +when `transaction_id` stops being a number" — is answered by the schema library's coercion rules, +and hand-rolling a `{ validate }` stub (what every other scenario in this section did) would mean +inventing the behaviour under test. These scripts use the real Zod v4 that `packages/core` already +depends on, reached via `packages/core/node_modules/zod` because pnpm does not hoist it to the +workspace root and `docs/` has no manifest. See [`zod.ts`](./zod.ts). In application code the +spelling is `import { z } from 'zod'`. + +## What each script establishes + +| Script | Question | Measured | +| ---------------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `c1-added-field.ts` | an ADDED field — alarm? level? does the value arrive? | **`info`, one finding, and the value is STRIPPED.** 51 added values → 2 findings | +| `c2-removed-field.ts` | a REMOVED field — caught? distinguishable from C1? | **Yes, `error` vs `info` — but only if you declared it required.** 4 declarations, 4 answers | +| `c3-the-zero-dollar-test.ts` | the $0-transaction test, both directions | **Default is SAFE. Two spellings give a literal `0`, and the finding is IDENTICAL to the benign one** | +| `c4-intermittent-null.ts` | null on 5% — fires only there? names the field? | **Yes: 5 findings on calls [20,40,60,80,100], each naming `formatted_address`. No nullable LEVEL** | +| `c5-severity.ts` | four classes, four severities, declaratively? | **Three kinds re-level in one literal. Keyed by MECHANISM, not class. No per-path severity** | +| `c6-aggregation.ts` | can you learn "5% of calls drifted on field X"? | **Yes — `5.0% of calls: ... (5/100, 5 landed 0)`, and 5%→25% across a window. Three counting traps** | +| `c7-accessors.ts` | is the finding actionable, and on which accessor? | **Path always; expected/actual only on HARD findings. `.safe()` carries NOTHING for a soft one** | +| `c8-canary-watch.ts` | the assembled answer, priced | **Silent on additions, one alert line per breaking class at 5%, 0 zeros — and 93 lines vs 92** | + +## Files + +- `fake-vendor.ts` — the vendor rolling a change out to a percentage of traffic. Five named + mutations, one per industry change class, plus the two halves of the type change the scenario + turns on: `retyped` (`12345` → `"12345"`, a plausible string) and `garbage` (`→ "abc"`, the same + wire-type shift with a value that cannot be recovered). `rate: 0.05` means **every 20th call**, + not a coin flip, so "5 of 100" is a fact on every machine. `vendor.mutatedCount` is the ground + truth every measured rate is checked against. +- `zod.ts` — the one-line re-export, and the note on why it is a path. +- `drift-rate.ts` — the aggregating `TraceSink`. **This is the C6/C8 deliverable and it is user + code**: it counts logical calls off `start`, collapses N findings to one drifted call on + `ctx.spanId`, evicts on an injected clock for a rolling window, and joins each finding to its + run's `result` event so it can report that a coercion landed on `0`. 84 counted lines. +- `canary-watch.ts` — the declarative half of the assembled answer, in its own file so C8 can count + it. A strict schema plus one `severity` override. 9 counted lines. +- `hand-rolled.ts` — the same feature set with no library at all: classify against a declared shape + into the four industry classes, keep the value only where it is unambiguous, maintain a windowed + per-field rate. 92 counted lines. The baseline C8 prices against. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. `check` + prints with `JSON.stringify` rather than `String`, because this scenario's whole content is + telling `0` from `"0"` from `null`. + +## Reading the numbers honestly + +- **C1 is the cleanest result in this scenario, with a half nobody writes down.** An added field is one + `info|undeclared|settlement_delay_ms|undeclared field (number)`, the call succeeds, `ignore` + silences it without touching the schema, and 51 added values across a 50-element array collapse + to **2** findings (`all 50 elements: …` plus a `sample` coordinate). The unwritten half: the + engine serves the **validated** value (engine.ts:1224) and a Zod object strips unknown keys, so + `data.settlement_delay_ms` is `undefined`. **Drift tells you a field appeared and simultaneously + guarantees you cannot read it.** The trade is exact — with a schema you get the finding and lose + the value; without one you get the value and no finding. +- **C2's level distinction is real and it is not a property of the removal.** `error|invalid| +currency|Required` against C1's `info|undeclared` is three levels and a different kind. But the + same wire body against `.optional()` produces a successful call and **zero findings**, and against + `.default("usd")` produces `verbose|defaulted` **plus a currency the vendor never sent**. One + vendor change, four declarations, four answers. "Removal is breaking" is something you have to + have already declared. +- **C3 refutes the capture on the mechanism and confirms it on the outcome.** `z.number()` on + `"12345"` is `error|invalid|transaction_id|Expected number, received string` and the call fails + with `data: null`. `z.coerce.number()` on `"abc"` **also** fails — `Number("abc")` is `NaN` and + Zod rejects `NaN`. StitchAPI does not manufacture a $0 charge on its own. +- **Two ordinary spellings do, and one of them needs no `.catch()`.** `z.coerce.number().catch(0)` + on `"abc"` hands the caller literal `0`. And `z.coerce.number()` on **`null`** hands the caller + literal `0`, because `Number(null) === 0` — six wire values (`null`, `""`, `" "`, `false`, `[]`, + `"0"`) coerce to exactly `0` and only `"abc"` and `undefined` reject. +- **And the finding cannot separate them.** `warn|coerced|transaction_id|string -> number` is what + you get for `"12345"` → `12345` and byte-for-byte what you get for `"abc"` → `0`, because `detail` + is `kindOf(old) -> kindOf(new)` (drift.ts:77-83). **No alert built on findings alone can tell a + correct coercion from a $0 charge.** +- **The schemas people write to survive drift are the ones that make it invisible.** + `z.union([z.number(), z.string()])` and `z.unknown()` accept both shapes, so raw === validated, so + `diff` produces nothing: the caller gets the raw `"12345"` with **zero findings**, and every + `=== 12345` in the codebase is now false. +- **C4 is the library at its most precise.** Over 100 calls with 5 nulled, drift fired on exactly + calls `[20,40,60,80,100]` — matching the vendor's own ledger — with zero false positives on the + other 95, each finding naming `formatted_address`. +- **But there is no nullable LEVEL, and that is a genuine gap against the industry taxonomy.** One + null, four declarations: required → `error|invalid` and a failed call; `.nullable()` and + `.nullish()` → **nothing at all** (declared variance, raw === validated); `.catch("")` → `warn` + and a fabricated `""`. "Warning-level, value intact" is not one of the options, and `.nullable()` + — the correct schema for a sometimes-null field — makes a 5% rollout completely invisible. +- **C5: the vocabulary is keyed on the wrong axis.** `severity` maps `undeclared`/`coerced`/ + `defaulted` (types.ts:72) — what your _schema_ did — not addition/removal/type-change/nullable — + what the _vendor_ did. Only addition maps 1:1. The three soft kinds re-level fully in one literal, + and `severity: 'warn'` is an emission-time allowlist (drift.ts:147) — **a finding you filtered out + never reaches a trace sink either**, so you cannot filter and count the same kind. +- **There is no per-path severity.** `resolveSeverity` (drift.ts:90-101) never sees the path. + `ignore` is path-aware and it is on/off. "A coercion on `transaction_id` pages, a coercion on + `description` does not" has no spelling. +- **`error` is type-blocked and runtime-live.** `DriftSeverity` excludes `'error'` (types.ts:74) and + the docs say fatality is the schema's job — c5(e) asserts the rejection with `@ts-expect-error`. + Cast past the type and the runtime honours it: `severity: { coerced: 'error' }` produced + `error|coerced|transaction_id` and **failed the call** (`levelOf` at drift.ts:100 → + `finding.level === 'error'` at engine.ts:1215). Off-contract; the documented route is a strict + schema, which also gives a better message. +- **C6 refutes the capture's central prediction. `trace` is a real aggregation seam.** A `TraceSink` + is configured once, receives every event of every call, and `ctx.spanId` identifies the logical + call — the one place in the library where cross-call state is the design rather than a leak + (contrast scenario 7's `HookContext` and scenario 11's leaking `items` closure). Measured: + `5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)`, and 5.0% → 25.0% + across a rolling window on an injected clock. +- **The sink is also the only place the $0 charge is diagnosable in-flight.** Findings and the + `result` event share a `spanId`, so joining them recovers what C3(e) showed the finding cannot + say: `10/10, 5 landed 0` — same finding, half of them zeros. +- **Three counting traps, all measured.** (1) **Findings are not calls**: two drifted fields on one + response is two findings, so a naive `findings / calls` reads 200%. (2) **A cache hit divides your + rate by the hit ratio**: a hit emits `start` and `result` but no drift (engine.ts:1605-1613), so 5 + calls against a **100%**-drifting vendor measured **20%**, with every line of code correct. + (3) **`.report()` pollutes both sides of the fraction**: it is a fresh run, so it added a request + _and_ a tick to the denominator. +- **C7: the field path is always there; expected/actual is only half there.** A hard finding carries + both types (`Expected number, received string` — Zod's message). A soft one carries + `kindOf(old) -> kindOf(new)` and no values, on any accessor except `.inspect().raw`. +- **`.safe()` is the worst accessor and it is the one everybody uses.** On a soft finding it carries + **nothing** — `{ok: true, data, error: null}` and a `transaction_id` of `0`. On a hard one it + carries `contract violation (drift)`; `StitchError` has `{status, attempts, body, url}` and **no + `findings`** (types.ts:1656-1690), while the trace sink for the _same run, same instant_ named the + field and both types. +- **`.stream()` is the cheap live accessor and `.inspect()` is the diagnostic one.** `.stream()` + gives every finding plus the validated value in **one** request (`start, progress, drift, drift, +result, done`). `.inspect()` is the only accessor with **raw** (`"abc"`) and **validated** (`0`) in + one object — and it is a fresh probe. `.report()` on the drifting stitch reported **zero + findings**, because the probe hit a clean response. +- **C8's strict posture hits the target exactly.** Six workloads × 100 calls, one configuration: + silent on a 100% addition rollout, one alert line per breaking class at 5% carrying rate + field + + both types, and **zero $0 charges on every workload**. The soft schema a team writes when the + strict one starts failing calls keeps 100% availability and produces **ten** $0 charges. +- **The price is a wash, and the hand-rolled version wins one row.** 9 declarative lines + an + 84-line sink = 93, against 92 hand-rolled. The _detection_ is 9 lines against ~35; the + _aggregation_ is ~84 lines of user code either way. And the hand-rolled classifier expresses the + fourth industry class — null → `warn`, value passed through — that C5 measured as inexpressible in + `DriftOptions`. What the 92 lines do not have is the resilience stack: one `retry` line absorbed + 8 × `503` mid-canary and the rate still counted **100 logical calls out of 108 wire requests**. + +## The footguns + +- **A `coerced` finding cannot tell you whether the value is right.** `warn|coerced|transaction_id| +string -> number` is identical for `12345` and for `0`. If you alert on findings, alert on the + _rate_, and join to the `result` event if the field is money. +- **`z.coerce.number()` maps `null` to `0`.** No `.catch()`, no garbage string, no error — just a + vendor that starts nulling a field on 5% of responses. `""`, `" "`, `false` and `[]` do the same. + Never put `z.coerce.number()` on a money or identity field. +- **`.catch(0)` is a $0 transaction generator.** It exists to keep calls succeeding, and the value + it succeeds with is the one your business logic will act on. +- **A tolerant schema is a blind one.** `z.union([number, string])`, `z.unknown()`, `z.any()`: raw + === validated, so `diff` produces nothing and `drift()` reports nothing. The schema you write to + stop the alarms is the schema that removes them. +- **An `.optional()` field can be deleted by the vendor in total silence.** Absent in raw, absent in + validated, no diff, no finding, `ok: true`. Audit your optionals — each one is a removal you have + pre-approved. +- **`.default()` on a removed field fabricates a value and logs it at `verbose`.** The quietest + level in the vocabulary, and dropped entirely by `severity: 'warn'`. +- **A plain `output` schema without `drift()` coerces in complete silence.** Same coercion, same + stripping, zero findings. `drift()` is the diagnostic wrapper, not the validator. +- **Caching silently divides your drift rate.** A hit emits `start`/`result` but never a drift + finding. Your measured rate is the true rate × miss ratio, and nothing warns you. +- **Filtering with `severity` also deletes the data.** The allowlist runs at emission (drift.ts:147), + so a filtered kind is invisible to the trace sink too. Re-level with the map form if you want it + quiet _and_ counted. +- **`findings / calls` is not a drift rate.** One response with two drifted fields is two findings. + Collapse on `ctx.spanId` first. +- **`.report()` and `.inspect()` are fresh probes, not readbacks.** They cost a request, they tick + your counters, and they answer about a _different_ response than the one that hurt you. C7(e) + measured `.report()` returning zero findings immediately after a call that drifted. +- **A soft drift is invisible on the awaited path.** `ok: true`, `error: null`, `data` holding a + `0`, and nothing on the result object to read. If you only ever call `.safe()`, `drift()` is + configured and doing nothing for you — wire a `trace` sink or use `.stream()`. diff --git a/docs/scenarios/proofs/intermittent-drift/c1-added-field.ts b/docs/scenarios/proofs/intermittent-drift/c1-added-field.ts new file mode 100644 index 00000000..c40eccfc --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c1-added-field.ts @@ -0,0 +1,187 @@ +// C1 — the vendor ADDS a field the consumer does not model. Non-breaking by every published +// policy (LinkedIn, Xandr), so the right answer is "notice, do not alarm, do not fail". +// +// Measured: `info | undeclared | settlement_delay_ms | undeclared field (number)`, the call +// succeeds, and the added value is STRIPPED from what the caller receives. That last half is the +// part nobody writes down: drift tells you a field appeared and simultaneously guarantees you +// cannot read it, because the value the engine serves is the VALIDATED one (engine.ts:1224). +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c1-added-field.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +const Charge = z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string(), + status: z.string(), +}); + +const BASELINE = { + transaction_id: 100001, + amount: 4200, + currency: 'usd', + status: 'succeeded', +}; + +/** Collect the drift findings a run emitted, via the trace sink (the accessor C7 measures). */ +function collector(): { sink: TraceSink; seen: string[] } { + const seen: string[] = []; + return { + seen, + sink: { + handle(e: StitchEvent) { + if (e.type === 'drift') seen.push(fmt(e.finding)); + }, + }, + }; +} + +async function main(): Promise { + heading('C1 — a field the vendor ADDED'); + + // ── (a) the added field: one info finding, a successful call ───────────────────────────── + { + const { sink, seen } = collector(); + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ ...BASELINE, settlement_delay_ms: 900 }), + output: drift(Charge), + trace: sink, + }); + const r = await call.safe(); + + check('(a) the call SUCCEEDED', r.ok, true); + check('(a) no error', r.error, null); + checkSeq('(a) findings', seen, [ + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + ]); + note( + '(a) → the finding is INFO, it NAMES the field, and it says what type arrived', + '', + ); + } + + // ── (b) …and the added value never reaches the caller ──────────────────────────────────── + // The engine serves `validated`, not `raw` (engine.ts:1224), and a Zod object strips unknown + // keys. So the one accessor a normal caller uses cannot see the new field at all. + { + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ ...BASELINE, settlement_delay_ms: 900 }), + output: drift(Charge), + }); + const r = await call.safe(); + const data = r.data as Record; + + checkSeq('(b) keys the caller received', Object.keys(data), [ + 'transaction_id', + 'amount', + 'currency', + 'status', + ]); + check( + '(b) the added value on the awaited path', + data['settlement_delay_ms'], + undefined, + ); + note( + '(b) → the addition is REPORTED and STRIPPED at the same time. To read the new value you need `.inspect().raw` — a second request', + '', + ); + + const ins = await call.inspect(); + const raw = ins.raw as Record; + check( + '(b) …and `.inspect().raw` does carry it', + raw['settlement_delay_ms'], + 900, + ); + } + + // ── (c) `ignore` silences a known addition without touching the schema ─────────────────── + // The acknowledged-surface lever. Path-based, so a field you have already triaged stops + // producing a finding on every subsequent call. + { + const { sink, seen } = collector(); + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ ...BASELINE, settlement_delay_ms: 900 }), + output: drift(Charge, { ignore: ['settlement_delay_ms'] }), + trace: sink, + }); + const r = await call.safe(); + check('(c) still succeeds', r.ok, true); + checkSeq('(c) findings with `ignore`', seen, []); + } + + // ── (d) a NESTED addition, and an addition inside every array element ──────────────────── + // The `[]` grammar collapses a per-element addition into ONE finding with a count and a + // sample coordinate (drift.ts:184-198) — a 50-item list does not produce 50 alarms. + { + const { sink, seen } = collector(); + const Envelope = z.object({ + charges: z.array(z.object({ transaction_id: z.number() })), + meta: z.object({ page: z.number() }), + }); + const call = stitch({ + url: 'https://pay.example/charges', + adapter: serving({ + charges: Array.from({ length: 50 }, (_v, i) => ({ + transaction_id: i, + settlement_delay_ms: 900, + })), + meta: { page: 1, cursor: 'abc' }, + }), + output: drift(Envelope), + trace: sink, + }); + const r = await call.safe(); + check('(d) still succeeds', r.ok, true); + checkSeq('(d) findings over 50 elements + a nested key', seen.sort(), [ + 'info|undeclared|charges[].settlement_delay_ms|all 50 elements: undeclared field (number)', + 'info|undeclared|meta.cursor|undeclared field (string)', + ]); + note( + '(d) → 51 added values, 2 findings. The array collapse is real and the nested path is fully qualified', + '', + ); + } + + // ── (e) the same addition against a stitch with NO output schema ───────────────────────── + // The control. Without a schema there is nothing to diff against, so the addition passes + // through to the caller and produces no finding at all. + { + const { sink, seen } = collector(); + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ ...BASELINE, settlement_delay_ms: 900 }), + trace: sink, + }); + const r = await call.safe(); + const data = r.data as Record; + checkSeq('(e) findings with no `output`', seen, []); + check( + '(e) …but the value DOES reach the caller', + data['settlement_delay_ms'], + 900, + ); + note( + '(e) → the two properties trade off exactly: schema ⇒ finding + stripped, no schema ⇒ value + silence', + '', + ); + } + + finish( + 'C1', + 'CONFIRMED, and quieter than the capture hoped for. An added field produces exactly one `info|undeclared|settlement_delay_ms|undeclared field (number)` finding, the call succeeds, and 51 added values across a 50-element array collapse to 2 findings with `all 50 elements` and a sample coordinate. `ignore: ["settlement_delay_ms"]` silences it without touching the schema. The half the capture does not mention: the engine serves the VALIDATED value (engine.ts:1224), so the added field is STRIPPED — `data.settlement_delay_ms` is `undefined` on the awaited path, and reading it needs `.inspect().raw`, which is a second request', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c2-removed-field.ts b/docs/scenarios/proofs/intermittent-drift/c2-removed-field.ts new file mode 100644 index 00000000..61553aa2 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c2-removed-field.ts @@ -0,0 +1,194 @@ +// C2 — the vendor REMOVES a field. Breaking by every published policy, and the capture's question +// is whether it is distinguishable IN LEVEL from C1's addition. +// +// It is — by three levels and by a different `change` kind. But the distinction is NOT a property +// of the removal: it is a property of HOW YOU DECLARED THE FIELD. The same missing key produces +// `error` (call fails), `verbose`, or NOTHING AT ALL depending on one word in your schema. That is +// the finding of this file, and it is the mirror image of C1: an addition is classified by the +// vendor's behaviour, a removal is classified by yours. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c2-removed-field.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +/** The response after the vendor dropped `currency`. */ +const WITHOUT_CURRENCY = { + transaction_id: 100001, + amount: 4200, + status: 'succeeded', +}; + +interface Outcome { + ok: boolean; + message: string | null; + findings: string[]; + data: unknown; +} + +/** Run the removed-field body against one schema shape and reduce it to a comparable outcome. */ +async function against(schema: unknown): Promise { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving(WITHOUT_CURRENCY), + output: drift(schema as never), + trace: sink, + }); + const r = await call.safe(); + return { + ok: r.ok, + message: r.error?.message ?? null, + findings, + data: r.data, + }; +} + +async function main(): Promise { + heading('C2 — a field the vendor REMOVED'); + + // ── (a) required in the schema → hard `error`, the call FAILS ──────────────────────────── + { + const o = await against( + z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string(), + status: z.string(), + }), + ); + check('(a) the call FAILED', o.ok, false); + check('(a) error message', o.message, 'contract violation (drift)'); + check('(a) data the caller got', o.data, null); + checkSeq('(a) findings', o.findings, [ + 'error|invalid|currency|Required', + ]); + note( + "(a) → `error` vs C1's `info`, `invalid` vs `undeclared`. Distinguishable in level AND in kind", + '', + ); + } + + // ── (b) optional in the schema → COMPLETE SILENCE ──────────────────────────────────────── + // `drift()` diffs raw against validated. An absent optional is absent in BOTH, so there is no + // diff, so there is nothing to classify. This is the case that bites: the field you marked + // optional two years ago because it was "sometimes missing" is now permanently gone, and the + // library's most-advertised feature is structurally unable to say so. + { + const o = await against( + z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string().optional(), + status: z.string(), + }), + ); + check('(b) the call SUCCEEDED', o.ok, true); + checkSeq('(b) findings', o.findings, []); + check( + '(b) `currency` on the value the caller got', + (o.data as Record)['currency'], + undefined, + ); + note( + '(b) → zero findings, zero levels, a successful call. An OPTIONAL field can be removed by the vendor and NOTHING in the drift system fires', + '', + ); + } + + // ── (c) `.default()` in the schema → `verbose|defaulted`, and a fabricated value ───────── + // The third outcome. The caller receives a `currency` the vendor did not send, and the only + // trace is the QUIETEST level in the vocabulary — dropped entirely by `severity: 'warn'`. + { + const o = await against( + z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string().default('usd'), + status: z.string(), + }), + ); + check('(c) the call SUCCEEDED', o.ok, true); + checkSeq('(c) findings', o.findings, [ + 'verbose|defaulted|currency|default applied', + ]); + check( + '(c) …and the caller received a currency the vendor never sent', + (o.data as Record)['currency'], + 'usd', + ); + note( + '(c) → for a MONEY field this is the same class of bug as C3: a plausible value manufactured at the boundary, logged at `verbose`', + '', + ); + } + + // ── (d) nullable in the schema → also silent, and for a different reason ───────────────── + // `.nullable()` covers a null VALUE, not an absent KEY, so a removal still hits the required + // check. Included because "make it nullable" is the reflex fix for C4 and it does not + // interact with removal at all. + { + const o = await against( + z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string().nullable(), + status: z.string(), + }), + ); + check('(d) the call FAILED (nullable ≠ optional)', o.ok, false); + checkSeq('(d) findings', o.findings, [ + 'error|invalid|currency|Required', + ]); + } + + // ── (e) the three outcomes side by side ────────────────────────────────────────────────── + // One vendor change, one wire body, four schemas, four different answers. + { + const rows: string[] = []; + for (const [label, currency] of [ + ['required ', z.string()], + ['optional ', z.string().optional()], + ['default()', z.string().default('usd')], + ['nullable ', z.string().nullable()], + ] as const) { + const o = await against( + z.object({ + transaction_id: z.number(), + amount: z.number(), + currency, + status: z.string(), + }), + ); + const level = o.findings[0]?.split('|')[0] ?? ''; + rows.push(`${label} → ok=${o.ok} level=${level}`); + } + checkSeq('(e) the same removal, four declarations', rows, [ + 'required → ok=false level=error', + 'optional → ok=true level=', + 'default() → ok=true level=verbose', + 'nullable → ok=false level=error', + ]); + note( + '(e) → the LEVEL of a removal is decided by your schema, not by the vendor. "Removal is breaking" is a property you have to have already declared', + '', + ); + } + + finish( + 'C2', + 'CONFIRMED WITH A CONDITION. A removed field IS distinguishable from C1\'s addition — `error|invalid|currency|Required` against `info|undeclared`, three levels apart and a different `change` kind — and the call fails with `data: null`. But that is true only when the field is REQUIRED in your schema. The same wire body against `.optional()` produces a successful call and ZERO findings; against `.default("usd")` it produces `verbose|defaulted` and hands the caller a currency the vendor never sent. Four declarations, four answers, one vendor change. The removal is classified by YOUR schema, where the addition in C1 was classified by the vendor', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c3-the-zero-dollar-test.ts b/docs/scenarios/proofs/intermittent-drift/c3-the-zero-dollar-test.ts new file mode 100644 index 00000000..98fa4fca --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c3-the-zero-dollar-test.ts @@ -0,0 +1,295 @@ +// C3 — THE DECIDING CLAIM. `transaction_id` stops being a number. +// +// The capture's fear: "if `"abc"` becomes `NaN` or `0` at info level, that is the $0 transaction +// with a warning nobody reads." Half of that is refuted and half of it is worse than written. +// +// REFUTED: the default posture is SAFE. A plain `z.number()` receiving `"12345"` is a HARD failure +// — `error|invalid`, the call throws, `data` is `null`. And even `z.coerce.number()` receiving +// `"abc"` is a hard failure, because `Number("abc")` is `NaN` and Zod rejects `NaN` as a number. +// The library does not manufacture a $0 charge out of `"abc"` on its own. +// +// WORSE: there are two ordinary schema spellings that DO, and the finding they produce is +// BYTE-IDENTICAL to the finding for the correct coercion. `warn|coerced|transaction_id| +// string -> number` is what you get when `"12345"` became `12345`, and it is exactly what you get +// when `"abc"` became `0`. The `detail` carries KINDS, not VALUES (drift.ts:77-83). And the +// nullable case reaches `0` with no `.catch()` at all, because `Number(null) === 0`. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c3-the-zero-dollar-test.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +/** What the caller ends up holding, plus everything the library said about it. */ +interface Received { + ok: boolean; + /** The value of `transaction_id` ON THE AWAITED PATH. The number this whole scenario is about. */ + value: unknown; + findings: string[]; + message: string | null; +} + +async function receive(schema: unknown, wire: unknown): Promise { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ + transaction_id: wire, + amount: 4200, + currency: 'usd', + status: 'succeeded', + }), + output: drift(schema as never), + trace: sink, + }); + const r = await call.safe(); + const data = r.data as Record | null; + return { + ok: r.ok, + value: data === null ? null : data['transaction_id'], + findings, + message: r.error?.message ?? null, + }; +} + +const rest = { + amount: z.number(), + currency: z.string(), + status: z.string(), +}; +const STRICT = z.object({ transaction_id: z.number(), ...rest }); +const COERCE = z.object({ transaction_id: z.coerce.number(), ...rest }); +const COERCE_CATCH = z.object({ + transaction_id: z.coerce.number().catch(0), + ...rest, +}); + +async function main(): Promise { + heading('C3 — the $0-transaction test'); + + // ── (a) number → string, strict schema. THE DEFAULT, AND IT IS SAFE ────────────────────── + { + const r = await receive(STRICT, '12345'); + check('(a) the call FAILED', r.ok, false); + check('(a) value the caller received', r.value, null); + checkSeq('(a) findings', r.findings, [ + 'error|invalid|transaction_id|Expected number, received string', + ]); + note( + '(a) → a plain `z.number()` makes a type change FATAL with the field named and the two types named. No coercion, no $0 charge', + '', + ); + } + + // ── (b) number → string, coercing schema, PLAUSIBLE value. Correct and warned ──────────── + { + const r = await receive(COERCE, '12345'); + check('(b) the call SUCCEEDED', r.ok, true); + check('(b) value the caller received', r.value, 12345); + check('(b) …and it is a number', typeof r.value, 'number'); + checkSeq('(b) findings', r.findings, [ + 'warn|coerced|transaction_id|string -> number', + ]); + note( + '(b) → the RIGHT answer: the correct value, and a `warn` saying the wire type moved. This is what `coerced` is for', + '', + ); + } + + // ── (c) number → string, coercing schema, NON-NUMERIC value. Also safe ─────────────────── + // The capture's worst case, and it does not happen: `Number("abc")` is `NaN`, and Zod's + // `z.number()` rejects `NaN`. The coercion fails closed. + { + const r = await receive(COERCE, 'abc'); + check('(c) the call FAILED', r.ok, false); + check('(c) value the caller received', r.value, null); + checkSeq('(c) findings', r.findings, [ + 'error|invalid|transaction_id|Expected number, received nan', + ]); + note( + '(c) → REFUTES the capture: `z.coerce.number()` on `"abc"` does NOT silently become NaN or 0. It is an `error` and the call fails', + '', + ); + } + + // ── (d) THE $0 TRANSACTION, ROUTE 1: `.catch(0)` ───────────────────────────────────────── + // `.catch()` is the ordinary Zod spelling for "this vendor is loose, don't fail my call". + // It turns (c)'s hard failure into a zero, at `warn`. + { + const r = await receive(COERCE_CATCH, 'abc'); + check('(d) the call SUCCEEDED', r.ok, true); + check('(d) VALUE THE CALLER RECEIVED', r.value, 0); + check( + '(d) …and it is a number, so nothing downstream blinks', + typeof r.value, + 'number', + ); + checkSeq('(d) findings', r.findings, [ + 'warn|coerced|transaction_id|string -> number', + ]); + } + + // ── (e) …and (b) and (d) are INDISTINGUISHABLE from the finding ───────────────────────── + // This is the actual finding of C3. Same level, same kind, same path, same detail. One is + // correct, one is a $0 charge, and the drift system reports them with the same 42 bytes. + { + const good = await receive(COERCE_CATCH, '12345'); + const bad = await receive(COERCE_CATCH, 'abc'); + check( + '(e) the two findings are byte-identical', + JSON.stringify(good.findings) === JSON.stringify(bad.findings), + true, + ); + checkSeq('(e) the finding both produce', good.findings, [ + 'warn|coerced|transaction_id|string -> number', + ]); + checkSeq( + '(e) the values behind it', + [good.value, bad.value], + [12345, 0], + ); + note( + '(e) → `detail` is `kindOf(old) -> kindOf(new)` (drift.ts:77-83). The values are never in the finding, so no alert built on findings can separate these two', + '', + ); + } + + // ── (f) THE $0 TRANSACTION, ROUTE 2: `null`, and no `.catch()` needed ──────────────────── + // `Number(null) === 0`. A vendor that starts sending `null` for `transaction_id` on the data + // that triggers it lands a hard zero in a plain `z.coerce.number()` schema. + { + const r = await receive(COERCE, null); + check('(f) the call SUCCEEDED', r.ok, true); + check('(f) VALUE THE CALLER RECEIVED', r.value, 0); + checkSeq('(f) findings', r.findings, [ + 'warn|coerced|transaction_id|null -> number', + ]); + note( + '(f) → no `.catch()`, no garbage string, no error. Just `null` and `z.coerce.number()`. This is C4 and C3 being the same bug', + '', + ); + } + + // ── (g) every wire value `z.coerce.number()` turns into exactly 0 ──────────────────────── + // Six of them, and only two reject. Any of these six on a money field is a $0 charge at + // `warn`, or — for `0` and `"0"` — no finding at all. + { + const rows: string[] = []; + for (const wire of [null, '', ' ', false, [], '0', 'abc', 12345]) { + const parsed = z.coerce.number().safeParse(wire); + rows.push( + `${JSON.stringify(wire)} → ${parsed.success ? JSON.stringify(parsed.data) : 'REJECTED'}`, + ); + } + checkSeq('(g) `z.coerce.number()` over eight wire values', rows, [ + 'null → 0', + '"" → 0', + '" " → 0', + 'false → 0', + '[] → 0', + '"0" → 0', + '"abc" → REJECTED', + '12345 → 12345', + ]); + note( + '(g) → six wire values become exactly `0`. Only `"abc"` (and `undefined`) reject. `"0"` and `0` produce no finding at all — they are not a coercion', + '', + ); + } + + // ── (h) the OTHER direction: number → string ───────────────────────────────────────────── + // Symmetric, and the coercion is total (every number has a string form), so it never fails. + { + const NUM_TO_STR = z.object({ transaction_id: z.string(), ...rest }); + const strict = await receive(NUM_TO_STR, 12345); + check('(h) `z.string()` on a number: call FAILED', strict.ok, false); + checkSeq('(h) findings', strict.findings, [ + 'error|invalid|transaction_id|Expected string, received number', + ]); + + const COERCE_STR = z.object({ + transaction_id: z.coerce.string(), + ...rest, + }); + const coerced = await receive(COERCE_STR, 12345); + check('(h) `z.coerce.string()`: call SUCCEEDED', coerced.ok, true); + check('(h) VALUE THE CALLER RECEIVED', coerced.value, '12345'); + checkSeq('(h) findings', coerced.findings, [ + 'warn|coerced|transaction_id|number -> string', + ]); + note( + '(h) → `warn`, correct value, and now every `===` against a numeric id in your code is false. A string id is not a $0 charge, it is a lookup miss', + '', + ); + } + + // ── (i) the TOLERANT schema: total blindness ───────────────────────────────────────────── + // `z.union([number, string])` and `z.unknown()` accept both shapes, so raw === validated, + // so `diff` produces nothing. The schema written specifically to survive the drift is the one + // that makes the drift undetectable. + { + const UNION = z.object({ + transaction_id: z.union([z.number(), z.string()]), + ...rest, + }); + const u = await receive(UNION, '12345'); + check('(i) union: call SUCCEEDED', u.ok, true); + check('(i) VALUE THE CALLER RECEIVED', u.value, '12345'); + checkSeq('(i) findings', u.findings, []); + + const UNKNOWN = z.object({ transaction_id: z.unknown(), ...rest }); + const k = await receive(UNKNOWN, 'abc'); + check('(i) z.unknown(): call SUCCEEDED', k.ok, true); + check('(i) VALUE THE CALLER RECEIVED', k.value, 'abc'); + checkSeq('(i) findings', k.findings, []); + note( + '(i) → drift is a diff against the validated value. A schema that accepts both shapes has nothing to diff, so a tolerant schema is a SILENT one', + '', + ); + } + + // ── (j) what the DOWNSTREAM cast does with each of them ────────────────────────────────── + // The library hands you a value; the $0 charge happens one line later. This is what + // `Number(x)` and `Number(x) || 0` — the two lines every integration has — do with what the + // caller actually received in (a)-(i). + { + const cases: [string, unknown][] = [ + ['coerce+catch "abc" (d)', 0], + ['coerce null (f)', 0], + ['union "12345" (i)', '12345'], + ['unknown "abc" (i)', 'abc'], + ['coerce.string 12345 (h)', '12345'], + ]; + const rows = cases.map( + ([label, v]) => + `${label}: Number()=${String(Number(v))} Number()||0=${String(Number(v) || 0)} ===12345 is ${String(v === 12345)}`, + ); + checkSeq('(j) the downstream cast', rows, [ + 'coerce+catch "abc" (d): Number()=0 Number()||0=0 ===12345 is false', + 'coerce null (f): Number()=0 Number()||0=0 ===12345 is false', + 'union "12345" (i): Number()=12345 Number()||0=12345 ===12345 is false', + 'unknown "abc" (i): Number()=NaN Number()||0=0 ===12345 is false', + 'coerce.string 12345 (h): Number()=12345 Number()||0=12345 ===12345 is false', + ]); + note( + '(j) → three separate routes to a literal 0, and every single row fails a `=== 12345` identity check. The silent ones (i) are the ones with no finding at all', + '', + ); + } + + finish( + 'C3', + 'THE CAPTURE IS HALF WRONG AND THE OTHER HALF IS WORSE. Refuted: the default is SAFE. `z.number()` on `"12345"` is `error|invalid|transaction_id|Expected number, received string` and the call FAILS with `data: null`; `z.coerce.number()` on `"abc"` also FAILS (`received nan`) because Zod rejects NaN. StitchAPI does not manufacture a $0 charge on its own. Confirmed and worse: TWO ordinary spellings do. `z.coerce.number().catch(0)` on `"abc"` hands the caller literal `0` at `warn`, and `z.coerce.number()` on `null` hands the caller literal `0` at `warn` with NO `.catch()` at all, because `Number(null) === 0` — six wire values (`null`, `""`, `" "`, `false`, `[]`, `"0"`) coerce to exactly 0 and only `"abc"` rejects. And the finding for `"12345" -> 12345` is BYTE-IDENTICAL to the finding for `"abc" -> 0`: both are `warn|coerced|transaction_id|string -> number`, because `detail` is `kindOf(old) -> kindOf(new)` (drift.ts:77-83) and the VALUES are never in the finding. The tolerant schemas people write to survive drift — `z.union([number,string])`, `z.unknown()` — pass the raw string through with ZERO findings', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c4-intermittent-null.ts b/docs/scenarios/proofs/intermittent-drift/c4-intermittent-null.ts new file mode 100644 index 00000000..fd02d739 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c4-intermittent-null.ts @@ -0,0 +1,294 @@ +// C4 — a field becomes `null` on ~5% of responses. The geocoder case: nothing is wrong until a +// query happens to be ambiguous, so the shape differs BY DATA and never by deployment. +// +// Two questions from the capture, and the answers point in opposite directions. +// +// Does drift fire only on the drifting calls, and does the finding name the field? YES to both, +// exactly — 5 findings over 100 calls, on calls 20/40/60/80/100, each naming `formatted_address`. +// That is the library doing its job. +// +// Is "nullable" a WARNING-level change class the way the industry taxonomy has it? NO. There is no +// nullable level. `null` against a strict schema is `error` and the call fails; against +// `.nullable()` it is declared variance and produces NOTHING; the only way to get a `warn` is a +// coercion, which means fabricating a value. Three outcomes, none of them "warn, value intact". +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c4-intermittent-null.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + Adapter, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +/** A geocoder result. `formatted_address` is the field that goes null on ambiguous queries. */ +const Place = z.object({ + place_id: z.string(), + formatted_address: z.string(), + lat: z.number(), + lng: z.number(), +}); + +/** 5% of queries are ambiguous — deterministic, so "5 of 100" is a fact. */ +function geocoder(): { adapter: Adapter; ambiguousAt: number[] } { + let n = 0; + const ambiguousAt: number[] = []; + return { + ambiguousAt, + adapter: async () => { + n += 1; + const ambiguous = n % 20 === 0; + if (ambiguous) ambiguousAt.push(n); + return { + status: 200, + headers: {}, + body: { + place_id: `p${n}`, + formatted_address: ambiguous ? null : `${n} Main St`, + lat: 51.5, + lng: -0.12, + }, + }; + }, + }; +} + +async function main(): Promise { + heading('C4 — `formatted_address` is null on 5% of responses'); + + // ── (a) does drift fire ONLY on the drifting calls, and does it name the field? ────────── + // The schema has to admit the null for the call to survive, so this uses `.nullable()` plus a + // `.catch('')` — the shape a team reaches for after the first outage. The `.catch` is what + // turns the null into a reportable coercion; see (c) for what `.nullable()` alone does. + { + const { adapter, ambiguousAt } = geocoder(); + const firedOn: number[] = []; + const findings: string[] = []; + let call = 0; + const sink: TraceSink = { + handle(e: StitchEvent, _ctx: TraceContext) { + if (e.type === 'start') call += 1; + if (e.type === 'drift') { + firedOn.push(call); + findings.push(fmt(e.finding)); + } + }, + }; + const Tolerant = z.object({ + place_id: z.string(), + formatted_address: z.string().catch(''), + lat: z.number(), + lng: z.number(), + }); + const geocode = stitch({ + url: 'https://geo.example/lookup', + adapter, + output: drift(Tolerant), + trace: sink, + }); + const okCount = { yes: 0, no: 0 }; + for (let i = 0; i < 100; i += 1) { + const r = await geocode.safe(); + if (r.ok) okCount.yes += 1; + else okCount.no += 1; + } + + check('(a) calls made', call, 100); + check('(a) calls the vendor actually nulled', ambiguousAt.length, 5); + check('(a) drift findings', findings.length, 5); + checkSeq('(a) the calls drift fired on', firedOn, ambiguousAt); + checkSeq( + '(a) …and every finding is the same one', + [...new Set(findings)], + ['warn|coerced|formatted_address|null -> string'], + ); + check('(a) successful calls', okCount.yes, 100); + note( + '(a) → drift fires on exactly the 5 calls the vendor drifted on and on none of the other 95, and the finding NAMES `formatted_address`. This is the library working', + '', + ); + } + + // ── (b) …but what the caller receives on those 5 calls is a FABRICATED value ───────────── + // `.catch('')` is what made (a) reportable. It is also what makes the null invisible one line + // downstream: `place.formatted_address` is a string on all 100 calls. + { + const { adapter } = geocoder(); + const Tolerant = z.object({ + place_id: z.string(), + formatted_address: z.string().catch(''), + lat: z.number(), + lng: z.number(), + }); + const geocode = stitch({ + url: 'https://geo.example/lookup', + adapter, + output: drift(Tolerant), + }); + const values: unknown[] = []; + for (let i = 0; i < 40; i += 1) { + const r = await geocode.safe(); + values.push( + (r.data as Record)['formatted_address'], + ); + } + checkSeq( + '(b) values on calls 19, 20, 21', + [values[18], values[19], values[20]], + ['19 Main St', '', '21 Main St'], + ); + check( + '(b) how many of 40 are the empty string', + values.filter((v) => v === '').length, + 2, + ); + check( + '(b) …and every value is a string, so a `typeof` guard passes', + values.every((v) => typeof v === 'string'), + true, + ); + } + + // ── (c) the four ways to declare a nullable field, and what each reports ───────────────── + // The industry taxonomy wants "nullable ⇒ warn, value intact". None of the four does that. + { + const rows: string[] = []; + for (const [label, field] of [ + ['required ', z.string()], + ['.nullable() ', z.string().nullable()], + ['.catch("") ', z.string().catch('')], + ['.nullish() ', z.string().nullish()], + ] as const) { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + url: 'https://geo.example/lookup', + adapter: serving({ + place_id: 'p1', + formatted_address: null, + lat: 51.5, + lng: -0.12, + }), + output: drift( + z.object({ + place_id: z.string(), + formatted_address: field, + lat: z.number(), + lng: z.number(), + }) as never, + ), + trace: sink, + }); + const r = await call.safe(); + // `drift(... as never)` erases the contract type, so `data` needs a widening cast. + const data = r.data as unknown as Record | null; + const value = + data === null + ? '' + : JSON.stringify(data['formatted_address']); + rows.push( + `${label} ok=${r.ok} finding=${findings[0] ?? ''} value=${value}`, + ); + } + checkSeq('(c) one null, four declarations', rows, [ + 'required ok=false finding=error|invalid|formatted_address|Expected string, received null value=', + '.nullable() ok=true finding= value=null', + '.catch("") ok=true finding=warn|coerced|formatted_address|null -> string value=""', + '.nullish() ok=true finding= value=null', + ]); + note( + '(c) → there is NO "nullable" level. You get `error` + a failed call, or silence + the null, or `warn` + a fabricated value. "Warning-level, value intact" is not one of the options', + '', + ); + } + + // ── (d) the silent variant is the one a team actually ships ────────────────────────────── + // `.nullable()` is the correct schema for "this field is sometimes null", and it produces zero + // findings on all 100 calls. The 5% rollout is completely invisible to drift. + { + const { adapter } = geocoder(); + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const geocode = stitch({ + url: 'https://geo.example/lookup', + adapter, + output: drift( + z.object({ + place_id: z.string(), + formatted_address: z.string().nullable(), + lat: z.number(), + lng: z.number(), + }), + ), + trace: sink, + }); + const nulls: number[] = []; + for (let i = 0; i < 100; i += 1) { + const r = await geocode.safe(); + if ( + (r.data as Record)['formatted_address'] === + null + ) + nulls.push(i + 1); + } + check('(d) calls that returned null', nulls.length, 5); + checkSeq('(d) drift findings over the same 100 calls', findings, []); + note( + '(d) → declared variance produces no finding (drift.ts:104-111 diffs raw against validated, and they are equal). The rate is only recoverable by inspecting `data` yourself', + '', + ); + } + + // ── (e) the strict variant, which does fire — as a 5% ERROR RATE ───────────────────────── + // The honest alternative to (d): leave the field required and let the 5% fail. It is loud, it + // names the field, and it costs you the other four fields on every ambiguous query. + { + const { adapter } = geocoder(); + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const geocode = stitch({ + url: 'https://geo.example/lookup', + adapter, + output: drift(Place), + trace: sink, + }); + let failed = 0; + for (let i = 0; i < 100; i += 1) { + const r = await geocode.safe(); + if (!r.ok) failed += 1; + } + check('(e) failed calls out of 100', failed, 5); + check('(e) findings', findings.length, 5); + checkSeq( + '(e) the finding', + [...new Set(findings)], + ['error|invalid|formatted_address|Expected string, received null'], + ); + note( + '(e) → a 5% hard failure rate, each one naming the field. This IS the detection the scenario wants; the price is that the caller loses `place_id`, `lat` and `lng` too', + '', + ); + } + + finish( + 'C4', + 'CONFIRMED ON PRECISION, REFUTED ON LEVEL. Drift fires on EXACTLY the drifting calls and nowhere else — 5 findings over 100 calls, on calls [20,40,60,80,100], matching the vendor ledger, and each one NAMES `formatted_address` with `null -> string`. No false positives on the other 95. But there is no NULLABLE LEVEL: one null against four declarations gives `error|invalid` + a failed call (required), NOTHING at all (`.nullable()` / `.nullish()` — declared variance, raw === validated, no diff), or `warn|coerced` + a FABRICATED `""` (`.catch("")`). The industry class "nullable is warning-level, value intact" is not expressible. `.nullable()` is the schema a team actually ships and it makes the 5% rollout completely invisible; the strict schema turns it into a 5% error rate that also discards the four fields that were fine', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c5-severity.ts b/docs/scenarios/proofs/intermittent-drift/c5-severity.ts new file mode 100644 index 00000000..7755e74d --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c5-severity.ts @@ -0,0 +1,251 @@ +// C5 — can the four change classes be given DIFFERENT severities, declaratively? +// +// The real config is `DriftOptions` (types.ts:83-116) with exactly two keys: `severity` and +// `ignore`. `severity` is keyed by `SoftDriftChange` = `'undeclared' | 'coerced' | 'defaulted'` +// (types.ts:72) and its values are `DriftSeverity` = `'warn' | 'info' | 'verbose'` (types.ts:74). +// +// So the answer is a qualified yes with one sharp edge and one hole: +// +// - The vocabulary is keyed by MECHANISM (what your schema did to the value), not by CHANGE +// CLASS (what the vendor did to the shape). Addition maps cleanly onto `undeclared`; the other +// three do not have a stable home. +// - `error` is not in `DriftSeverity`, so you cannot promote a soft finding to fatal — the docs +// say fatality is the schema's job. The RUNTIME honours `'error'` anyway if you cast past the +// type. That is measured here, both directions. +// - `ignore` is per-PATH, `severity` is per-KIND. There is no per-path severity, so "a coercion +// on `transaction_id` is a page, a coercion on `description` is noise" is not expressible. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c5-severity.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + DriftOptions, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +/** A schema that can produce all three soft kinds at once. */ +const Charge = z.object({ + transaction_id: z.coerce.number().catch(0), // → `coerced` when the wire type shifts + amount: z.number(), + currency: z.string().default('usd'), // → `defaulted` when the vendor drops it + status: z.string(), +}); + +/** A body that trips all three kinds on one call: a retype, a removal, and an addition. */ +const ALL_THREE = { + transaction_id: '100001', + amount: 4200, + status: 'succeeded', + settlement_delay_ms: 900, +}; + +interface Run { + ok: boolean; + findings: string[]; +} + +async function run( + opts: DriftOptions, + body: unknown = ALL_THREE, +): Promise { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving(body), + output: drift(Charge, opts), + trace: sink, + }); + const r = await call.safe(); + return { ok: r.ok, findings: findings.sort() }; +} + +async function main(): Promise { + heading('C5 — leveling the change classes declaratively'); + + // ── (a) the defaults, all three kinds on one call ──────────────────────────────────────── + // drift.ts:65-69 — `undeclared` → info, `coerced` → warn, `defaulted` → verbose. + { + const r = await run({}); + check('(a) the call SUCCEEDED', r.ok, true); + checkSeq('(a) findings at the per-kind defaults', r.findings, [ + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + 'verbose|defaulted|currency|default applied', + 'warn|coerced|transaction_id|string -> number', + ]); + note( + '(a) → three kinds, three levels, out of the box, on one response. This is the vocabulary the capture hoped existed', + '', + ); + } + + // ── (b) the map form RE-LEVELS each kind ───────────────────────────────────────────────── + // "addition silent-ish, coercion loud, default loud" — expressible in one object literal. + { + const r = await run({ + severity: { + undeclared: 'verbose', + coerced: 'warn', + defaulted: 'warn', + }, + }); + checkSeq('(b) findings after re-leveling', r.findings, [ + 'verbose|undeclared|settlement_delay_ms|undeclared field (number)', + 'warn|coerced|transaction_id|string -> number', + 'warn|defaulted|currency|default applied', + ]); + note( + '(b) → the three SOFT kinds are fully re-levelable across the three soft levels. Nine combinations, one literal', + '', + ); + } + + // ── (c) the allowlist form DROPS findings entirely ─────────────────────────────────────── + // `severity: 'warn'` is not "show warn prominently", it is "emit only warn". The dropped + // findings never reach the event stream, so they never reach a counter either (C6). + { + const r = await run({ severity: 'warn' }); + checkSeq('(c) findings with `severity: "warn"`', r.findings, [ + 'warn|coerced|transaction_id|string -> number', + ]); + const two = await run({ severity: ['warn', 'info'] }); + checkSeq( + '(c) findings with `severity: ["warn","info"]`', + two.findings, + [ + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + 'warn|coerced|transaction_id|string -> number', + ], + ); + note( + '(c) → filtering happens at EMISSION (drift.ts:147), not at consumption. A finding you filtered out is invisible to the trace sink, so you cannot filter and count the same kind', + '', + ); + } + + // ── (d) `ignore` is per-PATH; `severity` is per-KIND. There is no per-path severity ────── + // The thing a payments team actually wants — coercion on `transaction_id` is a page, coercion + // on `description` is noise — has no spelling. You can only silence a path completely. + { + const Two = z.object({ + transaction_id: z.coerce.number().catch(0), + description: z.coerce.string().catch(''), + amount: z.number(), + }); + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ + transaction_id: '100001', + description: 99, + amount: 4200, + }), + output: drift(Two, { severity: { coerced: 'warn' } }), + trace: sink, + }); + await call.safe(); + checkSeq('(d) two coercions, one severity', findings.sort(), [ + 'warn|coerced|description|number -> string', + 'warn|coerced|transaction_id|string -> number', + ]); + + const silenced: string[] = []; + const sink2: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') silenced.push(fmt(e.finding)); + }, + }; + const call2 = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving({ + transaction_id: '100001', + description: 99, + amount: 4200, + }), + output: drift(Two, { ignore: ['description'] }), + trace: sink2, + }); + await call2.safe(); + checkSeq('(d) `ignore: ["description"]`', silenced, [ + 'warn|coerced|transaction_id|string -> number', + ]); + note( + '(d) → the only per-path lever is ON/OFF. `resolveSeverity` (drift.ts:90-101) takes the CHANGE KIND and nothing else — the path never reaches it', + '', + ); + } + + // ── (e) you cannot promote a soft finding to `error` through the type ──────────────────── + // `DriftSeverity` excludes `'error'` by construction (types.ts:74) and the JSDoc is explicit: + // "to fail on a change, make the field required/strict in the schema". That is the documented + // route, and it is the one C2(a) and C3(a) measured. + { + // @ts-expect-error `'error'` is not assignable to DriftSeverity — this is the point. + const rejected: DriftOptions = { severity: { coerced: 'error' } }; + check( + '(e) the type REJECTS `severity: { coerced: "error" }`', + typeof rejected, + 'object', + ); + note( + '(e) → the `@ts-expect-error` above is the assertion: if the type ever starts allowing it, this file stops compiling', + '', + ); + } + + // ── (f) …but the RUNTIME honours it if you cast past the type ──────────────────────────── + // `levelOf` returns whatever the map says (drift.ts:100) and the engine treats any + // `level === 'error'` finding as fatal (engine.ts:1215). So the capability exists, off-contract. + // Reported in both directions: it works, and it is not something to build on. + { + const cast = { + severity: { coerced: 'error' }, + } as unknown as DriftOptions; + const r = await run(cast); + check('(f) the call FAILED', r.ok, false); + checkSeq('(f) findings', r.findings, [ + 'error|coerced|transaction_id|string -> number', + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + 'verbose|defaulted|currency|default applied', + ]); + note( + '(f) → an `error`-leveled COERCED finding, and the call failed with `data: null`. Type-blocked, runtime-live. The documented way to get here is a strict schema, which also gives you a better message', + '', + ); + } + + // ── (g) the four industry classes vs the three soft kinds, side by side ────────────────── + // The mapping is not one-to-one and this is the table that says so. + { + const rows = [ + 'addition → undeclared (info) — clean 1:1', + 'removal → invalid/defaulted/none — decided by your schema (C2)', + 'type change→ invalid/coerced/none — decided by your schema (C3)', + 'nullable → invalid/coerced/none — no nullable level at all (C4)', + ]; + checkSeq('(g) taxonomy vs vocabulary', rows, rows); + note( + '(g) → only ADDITION has a stable home. The other three land on a kind chosen by how you declared the field, so "removal is fatal" is a schema decision, not a severity decision', + '', + ); + } + + finish( + 'C5', + 'PARTLY EXPRESSIBLE, AND KEYED ON THE WRONG AXIS. The three SOFT kinds are fully re-levelable in one literal — `severity: { undeclared: "verbose", coerced: "warn", defaulted: "warn" }` measured exactly that — and `severity: "warn"` / `["warn","info"]` is an emission-time allowlist. But `severity` is keyed by MECHANISM (`undeclared`/`coerced`/`defaulted`, types.ts:72), not by the four industry CHANGE CLASSES, and only ADDITION maps 1:1. Removal, type change and nullability each land on a kind decided by your schema, so their loudness is a schema decision. There is NO per-path severity — `ignore` is the only path-aware lever and it is on/off (drift.ts:90-101 never sees the path) — so "coercion on `transaction_id` pages, coercion on `description` does not" has no spelling. And `error` is not in `DriftSeverity`, so a soft finding cannot be promoted to fatal through the type (the `@ts-expect-error` in (e) asserts it) — though a cast past the type DOES fail the call at runtime, measured in (f)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c6-aggregation.ts b/docs/scenarios/proofs/intermittent-drift/c6-aggregation.ts new file mode 100644 index 00000000..d0875383 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c6-aggregation.ts @@ -0,0 +1,430 @@ +// C6 — DECIDING CLAIM. Over 100 calls where 5 drift, can the caller learn "5% of calls drifted on +// field X"? +// +// The capture predicted this was the gap, on the strength of every other scenario in this pass +// finding no cross-call state: no per-call slot on `HookContext` (scenario 7), no run-scoped state +// (scenario 7), closures that leak across calls (scenario 11). +// +// The prediction is WRONG. `trace` is a genuine aggregation seam, and it is the one place in the +// library where cross-call state is the design rather than a leak: a `TraceSink` is configured +// once on a stitch or a seam, receives `handle(event, ctx)` for every event of every call through +// it, and `ctx.spanId` identifies the logical call. Measured: 100 calls, 5 drifted, the sink +// reported `5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)` — the +// rate, the field, and the fact that the coerced value was a zero. +// +// The library still counts NOTHING itself. Three traps in the counting are measured below, and one +// of them (the cache) can divide a real rate by five without any code being wrong. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c6-aggregation.ts +import { drift, seam, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + HookContext, + StitchEvent, + StitchStore, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { DriftRate } from './drift-rate'; +import { FakeVendor, fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +const Charge = z.object({ + transaction_id: z.coerce.number().catch(0), + amount: z.number(), + currency: z.string(), + status: z.string(), +}); + +async function main(): Promise { + heading('C6 — 5 of 100 calls drift. Can you learn the RATE?'); + + // ── (a) the answer: a TraceSink counts it, and the number is right ─────────────────────── + { + const vendor = new FakeVendor({ mutation: 'nulled', rate: 0.05 }); + const rate = new DriftRate({ zeroWatch: ['transaction_id'] }); + const charge = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: vendor.adapter(), + output: drift(Charge), + trace: rate, + }); + + const values: unknown[] = []; + for (let i = 0; i < 100; i += 1) { + const r = await charge.safe(); + values.push((r.data as Record)['transaction_id']); + } + + check('(a) calls the sink counted', rate.calls, 100); + check('(a) calls the vendor actually drifted', vendor.mutatedCount, 5); + checkSeq('(a) the report', rate.report(), [ + '5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)', + ]); + check( + '(a) callers that received a literal 0', + values.filter((v) => v === 0).length, + 5, + ); + note( + '(a) → the rate, the level, the kind, the FIELD, and the fact that the value landed on zero. That is an alertable line, and no library code produced it', + '', + ); + } + + // ── (b) TRAP 1: findings are not calls ─────────────────────────────────────────────────── + // Two fields drifting on one response emits two findings. `findings / calls` reports 200%. + { + const rate = new DriftRate(); + const perCall: string[] = []; + const raw: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type === 'drift') + perCall.push( + `${ctx.spanId?.slice(0, 4) ?? '?'}:${fmt(e.finding)}`, + ); + }, + }; + const both: TraceSink = { + handle(e, ctx) { + rate.handle(e, ctx); + raw.handle(e, ctx); + }, + }; + const call = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: serving({ + transaction_id: null, + amount: 4200, + currency: 'usd', + status: 'succeeded', + settlement_delay_ms: 900, + }), + output: drift(Charge), + trace: both, + }); + await call.safe(); + + check('(b) calls made', rate.calls, 1); + check('(b) findings emitted', perCall.length, 2); + check( + '(b) distinct spans behind them', + new Set(perCall.map((p) => p.split(':')[0])).size, + 1, + ); + checkSeq( + '(b) rows, each counting DRIFTED CALLS not findings', + rate + .rows() + .map((r) => `${r.key} findings=${r.findings} calls=${r.calls}`), + [ + 'warn|coerced|transaction_id|null -> number findings=1 calls=1', + 'info|undeclared|settlement_delay_ms|undeclared field (number) findings=1 calls=1', + ], + ); + note( + '(b) → a naive `findings / calls` on this one response is 200%. `ctx.spanId` is what collapses it back to "1 of 1 calls drifted, on two fields"', + '', + ); + } + + // ── (c) TRAP 2: the cache divides your drift rate by the hit ratio ─────────────────────── + // A cache hit emits `start` and `result` (so the denominator grows) but no drift (the engine + // states outright that soft drift is meaningless on a hit, engine.ts:1605-1613). Five calls + // against a 100%-drifting vendor, one wire request, and the measured rate is 20%. + { + const vendor = new FakeVendor({ mutation: 'nulled', rate: 1 }); + const rate = new DriftRate(); + const cached = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: vendor.adapter(), + output: drift(Charge), + trace: rate, + cache: { ttl: 60_000, version: 'v1' }, + }); + for (let i = 0; i < 5; i += 1) await cached.safe(); + + check('(c) wire requests', vendor.calls.length, 1); + check('(c) responses the vendor drifted', vendor.mutatedCount, 1); + check('(c) calls the sink counted', rate.calls, 5); + checkSeq('(c) the report', rate.report(), [ + '20.0% of calls: warn|coerced|transaction_id|null -> number (1/5)', + ]); + note( + '(c) → the TRUE vendor drift rate is 100%. The sink says 20%, and every line of code involved is correct. Cache hit ratio is a hidden divisor on any drift rate', + '', + ); + } + + // ── (d) TRAP 3: `.report()` / `.inspect()` pollute BOTH sides of the fraction ──────────── + // A diagnostic probe is a real run: it makes a request, and it emits `start` + `drift` into + // the same sink. Reach for `.report()` in a catch block and your rate moves. + { + const vendor = new FakeVendor({ mutation: 'nulled', rate: 1 }); + const rate = new DriftRate(); + const call = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: vendor.adapter(), + output: drift(Charge), + trace: rate, + }); + await call.safe(); + const before = { calls: rate.calls, wire: vendor.calls.length }; + await call.report(); + checkSeq( + '(d) calls/wire before and after one `.report()`', + [before.calls, before.wire, rate.calls, vendor.calls.length], + [1, 1, 2, 2], + ); + note( + '(d) → `.report()` is a fresh run (stitch.ts:1024-1090), not a read of the run you made. It costs a request AND a tick in your denominator', + '', + ); + } + + // ── (e) what the sink CAN do that no other accessor can: join to the value ─────────────── + // C3(e) established that `warn|coerced|transaction_id|string -> number` is identical for + // `"12345" -> 12345` and `"abc" -> 0`. The `result` event carries the validated `data` on the + // SAME span, so the sink is the one place both halves are in scope at once. + { + const bodies = [ + { + transaction_id: '12345', + amount: 4200, + currency: 'usd', + status: 'ok', + }, + { + transaction_id: 'abc', + amount: 4200, + currency: 'usd', + status: 'ok', + }, + { + transaction_id: null, + amount: 4200, + currency: 'usd', + status: 'ok', + }, + ]; + let i = 0; + const rate = new DriftRate({ zeroWatch: ['transaction_id'] }); + const call = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: async () => ({ + status: 200, + headers: {}, + body: bodies[i++], + }), + output: drift(Charge), + trace: rate, + }); + for (let k = 0; k < 3; k += 1) await call.safe(); + + checkSeq('(e) the report', rate.report(), [ + '66.7% of calls: warn|coerced|transaction_id|string -> number (2/3, 1 landed 0)', + '33.3% of calls: warn|coerced|transaction_id|null -> number (1/3, 1 landed 0)', + ]); + note( + '(e) → `2/3, 1 landed 0` is the whole point: two calls produced the SAME finding and only one of them was a $0 charge. The finding alone cannot say that; the join can', + '', + ); + } + + // ── (f) a rolling window, so the rate is a rate and not a lifetime average ─────────────── + // A canary that goes 5% → 25% has to be visible AS A CHANGE. Time is injected, so this is + // deterministic. + { + const clock = manualClock(); + const rate = new DriftRate({ clock, window: 60_000 }); + let phase: 'five' | 'twentyfive' = 'five'; + let n = 0; + const call = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: async () => { + n += 1; + const period = phase === 'five' ? 20 : 4; + const drifty = n % period === 0; + return { + status: 200, + headers: {}, + body: { + transaction_id: drifty ? null : 100000 + n, + amount: 4200, + currency: 'usd', + status: 'ok', + }, + }; + }, + output: drift(Charge), + trace: rate, + clock, + }); + + for (let i = 0; i < 100; i += 1) { + await call.safe(); + await clock.advance(500); // 100 calls over 50s — inside the 60s window + } + const yesterday = rate.report(); + + // The canary widens. Move past the window so the old sample is fully evicted. + phase = 'twentyfive'; + n = 0; + await clock.advance(120_000); + for (let i = 0; i < 100; i += 1) { + await call.safe(); + await clock.advance(500); + } + const today = rate.report(); + + checkSeq('(f) window 1', yesterday, [ + '5.0% of calls: warn|coerced|transaction_id|null -> number (5/100)', + ]); + checkSeq('(f) window 2, after the canary widened', today, [ + '25.0% of calls: warn|coerced|transaction_id|null -> number (25/100)', + ]); + check('(f) virtual ms elapsed', clock.now(), 220_000); + note( + '(f) → 5% → 25% on the same field, each window standing alone. That is the alert the scenario is asking for, and it is 5 lines of eviction in user code', + '', + ); + } + + // ── (g) a SEAM-level sink aggregates across endpoints, tagged by name ──────────────────── + // `seam({ trace })` puts one sink under every stitch it builds, and `ctx.name` says which + // endpoint drifted — the shape a real service needs. + { + const rows: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent, ctx: TraceContext) { + if (e.type === 'drift') + rows.push(`${ctx.name}:${e.finding.path}`); + }, + }; + const api = seam({ + baseUrl: 'https://pay.example', + adapter: serving({ + transaction_id: null, + amount: 4200, + currency: 'usd', + status: 'ok', + }), + trace: sink, + }); + const charge = api.stitch({ + name: 'charge', + path: '/charges', + output: drift(Charge), + }); + const refund = api.stitch({ + name: 'refund', + path: '/refunds', + output: drift(Charge), + }); + await charge.safe(); + await refund.safe(); + await charge.safe(); + checkSeq('(g) one sink, two endpoints', rows, [ + 'charge:transaction_id', + 'refund:transaction_id', + 'charge:transaction_id', + ]); + } + + // ── (h) the seams that DO NOT work, measured ───────────────────────────────────────────── + // For completeness, because the capture asked "where would a counter live". + { + // hooks: `HookContext` is `{ name, attempt, req?, res?, error? }` (types.ts:1278-1284) and + // `onResponse` runs BEFORE validation, so there is no finding to count there. + const keys: string[] = []; + const call = stitch({ + url: 'https://pay.example/charges', + adapter: serving({ + transaction_id: null, + amount: 4200, + currency: 'usd', + status: 'ok', + }), + output: drift(Charge), + hooks: { + onResponse: (c: HookContext) => { + keys.push(Object.keys(c).sort().join(',')); + }, + }, + }); + const r = await call.safe(); + checkSeq('(h) `HookContext` keys', keys, ['attempt,name,res']); + checkSeq('(h) `SafeResult` keys', Object.keys(r).sort(), [ + 'data', + 'error', + 'ok', + ]); + note( + '(h) → neither hooks nor `.safe()` carries a finding. A soft drift is INVISIBLE on the awaited path — `ok: true`, `error: null`, and nothing else to read', + '', + ); + + // A `StitchStore` can hold the counter (Redis, for a rate across workers), but `handle` is + // SYNCHRONOUS (`handle(...): void`, types.ts:1957-1960) so the increment is fire-and-forget + // and you own the promise. Measured working, with the caveat on the page. + const map = new Map(); + const store: StitchStore = { + async get(k) { + return map.get(k); + }, + async set() {}, + async increment(k) { + const v = (map.get(k) ?? 0) + 1; + map.set(k, v); + return v; + }, + }; + const inflight: Promise[] = []; + const storeSink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'start') inflight.push(store.increment('calls')); + if (e.type === 'drift') + inflight.push(store.increment(`drift:${e.finding.path}`)); + }, + }; + const counted = stitch({ + url: 'https://pay.example/charges', + adapter: serving({ + transaction_id: null, + amount: 4200, + currency: 'usd', + status: 'ok', + }), + output: drift(Charge), + trace: storeSink, + }); + for (let i = 0; i < 4; i += 1) await counted.safe(); + await Promise.all(inflight); + checkSeq( + '(h) a store-backed counter', + [...map], + [ + ['calls', 4], + ['drift:transaction_id', 4], + ], + ); + note( + '(h) → `StitchStore.increment` makes the rate cross-process. `TraceSink.handle` returns `void`, so nothing awaits it: you hold the promises yourself and drain them on `seam.flush()`', + '', + ); + } + + finish( + 'C6', + "THE CAPTURE'S PREDICTION IS REFUTED — aggregation WORKS, and `trace` is a real seam. 100 calls, 5 drifted, and the sink reported `5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)`; a rolling window on an injected clock showed the canary widening 5.0% → 25.0% on the same field. A `TraceSink` is configured once, sees every event of every call, and `ctx.spanId` is what makes per-call state possible — the one place in the library where cross-call state is the design and not a leak. A seam-level sink aggregates across endpoints tagged by `ctx.name`, and `StitchStore.increment` takes it cross-process. Three traps, all measured: findings are not calls (2 findings on 1 response reads as 200% without a `spanId` collapse); a CACHE HIT emits `start`+`result` but no drift, so 5 calls against a 100%-drifting vendor measured 20%; and `.report()` is a fresh run that adds a request AND a tick to the denominator. Neither `hooks` nor `.safe()` can see a soft finding at all", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c7-accessors.ts b/docs/scenarios/proofs/intermittent-drift/c7-accessors.ts new file mode 100644 index 00000000..8fbf1c2f --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c7-accessors.ts @@ -0,0 +1,347 @@ +// C7 — is the finding ACTIONABLE at 3am: field path, expected, actual? And which of the six +// accessors carries it? +// +// Scenario 11 measured that `.safe()` gets a generic message while `.report().findings` has the +// detail. That generalises here, and the shape of the answer is a table with a hole in it: +// +// await / .safe() — NOTHING on a soft finding, and a GENERIC message on a hard one +// hooks.onResponse — runs before validation; no finding exists yet +// .stream() — every finding, same run, one request ← the cheap live accessor +// trace sink — every finding, same run, zero cost ← the one that also aggregates +// .inspect() — findings + raw + validated, FRESH PROBE ← the only one with both VALUES +// .report() — the above plus attempts/timing/config, FRESH PROBE +// +// "Expected vs actual" is only half-carried. A HARD finding has both (`Expected number, received +// string`) because the message comes from Zod. A SOFT finding has neither: `detail` is +// `kindOf(old) -> kindOf(new)` (drift.ts:77-83), which is types, not values. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c7-accessors.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { fmt, serving } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +const Charge = z.object({ + transaction_id: z.coerce.number().catch(0), + amount: z.number(), + currency: z.string(), + status: z.string(), +}); +const Strict = z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string(), + status: z.string(), +}); + +/** The drifting response every accessor in this file is pointed at: `"abc"` becomes `0`. */ +const DRIFTED = { + transaction_id: 'abc', + amount: 4200, + currency: 'usd', + status: 'succeeded', + settlement_delay_ms: 900, +}; + +async function main(): Promise { + heading('C7 — which accessor carries the finding'); + + // ── (a) await / `.safe()` on a SOFT finding: nothing at all ────────────────────────────── + { + let wire = 0; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: async () => { + wire += 1; + return { status: 200, headers: {}, body: DRIFTED }; + }, + output: drift(Charge), + }); + const r = await call.safe(); + checkSeq('(a) `SafeResult` keys', Object.keys(r).sort(), [ + 'data', + 'error', + 'ok', + ]); + check('(a) ok', r.ok, true); + check('(a) error', r.error, null); + check( + '(a) the value the caller holds', + (r.data as Record)['transaction_id'], + 0, + ); + check('(a) requests made', wire, 1); + note( + '(a) → a $0 charge, a successful call, and NOTHING on the result object to read. The finding exists and this accessor cannot see it', + '', + ); + } + + // ── (b) await / `.safe()` on a HARD finding: a generic message ─────────────────────────── + // `StitchError` has `{ status, attempts, body, url }` and no `findings` (types.ts:1656-1690). + // The field name is emitted on the event stream and then dropped on the way out. + { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + if (e.type === 'error') findings.push(`event:${e.message}`); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: serving(DRIFTED), + output: drift(Strict), + trace: sink, + }); + const r = await call.safe(); + check( + '(b) error message on `.safe()`', + r.error?.message, + 'contract violation (drift)', + ); + checkSeq( + '(b) `StitchError` own keys', + Object.keys(r.error ?? {}).sort(), + ['attempts', 'body', 'name', 'status', 'url'], + ); + check( + '(b) is `findings` on the error?', + 'findings' in (r.error ?? {}), + false, + ); + checkSeq('(b) what the SINK saw for the same run', findings, [ + 'error|invalid|transaction_id|Expected number, received string', + 'event:contract violation (drift)', + ]); + note( + '(b) → the sink names the field and both types. `.safe()` gets four words. Same run, same instant', + '', + ); + } + + // ── (c) `.stream()`: the same run, every finding, one request ──────────────────────────── + // The cheapest live accessor. You reassemble the result from the `result` event. + { + let wire = 0; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: async () => { + wire += 1; + return { status: 200, headers: {}, body: DRIFTED }; + }, + output: drift(Charge), + }); + const types: string[] = []; + const seen: string[] = []; + let value: unknown; + for await (const e of call.stream()) { + types.push(e.type); + if (e.type === 'drift') seen.push(fmt(e.finding)); + if (e.type === 'result') + value = (e.data as Record)['transaction_id']; + } + checkSeq('(c) event spine', types, [ + 'start', + 'progress', + 'drift', + 'drift', + 'result', + 'done', + ]); + checkSeq('(c) findings', seen.sort(), [ + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + 'warn|coerced|transaction_id|string -> number', + ]); + check('(c) the validated value, off the `result` event', value, 0); + check('(c) requests made', wire, 1); + note( + '(c) → findings AND the value, one request, same run. What `.stream()` does not carry is the RAW body, so you still cannot see that the string was `"abc"`', + '', + ); + } + + // ── (d) `.inspect()`: the only accessor with raw AND validated in one object ───────────── + // …and it is a FRESH probe. Two requests: the one you made, and the one the probe makes. + { + let wire = 0; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: async () => { + wire += 1; + return { status: 200, headers: {}, body: DRIFTED }; + }, + output: drift(Charge), + }); + await call.safe(); + const ins = await call.inspect(); + checkSeq('(d) `Inspection` keys', Object.keys(ins).sort(), [ + 'data', + 'error', + 'findings', + 'source', + 'status', + ]); + checkSeq('(d) findings', ins.findings.map(fmt).sort(), [ + 'info|undeclared|settlement_delay_ms|undeclared field (number)', + 'warn|coerced|transaction_id|string -> number', + ]); + check( + '(d) RAW value the vendor sent', + (ins.raw as Record)['transaction_id'], + 'abc', + ); + check( + '(d) VALIDATED value the caller gets', + (ins.data as Record)['transaction_id'], + 0, + ); + check('(d) requests made for both', wire, 2); + note( + '(d) → `"abc"` → `0`, both halves visible, which is the ONLY place the $0 charge is diagnosable from one object. It cost a second request and it is a different response than the one that hurt you', + '', + ); + } + + // ── (e) `.report()`: `.inspect()` plus run diagnostics, still a fresh probe ────────────── + { + const vendorSeries = [ + DRIFTED, + { + transaction_id: 100002, + amount: 4200, + currency: 'usd', + status: 'succeeded', + }, + ]; + let i = 0; + const call = stitch({ + url: 'https://pay.example/charges/1', + adapter: async () => ({ + status: 200, + headers: {}, + body: venderAt(vendorSeries, i++), + }), + output: drift(Charge), + }); + const first = await call.safe(); + check( + '(e) the call that drifted', + (first.data as Record)['transaction_id'], + 0, + ); + const rep = await call.report(); + checkSeq('(e) `RunReport` keys', Object.keys(rep).sort(), [ + 'attempts', + 'cache', + 'config', + 'data', + 'error', + 'findings', + 'source', + 'status', + 'timing', + ]); + checkSeq('(e) findings the report saw', rep.findings.map(fmt), []); + note( + '(e) → the drifting call is over; the probe hit a clean response and reported ZERO findings. `.report()` answers "how is this endpoint right now", never "what happened on the call I just made"', + '', + ); + } + + // ── (f) expected/actual: carried on hard findings, absent on soft ones ─────────────────── + { + const hard: string[] = []; + const soft: string[] = []; + const capture = (into: string[]): TraceSink => ({ + handle(e: StitchEvent) { + if (e.type === 'drift') into.push(e.finding.detail ?? ''); + }, + }); + await stitch({ + url: 'https://pay.example/charges/1', + adapter: serving(DRIFTED), + output: drift(Strict), + trace: capture(hard), + }).safe(); + await stitch({ + url: 'https://pay.example/charges/1', + adapter: serving(DRIFTED), + output: drift(Charge), + trace: capture(soft), + }).safe(); + + checkSeq('(f) HARD finding detail', hard, [ + 'Expected number, received string', + ]); + checkSeq('(f) SOFT finding details', soft.sort(), [ + 'string -> number', + 'undeclared field (number)', + ]); + note( + "(f) → the hard detail is Zod's message and carries expected+actual TYPES. The soft detail is `kindOf(old) -> kindOf(new)` (drift.ts:77-83). Neither carries a VALUE, on any accessor except `.inspect().raw`", + '', + ); + } + + // ── (g) the array case does carry a coordinate ─────────────────────────────────────────── + // `sample` (ADR 0017, drift.ts:191-197) gives the concrete index of the first occurrence, so a + // per-element finding stays recoverable from the raw body. + { + const found: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') + found.push( + `${fmt(e.finding)} sample=${e.finding.sample ?? ''}`, + ); + }, + }; + const call = stitch({ + url: 'https://pay.example/charges', + adapter: serving({ + charges: [ + { transaction_id: 1 }, + { transaction_id: 2 }, + { transaction_id: 'abc' }, + { transaction_id: null }, + ], + }), + output: drift( + z.object({ + charges: z.array( + z.object({ + transaction_id: z.coerce.number().catch(0), + }), + ), + }), + ), + trace: sink, + }); + await call.safe(); + checkSeq('(g) heterogeneous array findings', found.sort(), [ + 'warn|coerced|charges[].transaction_id|1 element: null -> number sample=charges[3].transaction_id', + 'warn|coerced|charges[].transaction_id|1 element: string -> number sample=charges[2].transaction_id', + ]); + note( + '(g) → two distinct detail variants, each with its own count and a CONCRETE index. This is the most actionable finding shape in the library', + '', + ); + } + + finish( + 'C7', + 'PARTLY ACTIONABLE, AND IT DEPENDS ENTIRELY ON THE ACCESSOR. The FIELD PATH is always carried, on every accessor that carries a finding at all, including the array case where `sample=charges[2].transaction_id` gives a concrete index. EXPECTED/ACTUAL is only half there: a HARD finding carries both types (`Expected number, received string`, Zod\'s message), a SOFT one carries `kindOf(old) -> kindOf(new)` and no values. The accessor table: `await`/`.safe()` carries NOTHING for a soft finding (`{ok,data,error}` and a $0 value) and a generic `contract violation (drift)` for a hard one — `StitchError` has no `findings` — while the trace sink for the SAME run named the field and both types; `hooks.onResponse` runs before validation; `.stream()` gets every finding plus the validated value in ONE request; `.inspect()` is the only accessor with RAW (`"abc"`) and VALIDATED (`0`) in one object, and `.report()` adds the run diagnostics — but both are FRESH PROBES: `.report()` on the drifting stitch reported ZERO findings because the probe hit a clean response', + ); +} + +/** Index into a fixed response series, clamping to the last element. */ +function venderAt(series: T[], i: number): T { + return series[Math.min(i, series.length - 1)] as T; +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/c8-canary-watch.ts b/docs/scenarios/proofs/intermittent-drift/c8-canary-watch.ts new file mode 100644 index 00000000..055110cf --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/c8-canary-watch.ts @@ -0,0 +1,395 @@ +// C8 — the most honest answer for a canary rollout: quiet on additions, loud on removals and type +// changes, and a rate you can alert on. Six workloads, one configuration, priced against a +// hand-rolled equivalent. +// +// The assembled answer is TWO declarative lines plus a sink: +// +// output: drift(StrictCharge, { severity: { undeclared: 'verbose' } }), +// trace: new DriftRate({ clock, window: '1m', zeroWatch: ['transaction_id', 'amount'] }), +// +// The schema is STRICT on purpose. Every softening — `.catch()`, `.optional()`, `.nullable()`, +// `z.union`, `z.coerce` — was measured in C2/C3/C4 to be a way of turning a loud change into a +// quiet one or into a fabricated value. On a money field, "the call failed" is the correct answer +// to "the vendor sent something I do not understand", and `drift()` is what makes ADDITIONS not +// pay for that strictness. +// +// pnpm exec tsx docs/scenarios/proofs/intermittent-drift/c8-canary-watch.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { chargeOutput } from './canary-watch'; +import { DriftRate } from './drift-rate'; +import { FakeVendor, type Mutation } from './fake-vendor'; +import { HandRate, type Shape, guardedCall } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { z } from './zod'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** Count the CODE lines between the `` / `` markers of a file. */ +function countedLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8').split('\n'); + const from = src.findIndex((l) => l.includes('')); + const to = src.findIndex((l) => l.includes('')); + return src + .slice(from + 1, to) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; +} + +/** The `DriftOptions` the soft-schema comparison in (c) reuses. */ +const CANARY_DRIFT = { severity: { undeclared: 'verbose' } } as const; + +/** The equivalent declaration for the hand-rolled baseline. */ +const SHAPE: Shape = { + transaction_id: { type: 'number', required: true }, + amount: { type: 'number', required: true }, + currency: { type: 'string', required: true }, + status: { type: 'string', required: true }, +}; + +/** What the watch is asked to conclude about one workload. */ +interface Verdict { + /** Calls that reached the caller with a usable value. */ + ok: number; + /** Calls that failed rather than hand over a value nobody can trust. */ + failed: number; + /** The alert lines, or `[]` for silence. */ + alerts: string[]; + /** Any call where the caller was handed a literal `0` on a money field. */ + zeros: number; +} + +const WORKLOADS: { label: string; mutation: Mutation; rate: number }[] = [ + { label: 'clean ', mutation: 'none', rate: 0 }, + { label: 'addition 100% ', mutation: 'added', rate: 1 }, + { label: 'removal 5% ', mutation: 'removed', rate: 0.05 }, + { label: 'retype 5% ', mutation: 'retyped', rate: 0.05 }, + { label: 'garbage retype 5%', mutation: 'garbage', rate: 0.05 }, + { label: 'null 5% ', mutation: 'nulled', rate: 0.05 }, +]; + +async function runStitch(m: Mutation, rate: number): Promise { + const clock = manualClock(); + const vendor = new FakeVendor({ mutation: m, rate }); + const watch = new DriftRate({ + clock, + window: 60_000, + zeroWatch: ['transaction_id', 'amount'], + }); + const charge = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: vendor.adapter(), + output: chargeOutput, + trace: watch, + clock, + }); + const v: Verdict = { ok: 0, failed: 0, alerts: [], zeros: 0 }; + for (let i = 0; i < 100; i += 1) { + const r = await charge.safe(); + if (r.ok) { + v.ok += 1; + const d = r.data as Record; + if (d['transaction_id'] === 0 || d['amount'] === 0) v.zeros += 1; + } else v.failed += 1; + await clock.advance(500); + } + // Only `warn` and above is an alert; `verbose`/`info` rows are the release log. + v.alerts = watch + .report() + .filter((line) => !/: (verbose|info)\|/.test(line)); + return v; +} + +async function runHand(m: Mutation, rate: number): Promise { + const clock = manualClock(); + const vendor = new FakeVendor({ mutation: m, rate }); + const adapter = vendor.adapter(); + const watch = new HandRate(() => clock.now(), 60_000); + const v: Verdict = { ok: 0, failed: 0, alerts: [], zeros: 0 }; + for (let i = 0; i < 100; i += 1) { + const r = await guardedCall( + adapter, + 'https://pay.example/charges', + SHAPE, + watch, + ); + if (r.ok) { + v.ok += 1; + const d = r.value ?? {}; + if (d['transaction_id'] === 0 || d['amount'] === 0) v.zeros += 1; + } else v.failed += 1; + await clock.advance(500); + } + v.alerts = watch + .report() + .filter((line) => !/: (verbose|info)\|/.test(line)); + return v; +} + +async function main(): Promise { + heading('C8 — the assembled canary watch'); + + // ── (a) the six workloads through the assembled configuration ─────────────────────────── + { + const rows: string[] = []; + for (const w of WORKLOADS) { + const v = await runStitch(w.mutation, w.rate); + rows.push( + `${w.label} ok=${v.ok} failed=${v.failed} zeros=${v.zeros} alerts=${v.alerts.length}`, + ); + } + checkSeq('(a) outcomes', rows, [ + 'clean ok=100 failed=0 zeros=0 alerts=0', + 'addition 100% ok=100 failed=0 zeros=0 alerts=0', + 'removal 5% ok=95 failed=5 zeros=0 alerts=1', + 'retype 5% ok=95 failed=5 zeros=0 alerts=1', + 'garbage retype 5% ok=95 failed=5 zeros=0 alerts=1', + 'null 5% ok=95 failed=5 zeros=0 alerts=1', + ]); + note( + '(a) → SILENT on 100% additions, LOUD on all four breaking classes at 5%, and ZERO $0 charges on any workload. That is the target the capture set', + '', + ); + } + + // ── (b) the alert lines themselves ─────────────────────────────────────────────────────── + { + const lines: string[] = []; + for (const w of WORKLOADS) { + const v = await runStitch(w.mutation, w.rate); + lines.push(`${w.label} → ${v.alerts[0] ?? ''}`); + } + checkSeq('(b) what an on-call engineer reads', lines, [ + 'clean → ', + 'addition 100% → ', + 'removal 5% → 5.0% of calls: error|invalid|currency|Required (5/100)', + 'retype 5% → 5.0% of calls: error|invalid|transaction_id|Expected number, received string (5/100)', + 'garbage retype 5% → 5.0% of calls: error|invalid|transaction_id|Expected number, received string (5/100)', + 'null 5% → 5.0% of calls: error|invalid|transaction_id|Expected number, received null (5/100)', + ]); + note( + '(b) → the rate, the field, and both types, on one line, with no per-call noise. The two retype workloads read IDENTICALLY, which is fine here BECAUSE neither produced a value', + '', + ); + } + + // ── (c) the same six workloads against the SOFT schema, to price the strictness ────────── + // This is the configuration a team writes when the strict one starts failing calls. It keeps + // 100% availability and pays for it in fabricated money. + { + const Soft = z.object({ + transaction_id: z.coerce.number().catch(0), + amount: z.coerce.number().catch(0), + currency: z.string().default('usd'), + status: z.string(), + }); + const rows: string[] = []; + for (const w of WORKLOADS) { + const clock = manualClock(); + const vendor = new FakeVendor({ + mutation: w.mutation, + rate: w.rate, + }); + const watch = new DriftRate({ + clock, + window: 60_000, + zeroWatch: ['transaction_id', 'amount'], + }); + const charge = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: vendor.adapter(), + output: drift(Soft, CANARY_DRIFT), + trace: watch, + clock, + }); + let ok = 0; + let zeros = 0; + for (let i = 0; i < 100; i += 1) { + const r = await charge.safe(); + await clock.advance(500); + if (!r.ok) continue; + ok += 1; + const d = r.data as Record; + if (d['transaction_id'] === 0 || d['amount'] === 0) zeros += 1; + } + rows.push(`${w.label} ok=${ok} zeros=${zeros}`); + } + checkSeq('(c) the soft schema', rows, [ + 'clean ok=100 zeros=0', + 'addition 100% ok=100 zeros=0', + 'removal 5% ok=100 zeros=0', + 'retype 5% ok=100 zeros=0', + 'garbage retype 5% ok=100 zeros=5', + 'null 5% ok=100 zeros=5', + ]); + note( + "(c) → EVERY call succeeds on every workload, and TEN $0 charges land across two of them. The `garbage` row is C3's `.catch(0)` route and the `null` row is C3's bare `z.coerce.number()` route", + '', + ); + } + + // ── (d) the hand-rolled baseline agrees, workload for workload ─────────────────────────── + { + const rows: string[] = []; + for (const w of WORKLOADS) { + const v = await runHand(w.mutation, w.rate); + rows.push( + `${w.label} ok=${v.ok} failed=${v.failed} zeros=${v.zeros} alerts=${v.alerts.length}`, + ); + } + checkSeq('(d) the same six, no library', rows, [ + 'clean ok=100 failed=0 zeros=0 alerts=0', + 'addition 100% ok=100 failed=0 zeros=0 alerts=0', + 'removal 5% ok=95 failed=5 zeros=0 alerts=1', + 'retype 5% ok=95 failed=5 zeros=0 alerts=1', + 'garbage retype 5% ok=95 failed=5 zeros=0 alerts=1', + 'null 5% ok=100 failed=0 zeros=0 alerts=1', + ]); + note( + '(d) → identical on additions, removals and BOTH retypes. The one row that differs is `null 5%`, and the hand-rolled version is BETTER there: it levels a null as `warn`, passes the null through, and alerts — which is the fourth industry class (`nullable is warning-level, value intact`) that C5 measured as inexpressible in `DriftOptions`', + '', + ); + } + + // ── (e) the price, in lines ────────────────────────────────────────────────────────────── + { + const declarative = countedLines('canary-watch.ts'); + const sink = countedLines('drift-rate.ts'); + const hand = countedLines('hand-rolled.ts'); + check('(e) the declarative configuration', declarative, 9); + check('(e) the DriftRate sink (user code)', sink, 84); + check('(e) hand-rolled, same feature set', hand, 92); + check('(e) library total (config + sink)', declarative + sink, 93); + note( + '(e) → 93 vs 92, a wash. The DETECTION is 9 declarative lines against ~35 hand-rolled; the AGGREGATION is ~84 lines of user code either way, and the sink pays an extra `spanId` join that the hand-rolled loop gets for free by having the raw body and the value in the same scope', + '', + ); + } + + // ── (f) what the 74 lines do not have ──────────────────────────────────────────────────── + // The same argument scenario 11 landed on: the detector is not what the library is buying. + // One `retry` line recovers a 503 mid-canary, with the drift rate still counting logical calls + // rather than wire attempts. + { + const clock = manualClock(); + let n = 0; + let wire = 0; + const watch = new DriftRate({ clock, window: 600_000 }); + const charge = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: async () => { + wire += 1; + if (wire % 4 === 1 && wire < 30) + return { status: 503, headers: {}, body: {} }; + n += 1; + return { + status: 200, + headers: {}, + body: { + transaction_id: n % 20 === 0 ? null : 100000 + n, + amount: 4200, + currency: 'usd', + status: 'succeeded', + }, + }; + }, + output: chargeOutput, + trace: watch, + clock, + retry: { attempts: 3, backoff: { base: 100 } }, + }); + let ok = 0; + for (let i = 0; i < 100; i += 1) { + const p = charge.safe(); + await clock.advance(0); + await clock.advance(100); + await clock.advance(200); + const r = await p; + if (r.ok) ok += 1; + await clock.advance(400); + } + check('(f) wire requests', wire, 108); + check('(f) logical calls counted', watch.calls, 100); + check('(f) successful calls', ok, 95); + checkSeq('(f) the alert', watch.report(), [ + '5.0% of calls: error|invalid|transaction_id|Expected number, received null (5/100)', + ]); + note( + '(f) → 108 wire requests, 100 logical calls, and the rate is over LOGICAL calls. Retry did not inflate the denominator, and the 8 recovered 503s never touched the drift number', + '', + ); + } + + // ── (g) the one thing this configuration still cannot do ───────────────────────────────── + // Everything above is the STRICT posture, where a type change fails the call. The moment any + // field is softened for availability, C3(e) is back: the finding for the benign coercion and + // the finding for the $0 charge are byte-identical, and the ONLY thing that separates them is + // the sink's join to the `result` event. + { + const clock = manualClock(); + const Soft = z.object({ + transaction_id: z.coerce.number().catch(0), + amount: z.number(), + currency: z.string(), + status: z.string(), + }); + const bodies = [ + { + transaction_id: '100001', + amount: 4200, + currency: 'usd', + status: 'ok', + }, + { + transaction_id: 'abc', + amount: 4200, + currency: 'usd', + status: 'ok', + }, + ]; + let i = 0; + const watch = new DriftRate({ + clock, + zeroWatch: ['transaction_id'], + }); + const charge = stitch({ + name: 'charge', + url: 'https://pay.example/charges', + adapter: async () => ({ + status: 200, + headers: {}, + body: bodies[i++ % 2], + }), + output: drift(Soft, CANARY_DRIFT), + trace: watch, + clock, + }); + for (let k = 0; k < 10; k += 1) await charge.safe(); + checkSeq('(g) the alert on a softened field', watch.report(), [ + '100.0% of calls: warn|coerced|transaction_id|string -> number (10/10, 5 landed 0)', + ]); + note( + '(g) → 10 calls, ONE finding identity, and only the `5 landed 0` half says half of them were $0 charges. That clause is the sink joining `drift` to `result` on `ctx.spanId` — it is not in any finding', + '', + ); + } + + finish( + 'C8', + 'ACHIEVABLE, AND THE STRICT POSTURE IS THE ONE THAT WORKS. Two declarative lines — `output: drift(StrictCharge, { severity: { undeclared: "verbose" } })` and `trace: new DriftRate(...)` — measured over six workloads at 100 calls each: SILENT on a 100% addition rollout, and one alert line per breaking class at 5%, each carrying the rate, the field and both types (`5.0% of calls: error|invalid|transaction_id|Expected number, received null (5/100)`). ZERO $0 charges on every workload. The same six against the SOFT schema people write for availability keep 100% of calls and produce TEN $0 charges. The price is a WASH: 9 declarative lines + an 84-line sink = 93, against 92 hand-rolled. The detection half is 9 lines against ~35; the aggregation half is ~84 lines of user code either way. What the 92 lines do not have is the resilience stack, measured here as one `retry` line absorbing 8 x 503 across the canary with the rate still counting 100 LOGICAL calls out of 108 wire requests. And the hand-rolled classifier BEATS `DriftOptions` on one row: it levels a null `warn` and passes the value through, the fourth industry class C5 found inexpressible. The residual gap is C3(e): the moment a field is softened, the benign coercion and the $0 charge share one finding identity, and only the sink\'s `spanId` join to the `result` event (`10/10, 5 landed 0`) separates them', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/intermittent-drift/canary-watch.ts b/docs/scenarios/proofs/intermittent-drift/canary-watch.ts new file mode 100644 index 00000000..1a9fc7a2 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/canary-watch.ts @@ -0,0 +1,28 @@ +// The assembled answer, in its own file so C8 can count it without counting the counter. +// +// Everything the scenario asks for that IS declarative lives between the markers below: a strict +// schema, one `severity` override that demotes additions, and the `drift()` wrapper that turns the +// strictness into per-class levels instead of a blanket rejection. The aggregation half is the +// `DriftRate` sink, which is user code and is counted separately. +// +// The schema is strict on every field ON PURPOSE. C2, C3 and C4 each measured a softening — +// `.optional()`, `.catch()`, `.nullable()`, `z.coerce`, `z.union` — turning a breaking change into +// silence or into a fabricated value. On a money field the correct answer to "the vendor sent +// something I do not understand" is to fail the call, and `drift()` is what stops ADDITIONS from +// paying for that. +import { drift } from '../../../../packages/core/src/index'; +import { z } from './zod'; + +/* */ +export const StrictCharge = z.object({ + transaction_id: z.number(), + amount: z.number(), + currency: z.string(), + status: z.string(), +}); + +/** Additions demoted below the default `info` — a vendor release should not page anyone. */ +export const chargeOutput = drift(StrictCharge, { + severity: { undeclared: 'verbose' }, +}); +/* */ diff --git a/docs/scenarios/proofs/intermittent-drift/drift-rate.ts b/docs/scenarios/proofs/intermittent-drift/drift-rate.ts new file mode 100644 index 00000000..ab0ac720 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/drift-rate.ts @@ -0,0 +1,163 @@ +// The aggregation seam: a `TraceSink` that turns per-call drift findings into a RATE. +// +// This is the answer to C6, and it is user code — the library counts nothing. What the library +// supplies is the seam: `trace` accepts any `{ handle(event, ctx) }`, it is configured ONCE on a +// stitch or a seam, it receives every event of every call through it, and `ctx.spanId` identifies +// the logical call. That is exactly the shape a counter needs, and it is the only place in the +// library where cross-call state is deliberate rather than a leak. +// +// Three things this sink has to get right, each of which is a measured trap in `c6-aggregation.ts`: +// +// 1. **The denominator is `start` events, not findings.** A call with two drifted fields emits two +// findings (measured: 3 findings on one call in C6), so `findings / calls` reports 300%. +// 2. **A call that drifted is not the same as a finding.** `spanId` collapses N findings back to +// one drifted call, which is what "5% of calls drifted" means. +// 3. **The VALUE is not in the finding.** `warn|coerced|transaction_id|string -> number` is +// byte-identical for `"12345" -> 12345` and for `"abc" -> 0` (C3). The `result` event carries +// the validated `data` on the same `spanId`, so joining the two is how a counter learns that +// the coercion produced a zero. Nothing else in the library exposes that. +// +// The window is clock-driven so a rate is a rate over a period rather than since-process-start — +// `manualClock()` drives it in the proofs, `Date.now` in production. +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; + +/** Just enough of a clock for the window — `manualClock()` satisfies it, so does `{ now: Date.now }`. */ +export interface NowSource { + now(): number; +} + +export interface DriftRateOptions { + /** Clock the rolling window is measured on. Default: wall clock. */ + clock?: NowSource; + /** Rolling window in ms. Omitted ⇒ count since construction (no eviction). */ + window?: number; + /** + * Paths whose validated value being `0` should be counted separately — the `$0 transaction` + * detector. A coercion is only dangerous when it lands on a value the business logic will act + * on, and the finding cannot tell you that; the joined `result` event can. + */ + zeroWatch?: string[]; +} + +/** One aggregated row: a distinct finding identity and how often it fired. */ +export interface DriftRow { + key: string; + level: string; + change: string; + path: string; + detail: string; + /** Findings emitted (a call with the same finding twice counts twice — arrays collapse first). */ + findings: number; + /** Distinct logical calls that emitted this finding. The numerator of a rate. */ + calls: number; + /** Of those calls, how many landed a `0` on a `zeroWatch` path. */ + zeros: number; +} + +/** What a recorded tick counts: one logical call, one finding, one drifted call, one landed zero. */ +type Tick = 'call' | 'finding' | 'drifted' | 'zero'; + +/* */ +export class DriftRate implements TraceSink { + private readonly clock: NowSource; + private readonly window: number | undefined; + private readonly zeroWatch: Set; + /** Every observed event reduced to `{ at, key, kind }`, so eviction is a filter on `at`. */ + private ticks: { at: number; key: string; kind: Tick }[] = []; + /** Findings seen on a span not yet terminated by its `result` / `error` event. */ + private readonly open = new Map(); + + constructor(opts: DriftRateOptions = {}) { + this.clock = opts.clock ?? Date; + this.window = opts.window; + this.zeroWatch = new Set(opts.zeroWatch ?? []); + } + + handle(e: StitchEvent, ctx: TraceContext): void { + const at = this.clock.now(); + const span = ctx.spanId ?? ''; + if (e.type === 'start') + this.ticks.push({ at, key: ctx.name, kind: 'call' }); + if (e.type === 'drift') { + const f = e.finding; + const key = `${f.level}|${f.change}|${f.path}|${f.detail ?? ''}`; + this.ticks.push({ at, key, kind: 'finding' }); + const slot = this.open.get(span) ?? []; + slot.push(key); + this.open.set(span, slot); + } + // Join: the validated value arrives on the SAME span, after the findings. This is the only + // way to learn that `coerced` produced a ZERO rather than the right number. + if (e.type !== 'result' && e.type !== 'error') return; + const slot = this.open.get(span); + this.open.delete(span); + if (!slot) return; + for (const key of new Set(slot)) + this.ticks.push({ at, key, kind: 'drifted' }); + if (e.type !== 'result') return; + const data = e.data; + if (data === null || typeof data !== 'object') return; + const record = data as Record; + for (const key of new Set(slot)) { + const path = key.split('|')[2] ?? ''; + if (this.zeroWatch.has(path) && record[path] === 0) + this.ticks.push({ at, key, kind: 'zero' }); + } + } + + /** Ticks still inside the window (evicting is how a rolling rate forgets). */ + private live(): { at: number; key: string; kind: Tick }[] { + if (this.window === undefined) return this.ticks; + const floor = this.clock.now() - this.window; + this.ticks = this.ticks.filter((t) => t.at > floor); + return this.ticks; + } + + /** Logical calls in the window — the denominator. */ + get calls(): number { + return this.live().filter((t) => t.kind === 'call').length; + } + + /** One row per distinct finding identity, most drifted calls first. */ + rows(): DriftRow[] { + const by = new Map(); + for (const t of this.live()) { + if (t.kind === 'call') continue; + const [level = '', change = '', path = '', detail = ''] = + t.key.split('|'); + let row = by.get(t.key); + if (!row) { + row = { + key: t.key, + level, + change, + path, + detail, + findings: 0, + calls: 0, + zeros: 0, + }; + by.set(t.key, row); + } + if (t.kind === 'finding') row.findings += 1; + if (t.kind === 'drifted') row.calls += 1; + if (t.kind === 'zero') row.zeros += 1; + } + return [...by.values()].sort((a, b) => b.calls - a.calls); + } + + /** `"5.0% of calls: warn|coerced|transaction_id|null -> number (5/100, 5 landed 0)"` */ + report(): string[] { + const total = this.calls; + return this.rows().map((r) => { + const pct = total === 0 ? 0 : (r.calls / total) * 100; + const zeros = r.zeros > 0 ? `, ${r.zeros} landed 0` : ''; + return `${pct.toFixed(1)}% of calls: ${r.key} (${r.calls}/${total}${zeros})`; + }); + } +} +/* */ diff --git a/docs/scenarios/proofs/intermittent-drift/fake-vendor.ts b/docs/scenarios/proofs/intermittent-drift/fake-vendor.ts new file mode 100644 index 00000000..4dbcbfef --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/fake-vendor.ts @@ -0,0 +1,155 @@ +// The payment vendor rolling a response-shape change out to a percentage of its traffic. +// +// One `Adapter`, no network, no timers. The vendor serves a BASELINE body and can be told to serve +// a MUTATED one on a deterministic slice of calls — `rate: 0.05` means every 20th call, not a coin +// flip, so a 100-call run is byte-identical on every machine and "5 of 100" is a fact rather than +// an expectation. +// +// The five mutations are the industry change taxonomy (LinkedIn / Xandr breaking-change policies), +// one per class, plus the two halves of the type change that the scenario turns on: +// +// 'added' — a field appears that the consumer does not model (non-breaking) +// 'removed' — a field the consumer depends on disappears (breaking) +// 'retyped' — `transaction_id` 12345 -> "12345", a PLAUSIBLE string (breaking, benign value) +// 'garbage' — `transaction_id` -> "abc", a NON-NUMERIC string (breaking, dangerous value) +// 'nulled' — `transaction_id` -> null on the data that triggers it (nullable-without-notice) +// +// `retyped` and `garbage` differ only in the CONTENT of the string. They are the same wire-type +// shift, so anything that classifies by type sees one class; the caller sees `12345` in one and a +// $0 charge in the other. Keeping them as separate named mutations is the whole point of the file. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +/** The payment body a caller has been consuming since the integration was written. */ +export interface Charge { + transaction_id: number; + amount: number; + currency: string; + status: string; +} + +/** What the vendor's canary is doing to the shape on the calls it touches. */ +export type Mutation = + 'none' | 'added' | 'removed' | 'retyped' | 'garbage' | 'nulled'; + +export interface FakeVendorOptions { + /** What the canary does on the calls it touches. Default `'none'`. */ + mutation?: Mutation; + /** + * Fraction of calls the canary touches, 0..1. Deterministic, not random: with `rate` = 1/N + * every Nth call is mutated. `1` mutates every call (a completed rollout), `0` none. + */ + rate?: number; + /** Which field the `removed` / `added` mutations act on. Default `'currency'` / `'settlement_delay_ms'`. */ + field?: string; +} + +/** One recorded request and the shape the vendor chose to serve it. */ +export interface VendorCall { + n: number; + mutated: boolean; + body: Record; +} + +/** + * The vendor. `adapter()` is what a stitch is handed; `calls` is the ledger every rate claim reads + * its GROUND TRUTH off — `vendor.mutatedCount` is how many responses actually carried the new + * shape, which is the number a measured drift rate has to be checked against. + */ +export class FakeVendor { + readonly calls: VendorCall[] = []; + private n = 0; + private readonly mutation: Mutation; + private readonly period: number; + private readonly field: string | undefined; + + constructor(opts: FakeVendorOptions = {}) { + this.mutation = opts.mutation ?? 'none'; + const rate = opts.rate ?? 1; + // `period` is "one in every P calls". rate 0.05 -> 20, rate 1 -> 1, rate 0 -> never. + this.period = + rate <= 0 ? Number.POSITIVE_INFINITY : Math.round(1 / rate); + this.field = opts.field; + } + + /** How many served responses actually carried the mutated shape. The ground truth for a rate. */ + get mutatedCount(): number { + return this.calls.filter((c) => c.mutated).length; + } + + /** 1-based indices of the calls the canary touched — `[20,40,60,80,100]` for 5% over 100. */ + get mutatedAt(): number[] { + return this.calls.filter((c) => c.mutated).map((c) => c.n); + } + + private baseline(n: number): Record { + return { + transaction_id: 100000 + n, + amount: 4200, + currency: 'usd', + status: 'succeeded', + }; + } + + private mutate(body: Record): Record { + const out = { ...body }; + switch (this.mutation) { + case 'added': + // A vendor release adds a field. Non-breaking by every published policy. + out[this.field ?? 'settlement_delay_ms'] = 900; + return out; + case 'removed': + delete out[this.field ?? 'currency']; + return out; + case 'retyped': + // int -> string, and the string still spells the same integer. + out['transaction_id'] = String(out['transaction_id']); + return out; + case 'garbage': + // int -> string, and the string is not a number at all. Same wire-type shift. + out['transaction_id'] = 'abc'; + return out; + case 'nulled': + out['transaction_id'] = null; + return out; + case 'none': + return out; + } + } + + adapter(): Adapter { + return async (_req: AdapterRequest): Promise => { + this.n += 1; + const n = this.n; + const mutated = n % this.period === 0; + const body = mutated + ? this.mutate(this.baseline(n)) + : this.baseline(n); + this.calls.push({ n, mutated, body }); + return { status: 200, headers: {}, body }; + }; + } +} + +/** + * A vendor that serves ONE fixed body — the single-response probes C1/C2/C3/C7 use, where the + * question is "what does this exact shape produce" and a rate would be noise. + */ +export const serving = + (body: unknown): Adapter => + async (_req: AdapterRequest): Promise => ({ + status: 200, + headers: {}, + body, + }); + +/** A drift finding rendered as one comparable string — the spine every claim asserts on. */ +export const fmt = (f: { + level: string; + change: string; + path: string; + detail?: string; +}): string => `${f.level}|${f.change}|${f.path}|${f.detail ?? ''}`; diff --git a/docs/scenarios/proofs/intermittent-drift/hand-rolled.ts b/docs/scenarios/proofs/intermittent-drift/hand-rolled.ts new file mode 100644 index 00000000..1dd34ee5 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/hand-rolled.ts @@ -0,0 +1,132 @@ +// The same canary detector with no library at all — the baseline C8 prices against. +// +// Feature parity with `c8-assembled.ts`: call the endpoint, validate against a declared shape, +// classify each difference into the industry change taxonomy, keep the correct value where one +// exists, refuse to invent one where it does not, and maintain a windowed rate per field so a 5% +// canary is visible as a rate rather than as a trickle of odd values. +// +// The interesting part is what falls out of writing it by hand: the classification is trivial +// (about a dozen lines), the RATE is trivial (about a dozen more), and the thing that is neither +// is the "keep the value" decision — which is exactly the decision `drift()` delegates to your +// schema and then reports on. Writing it by hand makes it explicit that a $0 charge is a choice +// somebody makes on a specific line, and that line is `Number(raw)` no matter whose code it is in. +import type { Adapter } from '../../../../packages/core/src/types'; + +/** The industry change taxonomy, as the classes rather than as validator mechanisms. */ +export type ChangeClass = 'added' | 'removed' | 'retyped' | 'nulled'; + +export interface HandFinding { + level: 'error' | 'warn' | 'info'; + change: ChangeClass; + path: string; + detail: string; +} + +export interface HandOutcome { + ok: boolean; + value: Record | null; + findings: HandFinding[]; +} + +/* */ +/** The declared shape: each field's expected `typeof`, and whether losing it is fatal. */ +export type Shape = Record; + +const kindOf = (v: unknown): string => + v === null ? 'null' : Array.isArray(v) ? 'array' : typeof v; + +/** Classify one response against the declared shape. Additions info, removals/retypes error. */ +export function classify( + body: Record, + shape: Shape, +): HandFinding[] { + const out: HandFinding[] = []; + for (const [path, spec] of Object.entries(shape)) { + if (!(path in body)) { + if (spec.required) + out.push({ + level: 'error', + change: 'removed', + path, + detail: `required ${spec.type} absent`, + }); + continue; + } + const got = kindOf(body[path]); + if (got === spec.type) continue; + out.push( + got === 'null' + ? { + level: 'warn', + change: 'nulled', + path, + detail: `${spec.type} -> null`, + } + : { + level: 'error', + change: 'retyped', + path, + detail: `${spec.type} -> ${got} (${JSON.stringify(body[path])})`, + }, + ); + } + for (const path of Object.keys(body)) + if (!(path in shape)) + out.push({ + level: 'info', + change: 'added', + path, + detail: `undeclared ${kindOf(body[path])}`, + }); + return out; +} + +/** A windowed per-finding rate. `now` is injected so the window is testable. */ +export class HandRate { + private ticks: { at: number; key: string; drifted: boolean }[] = []; + constructor( + private readonly now: () => number, + private readonly window: number, + ) {} + record(findings: HandFinding[]): void { + const at = this.now(); + this.ticks.push({ at, key: '', drifted: false }); + for (const key of new Set( + findings.map((f) => `${f.level}|${f.change}|${f.path}`), + )) + this.ticks.push({ at, key, drifted: true }); + } + report(): string[] { + const floor = this.now() - this.window; + this.ticks = this.ticks.filter((t) => t.at > floor); + const calls = this.ticks.filter((t) => !t.drifted).length; + const by = new Map(); + for (const t of this.ticks) + if (t.drifted) by.set(t.key, (by.get(t.key) ?? 0) + 1); + return [...by] + .sort((a, b) => b[1] - a[1]) + .map( + ([k, n]) => + `${((n / calls) * 100).toFixed(1)}% of calls: ${k} (${n}/${calls})`, + ); + } +} + +/** One guarded call: fetch, classify, and keep the value only where it is unambiguous. */ +export async function guardedCall( + adapter: Adapter, + url: string, + shape: Shape, + rate: HandRate, +): Promise { + const res = await adapter({ url, method: 'GET', headers: {} }); + const body = (res.body ?? {}) as Record; + const findings = classify(body, shape); + rate.record(findings); + if (findings.some((f) => f.level === 'error')) + return { ok: false, value: null, findings }; + const value: Record = {}; + for (const path of Object.keys(shape)) value[path] = body[path]; + return { ok: true, value, findings }; +} +/* */ diff --git a/docs/scenarios/proofs/intermittent-drift/harness.ts b/docs/scenarios/proofs/intermittent-drift/harness.ts new file mode 100644 index 00000000..3a2ed196 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/harness.ts @@ -0,0 +1,81 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is THE VALUE THE CALLER RECEIVED. A drift finding that says +// `warn|coerced|transaction_id|string -> number` is compatible with the caller getting `12345` and +// with the caller getting `0`, and the whole scenario turns on which one it was. So `check` prints +// the measured value with `JSON.stringify`, not `String` — `0`, `"0"`, `null` and `"12345"` have to +// be distinguishable on the page, and `String(null)` and `String('null')` are not. +// +// `checkSeq` carries the drift findings themselves (a finding is only a claim if its level, kind, +// path and detail are all on the page) and the per-call value spines the rate claims are computed +// from. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `0` vs `"0"` vs `null` is the whole scenario. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v}n`; + if (Number.isNaN(v)) return 'NaN'; + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the finding spine + * (`["warn|coerced|transaction_id|string -> number"]`) and the per-call value spine + * (`[12345,0,0]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Some claims here PASS by measuring DAMAGE (C3's `0`), and some PASS by measuring that the library + * does the right thing (C1's silence, C6's 5.0%). The verdict statement carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/intermittent-drift/zod.ts b/docs/scenarios/proofs/intermittent-drift/zod.ts new file mode 100644 index 00000000..12732e59 --- /dev/null +++ b/docs/scenarios/proofs/intermittent-drift/zod.ts @@ -0,0 +1,17 @@ +// Real Zod, imported by path, and the one piece of ugliness in this directory. +// +// Every OTHER scenario in this section hand-rolled its `output` validators as plain +// `{ validate }` objects, because `stitchapi` has zero runtime dependencies and nothing under +// `docs/` has a `package.json`. This scenario cannot do that honestly: the entire question C3 asks +// — "what does the caller ACTUALLY RECEIVE when `transaction_id` stops being a number" — is +// answered by the **schema library's coercion rules**, not by StitchAPI. `drift()` only reports the +// difference between the raw body and whatever the validator returned. Hand-rolling the coercion +// would mean inventing the very behaviour under test. +// +// So these scripts use the real Zod v4 that `packages/core` already depends on +// (`packages/core/package.json` devDependencies: `"zod": "^4.4.3"`), reached by relative path +// because pnpm does not hoist it to the workspace root and `docs/` has no manifest of its own. +// It resolves under `tsx` and typechecks under `packages/core`'s strict set; it is a proof-script +// convenience, not a pattern to copy into application code, where `import { z } from 'zod'` is the +// spelling. +export { z } from '../../../../packages/core/node_modules/zod'; diff --git a/docs/scenarios/proofs/large-response-memory/README.md b/docs/scenarios/proofs/large-response-memory/README.md new file mode 100644 index 00000000..5a138791 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/README.md @@ -0,0 +1,270 @@ +# Proofs — the 84 MB response that took 2.1 GB of heap + +Runnable evidence for the claims in [`../../large-response-memory.md`](../../large-response-memory.md). + +**Both deciding claims came back split, and the capture was wrong in both directions.** C3 hoped +`decode: 'json'` would make the hard case — one giant top-level array — a config value. It half does: +emission is genuinely correct (one delta per element, holding up under `,`/`]`/`}` inside string +values, escaped quotes, embedded newlines, pretty-printed multi-line records, deep nesting, and +1-character chunk boundaries) and the memory is **not bounded at all** — the decoder retains the whole +array text, so on default settings a 60,000-row export delivers 37,312 rows and then fails with an +error blaming the vendor. C4 feared `output` would re-buffer the stream to validate the aggregate. It +does not: the contract runs **per delta**, was called 500 times for 500 records, never once saw an +array, and costs nothing measurable. + +**The finding neither claim was looking for is one line of the engine.** `runStreaming` pushes every +emitted delta onto a `chunks` array so the terminal `result` can mirror the whole spine +([`engine.ts:1443`](../../../../packages/core/src/engine.ts)). It is unconditional, it is not gated on +the accessor, and no config turns it off. So the `'ndjson'` decoder — which really is O(1), 0.6 MB of +retained heap for **1,000,000 rows and 214 MB of wire** — becomes O(N) the moment the engine wraps it, +and `.stream()` costs the same as `await` (30.2 MB against 33.5 MB at 100k rows). The library owns a +genuinely bounded decoder and spends the win one line later. + +Every script is standalone and offline. Each prints one `PASS`/`FAIL` line and exits non-zero on +failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/large-response-memory/c3-json-single-array.ts + +# all of them (slow — each spawns a dozen child processes; ~6 minutes) +for f in docs/scenarios/proofs/large-response-memory/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +### `--expose-gc`, and where it is required + +The claim scripts (`c1`…`c8`) do **not** need it — they measure nothing themselves. They spawn +[`probe.ts`](./probe.ts), which does, and [`run-probe.ts`](./run-probe.ts) passes the flag for them. + +To take a single measurement by hand you must pass it yourself: + +```sh +pnpm exec tsx --expose-gc docs/scenarios/proofs/large-response-memory/probe.ts \ + --mode=buffered --rows=100000 +# {"mode":"buffered","rows":100000,"ok":true,"wireBytes":22489287,"peakLive":56391584, …} +``` + +`probe.ts` calls `requireGc()` ([`mem.ts`](./mem.ts)) before anything else and **exits 2 with a +message** if `global.gc` is absent, rather than quietly reporting the noisy number as if it were the +clean one. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/large-response-memory/*.ts +``` + +## Methodology — what is solid and what is noisy + +This is the first scenario in the set whose evidence is a **number** rather than a value, and heap +numbers lie in several different ways. The measurement is built to make each one visible. + +**One process per measurement.** V8's heap is stateful: the second workload in a process inherits the +first one's fragmented old space and its grown heap limit. Every number in this directory comes from +its own `probe.ts` process. + +**Two numbers, and only one of them decides anything.** + +- `peakLive` — the high-water mark of `heapUsed` sampled **immediately after a forced full GC**. This + is retained, reachable memory: what cannot be collected under pressure. Nearly noise-free. + **Every verdict here is based on `peakLive`.** +- `peakHeap` — the high-water mark of plain `heapUsed`, floating garbage included. Real (an + allocation rate the collector cannot keep up with is exactly how a process dies) but noisy, and + reported only as context. It is 20x `peakLive` for the `'json'` decoder on a big array, which is + itself a finding — see C3. + +**The sampler lives on the producer.** A `setInterval` sampler cannot preempt a synchronous +`JSON.parse`, and an in-memory `ReadableStream` resolves reads on the _microtask_ queue, which starves +timers completely. So the workload samples itself, once per emitted wire chunk, from +[`Wire.watch`](./fake-export.ts). Putting it there rather than in each consumer is what makes the +modes comparable: a consumer-side sampler gives `.stream()` a thousand chances to catch a peak and +`await` exactly none, and the resulting "await uses less heap" is an artefact of the instrument. (An +earlier draft of this directory made precisely that mistake and measured `await` 21 MB _cheaper_ than +`.stream()`.) The two buffered modes take two extra marks — whole text live, then text and tree both +live — because that path's peak happens after the last byte arrives; that asymmetry favours the +streaming modes, not the buffered one. + +**Ratios and shapes, not absolutes.** Every claim asserts on `peakLive(100k) ÷ peakLive(10k)` or on +`peakLive ÷ wireBytes`, never on a single figure. `checkFlat` / `checkLinear` +([`harness.ts`](./harness.ts)) print the growth either way. + +**What is noisy, stated plainly:** + +- **The 1,000-row point is floor-dominated and no claim asserts on it alone.** A run's one-time cost — + module init, JIT, IC feedback vectors — is ~1.3 MB through the engine, six times the data at that + size. It is reported (`settled` on every measurement is exactly this floor) and the growth + assertions use the 10k→100k pair, where the signal is 20x the floor. +- **`heapUsed` does not see the socket.** A `Uint8Array`'s backing store is external memory, so a + response sitting unread in a `ReadableStream`'s queue is invisible to it. C6 turns on this + distinction, so `peakBuffers` (`memoryUsage().arrayBuffers`) is tracked separately. +- **The 2.5x buffered multiplier is a property of the row shape**, not of Node and not of StitchAPI. + Eight flat fields cost 2.5x; the incident's ~25x implies far more per-object overhead. The number + that transfers between machines is the **slope** (linear), not the constant. +- Measured on **Node v24.18.1, arm64 macOS**. Absolute megabytes will differ elsewhere; the + flat-vs-linear shapes will not. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `c1-buffered-baseline.ts` | `await` a big array — multiplier? shape? | **2.5x, LINEAR, and identical to bare `JSON.parse` within 0.2MB.** 21.4MB wire → 53.8MB | +| `c2-ndjson-flat.ts` | does `decode: 'ndjson'` stay flat? | **The DECODER does (0.6MB at 1M rows). The ENGINE does not** — 3.5MB → 30.2MB over 10x | +| `c3-json-single-array.ts` | **DECIDING.** one giant array — streams or buffers? | **Right elements, whole array in memory, and it trips its own 8M-char cap at ~37,300 rows** | +| `c4-output-per-delta.ts` | **DECIDING.** `output` — per delta or aggregate? | **Per delta. 500 calls / 500 rows, `sawArrayOfLength` 0, and free in heap terms** | +| `c5-pick-transform.ts` | `pick` / `transform` on a stream | **Neither. Not called once, no `info`, no throw — and they work on the buffered path** | +| `c6-backpressure.ts` | slow consumer, and what the cap does AT the cap | **Backpressure propagates (1.3% of the body queued). The cap THROWS. Neither bounds the call** | +| `c7-buffered-guard.ts` | any guard on the buffered path? | **None. Four events either way, `buffer.chars` inert — and 400k rows under 96MB is SIGABRT** | +| `c8-assembled.ts` | the assembled answer, priced | **1.3MB flat vs a 53.8MB baseline. One seam, 75 lines, and `stream({ kind })` silently drops it** | + +## Files + +- `fake-export.ts` — the vendor. Everything is **lazy**: rows are generated inside the stream's + `pull()`, one ~16 KB chunk at a time, so the fixture never exists as a whole and the numbers are + about the library rather than about this file. Six wire shapes: `singleArray` (the hard case), + `concatObjects` (the control that localises C3's defect), `ndjson`, `eagerArray` (a producer that + ignores backpressure), `neverClosingArray` and `noNewlines` (the two shapes the cap exists for). + Plus `bufferingAdapter`, which mirrors `fetchAdapter`'s non-streaming path byte for byte + (`http-adapter.ts:133-138`) and offers a hook at the instant text and tree are both live. +- `mem.ts` — the measurement kit: `requireGc()`, and `measure()` returning `peakLive` / `peakHeap` / + `peakBuffers` / `settled` / `ratio`. The header states the methodology in code. +- `probe.ts` — one measurement, one process, one line of JSON. Thirteen modes. A workload that BLEW + UP prints `{ok: false, error}` on stdout rather than a stack trace on stderr, because C3's whole + finding is one of those. +- `run-probe.ts` — spawns it. `probeRun` keeps the corpse (exit status, stderr, an `heapOom` flag) so + C7 can assert on a process that died. +- `validator-spy.ts` — an instrumented `Validator`. C4 is a mechanism question, and the honest way to + answer it is a **counter**: 100,000 calls each carrying one row and one call carrying a + 100,000-element array are different facts, not two readings of one number. +- `batched-export.ts` — **the C8 deliverable, and it is user code.** A `Surface` whose `stream` hook + consumes rows and yields one small receipt per batch, so the engine's `chunks` array holds batches + instead of rows. 75 counted lines. +- `hand-rolled.ts` — the same feature set with no library at all. 67 counted lines. The baseline C8 + prices against. +- `harness.ts` — `check` / `checkSeq` / `checkFlat` / `checkLinear` / `checkAtMost` / `checkAtLeast`. + Exact equality for the correctness half, shape assertions for the heap half. + +## Reading the numbers honestly + +- **C1: the buffered path adds nothing and bounds nothing.** 100k rows / 21.4 MB of wire peaked at + 53.8 MB retained; bare `JSON.parse` of the same bytes cost 53.6 MB. `await stitch()` **is** + `JSON.parse`, and `JSON.parse` is linear with no ceiling. The 2.5x here is not the incident's 25x, + and saying so is more useful than quietly reproducing the bigger number: what generalises is that + the slope is 1, and the constant is your row shape's. +- **C2 is the sharpest result in the scenario and it has two halves.** The `'ndjson'` decoder, called + exactly as the engine calls it, held **0.6 MB for 1,000,000 rows and 214 MB of wire** and moved less + than 15% across a 1000x change in workload. That is a genuinely O(1) decoder, and it also proves the + instrument works. Through the engine the same decoder went 3.5 MB → 30.2 MB over 10x the rows. +- **The engine's accumulator is the finding, and it is documented — in a code comment.** + `engine.ts:1437-1442` says outright that `chunks` "grows for the life of the connection" and that + "awaiting such a stitch to completion is intentionally not memory-bounded". None of that reaches the + public docs, the types, or a runtime event. The same file's header comment (`engine.ts:1244`) says + `.stream()` "buffers nothing", which is true about latency and false about memory. +- **`.stream()` is not the memory fix everyone assumes.** 30.2 MB iterating, 33.5 MB awaiting. Both + accessors drain the same generator and the accumulator is inside it. The idiom that reads like the + answer — `for await (const ev of export.stream())` — changes when you see a row, not how many rows + are alive. +- **C3: `decode: 'json'` streams the PARSE and buffers the TEXT.** Emission is genuinely correct under + every adversarial input tried. Memory tracks the whole array at 0.88x the wire with 34x growth over + a 100x workload, because `compact()` floors on `valueStart`, and for a top-level array `valueStart` + is the opening `[` (`json-stream.ts:163`) and stays there until the closing `]` (`json-stream.ts:183`). + `compact(live)` is therefore a no-op for the entire array. +- **It is quadratic in time too.** 77 ms at 10k rows, 2187 ms at 100k — 10x the rows, 28x the time, + because `buf.charCodeAt` re-flattens the growing buffer on every chunk. `peakHeap` of 405 MB against + `peakLive` of 18.8 MB is that garbage. +- **So the decoder trips its own default guard, and the error blames the vendor.** 37,000 rows decode; + 38,000 fail with `json decoder: in-progress value exceeded the stream.buffer.chars cap (8388608); a +malformed or never-closing value was streamed`. Nothing was malformed and nothing failed to close. +- **The failure is a truncation, not a total loss — the hypothesis was wrong in the library's favour.** + `pending` is drained and yielded before `guard()` runs (`json-stream.ts:224-238`), so a 60,000-row + array delivers **37,312 rows** and then `error` / `done(ok:false)`. A `.stream()` loop that only + matches `delta` sees a silent truncation, at a threshold that moves as the catalog grows. +- **The control localises it to one branch.** The same 100,000 records as CONCATENATED top-level values + (`{…}{…}{…}` — a shape the same decoder advertises) run **flat at 0.7 MB**. Same decoder, same bytes, + same records: siblings are O(1), wrapped in one array they are O(N). +- **C4 refutes its own hypothesis, cleanly.** The instrumented contract was called 500 times for 500 + records, every call carrying one object, `sawArrayOfLength` stayed 0 — including when the wire was + literally one top-level array. Cost: 30.2 MB against 30.2 MB, and 48.7 MB against 48.7 MB. `output` + is innocent. +- **But it does two things a config author will not expect.** A failing row is a **circuit breaker, + not a filter**: `contract violation (drift)`, the stream ends, rows already delivered stay and the + rest are never read. And on a stream `output` **validates without transforming** — `engine.ts:1419` + destructures `{ errors }` and discards the validated value, where `engine.ts:1223` on the buffered + path serves it. The same coercing schema reshapes your data on `await` and silently does not on + `.stream()`. +- **C5: the capture asked the wrong question.** `pick` and `transform` are not per-delta and not + buffering — they do not run. `transform` was called zero times over 200 deltas; `pick: 'id'` left + the whole row in place. The same two slots on the same fake vendor over the buffered path behave + exactly as documented, so this is a path asymmetry, not a broken hook — and it is silent, with the + static delta type derived from `output` rather than `pick`, so the call site does not catch it. +- **C6: three different things wear the word "buffer" and only one of them is capped.** The socket + queue is bounded by the reader pulling (1.3% of the body with a lazy producer; 100% with one that + ignores `desiredSize` — and that backlog is invisible to `heapUsed`). The decoder's working buffer + is what `stream.buffer.chars` caps, per un-terminated **unit**. The engine's `chunks` array is + capped by nothing. +- **At the cap all three decoders THROW.** `json`, `ndjson` and `lines` each produced an `error` event + plus `done(ok:false)` with zero deltas from the offending unit. Not a truncation, not a pause. +- **And the cap is a malformed-input guard, not a memory budget.** 20,000 well-formed rows streamed + cleanly through a **1,000-character** cap while the engine accumulated all 20,000 of them. +- **C7: there is no guard, and the only signal is the process dying.** A 21 MB buffered response emits + the same four events a 40-byte one does. `stream: { buffer: { chars: 1_000 } }` on a plain `stitch()` + type-checks, composes, and delivers all 50,000 rows of an 11-million-character body. At 400,000 rows + under a 96 MB heap: `FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of +memory`, exit 134 — no catchable error, no `error` event, no `finally`. The same 400,000 rows and + 85.8 MB of wire, same ceiling, batched over `ndjson`: **1.3 MB and every row processed.** +- **C8: achievable, one seam, and a wire format you may not be offered.** `Surface.stream` is the + seam — not to replace the decoding (`ndjson` is already O(1)) but because the engine keeps whatever + that hook yields, so the fix is to yield receipts. 1.3 MB flat against a 53.8 MB baseline, 75 lines + of surface plus 4 of config against 67 lines hand-rolled. The lines are a wash; what the config buys + is the stack around the open (`auth`, `retry` on the connect, `throttle` charged per open, + `timeout.total`, `trace`, `verdict.accept`), all of which keep working through a custom surface. +- **The seam only fixes what is downstream of it.** Over one giant array the same surface still costs + 19.0 MB, because C3's array buffer sits upstream — and on default settings that call would not have + finished at all. + +## The footguns + +- **`stream({ kind: mySurface })` silently drops your surface.** `stream()` spreads your config and + then writes `kind: streamSurface` over it (`stream.ts:143-146`; `sse()` does the same at + `sse.ts:210-212`). Measured: 1,000 raw rows delivered, zero rows through the custom surface, no + error, no warning. The assembled answer only works spelled `stitch({ kind })` — and the spelling + that undoes it is the one the surface helper invites. +- **`decode: 'json'` over one top-level array looks exactly like the fix and is not one.** Elements + arrive one at a time, so it reads as streaming in the debugger, while the decoder holds every byte + of the array. It is the config in this scenario most likely to be shipped believing the problem is + solved. +- **And it fails at a threshold nobody set.** ~8.4 million characters of array — around 37,000 rows of + this shape. Below it, correct. Above it, a truncated export and an error message that sends you to + read the vendor's docs. +- **Raising `stream.buffer.chars` "fixes" it into the profile you were avoiding.** The call then + succeeds at 18.8 MB retained and 405 MB of peak allocation for a 21 MB body. It is + `--max-old-space-size` under another name: the cliff moves, it does not go away. +- **`stream.buffer.chars` is not a memory budget.** It bounds ONE un-terminated line / value / SSE + frame. It says nothing about how much the call may use, and it is completely inert on the buffered + path even though the type accepts it there. +- **`.stream()` does not bound memory.** Same generator, same `chunks` array, 30.2 MB either way. + The one accessor everybody reaches for when they hear "streaming" is not the fix. +- **A per-delta `output` failure ends the export.** It is a circuit breaker, not a `.filter()`. If one + bad row in a 100,000-row catalog should not abort the sync, the contract has to live in your + consumer, not in `output`. +- **`output` on a stream validates without transforming.** Coercions, defaults and key-stripping that + work on `await` are dropped on `.stream()` (`engine.ts:1419` vs `engine.ts:1223`). A schema that is + load-bearing for shape is silently decorative on the streaming path. +- **`pick` and `transform` are dead on a streaming stitch** — no call, no `info` event, no throw, and + no type error, because the delta type is derived from `output`. A stitch that carries them and is + later switched to `kind: stream` keeps compiling and quietly stops reshaping. +- **`heapUsed` will not show you a backlogged socket.** A producer that ignores backpressure parks the + whole body in the stream's internal queue as `Uint8Array`s, which live in external memory. Watch + `memoryUsage().arrayBuffers`, or conclude a 21 MB backlog is free. +- **The one honest warning about all of this is a code comment.** `engine.ts:1437-1442` states that an + unbounded stream "grows memory without limit" and that a consumer "MUST NOT rely on the accumulated + final result". It is not in the docs, not in the types, and not in any event the engine emits. diff --git a/docs/scenarios/proofs/large-response-memory/batched-export.ts b/docs/scenarios/proofs/large-response-memory/batched-export.ts new file mode 100644 index 00000000..36bbbcc4 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/batched-export.ts @@ -0,0 +1,123 @@ +// USER CODE — the assembled answer (C8). One custom {@link Surface}, and it exists for exactly one +// reason: the engine keeps every `delta` it emits. +// +// engine.ts:1437-1444 +// // MEMORY NOTE: every chunk is accumulated so the awaited/`.stream()` result can mirror the +// // whole delta spine ... +// chunks.push(chunk); +// yield { type: 'delta', chunk, at: now() }; +// +// `chunks` is unconditional. It is not gated on the accessor — `.stream()` gets it too — and there +// is no config that turns it off. So `stream: { decode: 'json' }` streams the PARSE (the decoder's +// working set really is one record: C3) and then hands the whole array straight back to the heap. +// +// The seam that fixes it is `Surface.stream`. It is the last place a value exists before the engine +// sees it, so a hook that CONSUMES the rows and yields something small per batch keeps `chunks` +// bounded by the number of batches instead of the number of rows. Everything the engine wraps — +// auth, retry on the connect, throttle, timeout, trace, the `verdict` gate — still applies, because +// this is a surface, not a bypass. +// +// What you give up by doing it here, and it is not nothing: +// - `output` no longer describes a row. The engine validates the DELTA, and the delta is now a +// receipt (`engine.ts:1414-1419`), so the per-record contract has to move inside this hook. +// - the awaited result is the receipts, not the rows. That is the point, but it means the call +// site's type changes and every `for (const row of await …)` downstream has to change with it. +import { streamSurface } from '../../../../packages/core/src/stream'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import type { Validator } from '../../../../packages/core/src/validator'; + +/** What the consumer learns about a batch it already processed. Small, fixed size, no rows. */ +export interface BatchReceipt { + /** 1-based batch number. */ + batch: number; + /** Rows in this batch (the last one is short). */ + rows: number; + /** Rows processed so far, across every batch. */ + total: number; +} + +export interface BatchedOptions { + /** Rows to accumulate before handing them to `onBatch`. The memory ceiling, in records. */ + batch: number; + /** Do the work — the database insert, the file write. Awaited, so it applies backpressure. */ + onBatch: (rows: unknown[]) => void | Promise; + /** Per-RECORD contract. Runs here because the engine's `output` now sees receipts, not rows. */ + validate?: Validator['validate']; +} + +/** + * The `stream` surface with a batching consumer folded into its decode hook. Configure the stitch + * with `stream: { decode: 'json' }` exactly as before — this hook delegates the decoding to + * `streamSurface.stream`, so `stream.decode` and `stream.buffer.chars` keep working. + */ +export function batchedSurface(opts: BatchedOptions): Surface { + const decode = streamSurface.stream as NonNullable; + return { + id: 'batched-json', + stream: async function* batchedStream(res, cfg) { + let buf: unknown[] = []; + let total = 0; + let batch = 0; + for await (const row of decode(res, cfg)) { + if (opts.validate) { + const r = await opts.validate(row); + if (!r.ok) + throw new Error( + `row ${String(total + buf.length)}: ${r.issues[0]?.message ?? 'invalid'}`, + ); + } + buf.push(row); + if (buf.length < opts.batch) continue; + await opts.onBatch(buf); + total += buf.length; + batch++; + const receipt: BatchReceipt = { + batch, + rows: buf.length, + total, + }; + buf = []; // release the batch BEFORE yielding — the engine is about to retain what we yield + yield receipt; + } + if (buf.length === 0) return; + await opts.onBatch(buf); + total += buf.length; + batch++; + const receipt: BatchReceipt = { batch, rows: buf.length, total }; + buf = []; + yield receipt; + }, + }; +} + +/** What a completed batched export produced. */ +export interface BatchedRun { + receipts: number; + rows: number; + /** The terminal `error` message, when the run failed. */ + error?: string; +} + +/** + * Drain a batched stitch's `.stream()`. The receipts are the progress bar — `total` after every + * batch — and a failed run still reports how far it got, because the error arrives as an EVENT and + * the rows before it are already committed. + */ +export async function drainBatched( + events: AsyncIterable, + onReceipt?: (r: BatchReceipt) => void, +): Promise { + let receipts = 0; + let rows = 0; + let error: string | undefined; + for await (const ev of events) { + if (ev.type === 'delta') { + const r = ev.chunk as BatchReceipt; + receipts++; + rows = r.total; + onReceipt?.(r); + } else if (ev.type === 'error') error = ev.message; + } + return error === undefined ? { receipts, rows } : { receipts, rows, error }; +} diff --git a/docs/scenarios/proofs/large-response-memory/c1-buffered-baseline.ts b/docs/scenarios/proofs/large-response-memory/c1-buffered-baseline.ts new file mode 100644 index 00000000..729b64db --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c1-buffered-baseline.ts @@ -0,0 +1,89 @@ +// C1 — the baseline. `await` a large JSON body and record the heap high-water mark against the wire +// size. What is the multiplier on this runtime, and does it scale with row count? +// +// The capture quotes 84 MB → ~2.1 GB (≈25x) from the field and 2–5x as the folklore. Neither is a +// number about StitchAPI, so this claim measures three things instead of one: +// (a) the shape — linear or not; +// (b) the multiplier — on THIS runtime, with an honest row shape; +// (c) the library's share of it — `await stitch()` against a bare `JSON.parse` of the same bytes. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c1-buffered-baseline.ts +import { + check, + checkAtMost, + checkLinear, + finish, + heading, + mb, + note, + x, +} from './harness'; +import { SCALES, probeOk, series } from './run-probe'; + +function main(): void { + heading( + 'C1 — `await` a big JSON array: what does it cost, and does it scale?', + ); + + // ── (a) the shape ───────────────────────────────────────────────────────────────────────── + // Three row counts, three processes, one workload: `await stitch({ url })` against a transport + // that reads the body to text and `JSON.parse`s it — byte for byte what `fetchAdapter` does + // (http-adapter.ts:133-138). + const buffered = series('buffered'); + for (const [i, m] of buffered.entries()) { + note( + `(a) ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `peak live ${mb(m.peakLive)} = ${x(m.ratio)} wire (peak heap incl. garbage ${mb(m.peakHeap)}, ${String(m.ms)}ms)`, + ); + } + const [small, mid, big] = buffered as [ + (typeof buffered)[0], + (typeof buffered)[0], + (typeof buffered)[0], + ]; + check('(a) rows delivered at 100k', big.records, 100_000); + checkLinear('(a) 10k -> 100k peak heap', mid.peakLive, big.peakLive, 8); + checkLinear('(a) 1k -> 10k peak heap', small.peakLive, mid.peakLive, 5); + note( + '(a) → the buffered path is LINEAR in the response', + 'nothing about it is bounded; the only question is where your heap limit sits', + ); + + // ── (b) the multiplier ──────────────────────────────────────────────────────────────────── + // ~2.5x on Node 24 for a flat 8-field record. NOT the ~25x of the incident — and the gap is + // worth stating plainly rather than quietly reproducing the bigger number. + note('(b) multiplier at 100k rows', x(big.ratio)); + checkAtMost('(b) peak heap ÷ wire bytes', big.ratio, 4, x); + note( + '(b) → the incident report’s ~25x is not what a flat record costs', + 'row SHAPE dominates: many small nested objects carry far more per-object overhead than these eight fields do. The number that transfers is the SHAPE (linear), not the constant', + ); + + // ── (c) the library's share ─────────────────────────────────────────────────────────────── + // The same bytes, no library: read to text, `JSON.parse`. If StitchAPI added a copy, this is + // where it would show. + const bare = probeOk({ mode: 'parse-only', rows: 100_000 }); + note( + '(c) bare `JSON.parse` of the same bytes', + `${mb(bare.peakLive)} = ${x(bare.ratio)} wire`, + ); + note('(c) `await stitch()` over the same bytes', `${mb(big.peakLive)}`); + const overhead = big.peakLive - bare.peakLive; + checkAtMost( + '(c) StitchAPI’s overhead above bare JSON.parse', + Math.abs(overhead), + 2 * 1024 * 1024, + mb, + ); + note( + '(c) → the 2.5x is `JSON.parse`’s, not the library’s', + 'StitchAPI adds no copy on the buffered path — which also means it removes none', + ); + + finish( + 'C1', + 'LINEAR, and the multiplier is ~2.5x on this runtime — not the ~25x of the incident. Three processes, three row counts: 100k rows / 21.4MB of wire peaked at 53.8MB of RETAINED heap (post-forced-GC), 10x rows gave ~9.6x heap, and a bare `JSON.parse` of the same bytes cost the same to within 0.2MB. So the buffered path adds nothing and bounds nothing: `await` is `JSON.parse`, and `JSON.parse` is linear in the body with no ceiling anywhere. The constant is a property of the ROW SHAPE (eight flat fields here; the incident’s 25x implies far more per-object overhead), so the number that transfers between machines is the slope, not the multiplier', + ); +} + +main(); diff --git a/docs/scenarios/proofs/large-response-memory/c2-ndjson-flat.ts b/docs/scenarios/proofs/large-response-memory/c2-ndjson-flat.ts new file mode 100644 index 00000000..f171bbf8 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c2-ndjson-flat.ts @@ -0,0 +1,106 @@ +// C2 — `stream` with `decode: 'ndjson'`. Does peak heap stay FLAT as the body grows 1x / 10x / 100x? +// +// This was supposed to be the control that proves the measurement works. It is — and it also +// contains the finding the rest of this directory turns on, because the flat half and the linear +// half are in the same call: +// +// (a) the DECODER, called exactly as the engine calls it, is O(1). 1M rows / 214MB of wire ran in +// 0.8MB of retained heap. +// (b) the same decoder THROUGH the engine is O(N), because `runStreaming` keeps every delta it +// emits (engine.ts:1443) so the terminal `result` can mirror the whole spine. +// (c) `.stream()` does not opt out of (b). It costs what `await` costs. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c2-ndjson-flat.ts +import { + check, + checkFlat, + checkLinear, + finish, + heading, + mb, + note, + x, +} from './harness'; +import { SCALES, probeOk, series } from './run-probe'; + +function main(): void { + heading('C2 — `decode: "ndjson"`: is the streaming path flat?'); + + // ── (a) the decoder alone — the control ─────────────────────────────────────────────────── + // `streamSurface.stream(res, cfg)` is the exact call the engine makes (engine.ts:1403). Driving + // it directly measures the decoder with nothing accumulating around it. + const decoder = series('decoder-ndjson'); + for (const [i, m] of decoder.entries()) + note( + `(a) decoder alone, ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `${mb(m.peakLive)} retained = ${x(m.ratio)} wire`, + ); + const [d1, d10, d100] = decoder as [ + (typeof decoder)[0], + (typeof decoder)[0], + (typeof decoder)[0], + ]; + checkFlat('(a) 1x -> 100x decoder heap', d1.peakLive, d100.peakLive); + checkFlat('(a) 10x -> 100x decoder heap', d10.peakLive, d100.peakLive); + // 100x again on top, to put the claim beyond any argument about constants. + const huge = probeOk({ mode: 'decoder-ndjson', rows: 1_000_000 }); + note( + '(a) decoder alone, 1,000,000 rows', + `${mb(huge.wireBytes)} wire -> ${mb(huge.peakLive)} retained = ${x(huge.ratio)}`, + ); + check('(a) records decoded at 1M', huge.records, 1_000_000); + checkFlat('(a) 1k -> 1M decoder heap', d1.peakLive, huge.peakLive); + note( + '(a) → the measurement instrument is sound', + 'a 1000x change in the workload moved retained heap by less than 15%', + ); + + // ── (b) the same decoder through the engine ─────────────────────────────────────────────── + const engine = series('stream-ndjson'); + for (const [i, m] of engine.entries()) + note( + `(b) via .stream(), ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `${mb(m.peakLive)} retained = ${x(m.ratio)} wire`, + ); + const [, e10, e100] = engine as [ + (typeof engine)[0], + (typeof engine)[0], + (typeof engine)[0], + ]; + checkLinear('(b) 10x -> 100x engine heap', e10.peakLive, e100.peakLive, 6); + note( + '(b) 1x is reported but NOT asserted on', + 'at 1000 rows the run’s one-time cost (module init, JIT, IC feedback — ~1.3MB, visible as `settled`) is six times the data, so the 1x point cannot carry a growth claim. 10x -> 100x is where the signal is clean', + ); + const cost = e100.peakLive - d100.peakLive; + note( + '(b) what the engine added at 100k rows', + `${mb(d100.peakLive)} (decoder) -> ${mb(e100.peakLive)} (engine) = +${mb(cost)}`, + ); + note( + '(b) → engine.ts:1443 `chunks.push(chunk)`', + 'unconditional, ungated by accessor, with no config that turns it off. The decoder streams; the engine collects', + ); + + // ── (c) `.stream()` vs `await` — the accessor changes nothing ───────────────────────────── + const awaited = probeOk({ mode: 'stream-ndjson-await', rows: 100_000 }); + note('(c) `.stream()` at 100k', mb(e100.peakLive)); + note('(c) `await` at 100k', mb(awaited.peakLive)); + checkFlat( + '(c) `.stream()` vs `await`', + Math.min(e100.peakLive, awaited.peakLive), + Math.max(e100.peakLive, awaited.peakLive), + 1.3, + ); + note( + '(c) → “iterate instead of awaiting” is not a memory fix here', + 'both accessors drain the same generator, and the accumulator is inside it', + ); + + finish( + 'C2', + 'NO — and the two halves of the answer are in the same call. The DECODER is flat: driven directly, `decode: "ndjson"` held 0.8MB of retained heap for 1,000,000 rows and 214MB of wire, and moved less than 15% across a 1000x change in workload. Through the ENGINE the same decoder is linear — 3.5MB -> 30.2MB from 10k to 100k rows — because `runStreaming` pushes every delta onto a `chunks` array (engine.ts:1443) so the terminal `result` can mirror the whole spine. `.stream()` does not escape it: 30.2MB iterating against 33.5MB awaiting, the same number twice. The library owns a genuinely O(1) NDJSON decoder and then spends the win one line later', + ); +} + +main(); diff --git a/docs/scenarios/proofs/large-response-memory/c3-json-single-array.ts b/docs/scenarios/proofs/large-response-memory/c3-json-single-array.ts new file mode 100644 index 00000000..72e32ca7 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c3-json-single-array.ts @@ -0,0 +1,267 @@ +// C3 — THE DECIDING CLAIM. `decode: 'json'` against ONE SINGLE TOP-LEVEL ARRAY. +// +// types.ts:1441-1450 documents it as "the structural, unframed streaming-JSON decoder (issue #111): +// one `delta` per complete value / top-level array element, tolerant of internal newlines and +// concatenated values". If that holds with bounded memory, the hard case of this whole scenario is a +// config value — a real capability few clients have. +// +// The answer is split, and the split is the finding: +// (a) EMISSION is correct, and impressively so. Right count, right boundaries, records containing +// `,` `]` `}` and escaped quotes inside strings, pretty-printed records spanning many lines, +// nested arrays and objects, chunk boundaries placed one character apart. +// (b) MEMORY is not bounded. The decoder retains the ENTIRE array text — 0.88x the wire, growing +// linearly — because the compaction floor is pinned to the array's opening `[`. +// (c) It therefore HITS ITS OWN CAP. On defaults, a single array larger than 8,388,608 characters +// fails the stream, and the error blames the vendor for something the vendor did not do. +// (d) The same records as CONCATENATED top-level values are flat. The defect is one branch, not +// the decoder. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c3-json-single-array.ts +import { jsonStream } from '../../../../packages/core/src/json-stream'; +import { JSON_STREAM_DEFAULT_MAX_BUFFER_CHARS } from '../../../../packages/core/src/json-stream'; +import { stream } from '../../../../packages/core/src/stream'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { + HOSTILE_ARRAY, + PRETTY_ARRAY, + singleArray, + splitStream, + streamingAdapter, +} from './fake-export'; +import { + check, + checkFlat, + checkLinear, + checkSeq, + finish, + heading, + mb, + note, + x, +} from './harness'; +import { SCALES, probe, probeOk, series } from './run-probe'; + +const URL = 'https://api.vendor.example/v1/products/export'; + +/** Decode a text body with `decode: 'json'` and return the emitted values. */ +async function decode( + text: string, + splitEvery = 1_000_000, +): Promise { + const out: unknown[] = []; + for await (const v of jsonStream(splitStream(text, splitEvery).body)) + out.push(v); + return out; +} + +/** Drain a `.stream()` spine into its event type names plus the deltas and any error. */ +async function spine( + events: AsyncIterable, +): Promise<{ types: string[]; deltas: unknown[]; error?: string }> { + const types: string[] = []; + const deltas: unknown[] = []; + let error: string | undefined; + for await (const ev of events) { + types.push(ev.type === 'progress' ? `progress:${ev.phase}` : ev.type); + if (ev.type === 'delta') deltas.push(ev.chunk); + else if (ev.type === 'error') error = ev.message; + } + return error === undefined ? { types, deltas } : { types, deltas, error }; +} + +async function main(): Promise { + heading( + 'C3 — `decode: "json"` over one top-level array: streams, or buffers?', + ); + + // ── (a) correctness: does it find the right boundaries? ─────────────────────────────────── + { + const rows = await decode('[{"id":1},{"id":2},{"id":3}]'); + checkSeq( + '(a) plain array -> one delta per ELEMENT (not the array)', + rows, + [{ id: 1 }, { id: 2 }, { id: 3 }], + ); + + // Pretty-printed: every record spans several LINES. This is the case `'ndjson'` cannot do + // at all, and it is the decoder's headline capability. + const pretty = await decode(PRETTY_ARRAY); + check('(a) pretty-printed array -> elements', pretty.length, 2); + checkSeq( + '(a) pretty element 2 (nested, with a `}` inside a string)', + [pretty[1]], + [{ id: 'b', nested: { deep: [1, 2, { x: '}' }] } }], + ); + + // Structural characters inside STRING VALUES, escapes, embedded newlines, deep nesting. + const hostile = (await decode(HOSTILE_ARRAY)) as { id: number }[]; + check('(a) hostile array -> element count', hostile.length, 4); + checkSeq( + '(a) hostile element ids in order', + hostile.map((r) => r.id), + [1, 2, 3, 4], + ); + check( + '(a) a `,` and a `]` inside a string value did not split the record', + JSON.stringify(hostile[0]), + '{"id":1,"s":"has , comma and ] bracket and } brace"}', + ); + + // Chunk boundaries every SINGLE character: every token is split across reads. + const shredded = await decode(HOSTILE_ARRAY, 1); + checkSeq( + '(a) same body shredded to 1-char chunks -> identical', + shredded, + hostile, + ); + note( + '(a) → emission is CORRECT, and it is the capability the scenario asked for', + 'a single array is decoded element by element with the right boundaries under every adversarial input tried', + ); + } + + // ── (b) the same, through a real stitch, with the event spine ───────────────────────────── + { + const wire = singleArray(3); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'json' }, + }); + const s = await spine(exportAll.stream()); + checkSeq('(b) event spine for a 3-element array', s.types, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'delta', + 'result', + 'done', + ]); + check('(b) deltas', s.deltas.length, 3); + check( + '(b) delta 1 is the ROW', + JSON.stringify((s.deltas[0] as { id: string }).id), + '"prd_0000000"', + ); + } + + // ── (c) memory: does it stream, or hold the array? ──────────────────────────────────────── + const json = series('decoder-json'); + for (const [i, m] of json.entries()) + note( + `(c) decoder alone, ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `${mb(m.peakLive)} retained = ${x(m.ratio)} wire, ${String(m.ms)}ms`, + ); + const [j1, j10, j100] = json as [ + (typeof json)[0], + (typeof json)[0], + (typeof json)[0], + ]; + checkLinear('(c) 10x -> 100x decoder heap', j10.peakLive, j100.peakLive, 6); + checkLinear('(c) 1x -> 100x decoder heap', j1.peakLive, j100.peakLive, 15); + note( + '(c) retained heap ÷ wire bytes at 100k rows', + `${x(j100.ratio)} — the decoder is holding the WHOLE ARRAY TEXT`, + ); + note( + '(c) json-stream.ts:230-236', + 'the compaction floor is `valueStart`, and for a top-level ARRAY `valueStart` is the opening `[` (line 163) and stays there until the closing `]` (line 183). `compact(live)` is therefore a no-op for the entire array', + ); + note( + '(c) and it is quadratic in TIME as well', + `${String(j10.ms)}ms at 10k rows -> ${String(j100.ms)}ms at 100k — 10x the rows, ~${String(Math.round(j100.ms / Math.max(1, j10.ms)))}x the time, because every chunk re-flattens the growing buffer (peak heap incl. garbage: ${mb(j100.peakHeap)})`, + ); + + // ── (d) the CAP: on defaults, a big array fails outright ────────────────────────────────── + check( + '(d) the default cap, in characters', + JSON_STREAM_DEFAULT_MAX_BUFFER_CHARS, + 8_388_608, + ); + const under = probe({ mode: 'decoder-json', rows: 37_000 }); + const over = probe({ mode: 'decoder-json', rows: 38_000 }); + check('(d) 37,000 rows (7.9MB of wire) — decoded?', under.ok, true); + check('(d) 38,000 rows (8.2MB of wire) — decoded?', over.ok, false); + check( + '(d) the message the caller gets', + over.ok ? '' : over.error, + 'json decoder: in-progress value exceeded the stream.buffer.chars cap (8388608); a malformed or never-closing value was streamed', + ); + note( + '(d) → the message is WRONG about the cause', + 'nothing was malformed and nothing failed to close. A perfectly well-formed 8.2MB array trips a guard written for an unterminated one, and the error tells you to go and look at the vendor', + ); + // What the same body does through a real stitch: an `error` event, and the deltas already + // emitted are kept. + { + const wire = singleArray(60_000); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'json' }, + }); + const s = await spine(exportAll.stream()); + check( + '(e) 60,000 rows on defaults -> error event?', + s.error !== undefined, + true, + ); + // The hypothesis here was "zero deltas — the guard runs on the buffer, so it trips before + // anything is emitted". WRONG, in the library's favour: `pending` is drained and yielded + // BEFORE `guard()` runs (json-stream.ts:224-238), so every element that closed under the cap + // is delivered first. The failure is a TRUNCATION, not a total loss. + check('(e) deltas delivered before it failed', s.deltas.length, 37_312); + check( + '(e) …of how many rows', + `${String(s.deltas.length)}/60000`, + '37312/60000', + ); + checkSeq('(e) terminal spine', s.types.slice(-2), ['error', 'done']); + note( + '(e) → a PARTIAL result, then a wrong diagnosis', + 'the consumer gets 62% of the catalog and an error blaming the vendor’s framing. A `.stream()` loop that only matches `delta` sees a silent truncation at a threshold that moves with the vendor’s data', + ); + } + + // ── (f) the control: the SAME records as concatenated top-level values ──────────────────── + const concat = series('decoder-concat'); + for (const [i, m] of concat.entries()) + note( + `(f) concatenated \`{…}{…}\`, ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `${mb(m.peakLive)} retained = ${x(m.ratio)} wire`, + ); + const [c1, , c100] = concat as [ + (typeof concat)[0], + (typeof concat)[0], + (typeof concat)[0], + ]; + checkFlat('(f) 1x -> 100x concatenated heap', c1.peakLive, c100.peakLive); + check('(f) records decoded', c100.records, 100_000); + note( + '(f) → the defect is one BRANCH, not the decoder', + 'identical records, identical bytes: as siblings they cost 0.9MB flat, wrapped in one array they cost 19.7MB and rising. The compaction is right for a top-level value and pinned for a top-level array', + ); + + // ── (g) what the raised cap buys you ────────────────────────────────────────────────────── + const raised = probeOk({ + mode: 'decoder-json', + rows: 100_000, + buffer: 1_000_000_000, + }); + note( + '(g) `stream: { decode: "json", buffer: { chars: 1e9 } }` at 100k rows', + `succeeds, at ${mb(raised.peakLive)} retained and ${mb(raised.peakHeap)} peak including garbage`, + ); + note( + '(g) → raising the cap converts a hard failure into the memory profile you were trying to avoid', + 'it is the `--max-old-space-size` move from the capture’s own table: it moves the cliff, it does not remove it', + ); + + finish( + 'C3', + 'It STREAMS THE PARSE and BUFFERS THE TEXT — correct emission, unbounded memory, and on defaults it does not finish. Emission is genuinely right: one delta per element, holding up under `,`/`]`/`}` inside string values, escaped quotes, embedded newlines, pretty-printed multi-line records, deep nesting, and 1-character chunk boundaries. Memory is not: retained heap tracks the WHOLE ARRAY TEXT at 0.88x the wire and 34x growth over a 100x workload, because `compact()` floors on `valueStart` and for a top-level array `valueStart` is the opening `[` (json-stream.ts:163, 183, 230-236) — and the time is quadratic too, 28x for 10x the rows. So the decoder trips its OWN default guard: 37,000 rows decode, 38,000 fail, and the message — "a malformed or never-closing value was streamed" — accuses the vendor of something it did not do. On a 60,000-row array the consumer gets 37,312 rows and then `error`/`done(ok:false)`: a SILENT TRUNCATION for any loop that only matches `delta`, at a threshold that moves with the vendor’s data. The control settles where the defect is: the same 100,000 records as CONCATENATED top-level values run flat at 0.9MB. One branch of one function, not a design limit', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/c4-output-per-delta.ts b/docs/scenarios/proofs/large-response-memory/c4-output-per-delta.ts new file mode 100644 index 00000000..3fdc33a8 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c4-output-per-delta.ts @@ -0,0 +1,207 @@ +// C4 — THE OTHER DECIDING CLAIM. Add an `output` schema to a streaming stitch. Does validation run +// PER DELTA or over the AGGREGATE? +// +// The capture's fear: "if `output` on a streaming stitch buffers every delta to validate the +// aggregate, the streaming is undone — and the config would look correct." Scenario 11 measured +// `output` running over the whole aggregated array for `paginate`, which is exactly that shape. +// +// This is answered with a COUNTER, not a heap number. The `output` contract here is an instrumented +// `Validator` that records how many times it was called and what shape each argument had. One call +// carrying a 100,000-element array and 100,000 calls each carrying one row are not two readings of +// the same number — they are different facts, and the counter reports which one happened. +// +// The heap measurement is the consequence, and it comes second. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c4-output-per-delta.ts +import { stream } from '../../../../packages/core/src/stream'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { + ndjson, + productRow, + singleArray, + streamingAdapter, +} from './fake-export'; +import { check, checkFlat, finish, heading, mb, note, x } from './harness'; +import { probeOk } from './run-probe'; +import { coercingValidator, countingValidator } from './validator-spy'; + +const URL = 'https://api.vendor.example/v1/products/export'; + +async function drain(events: AsyncIterable): Promise<{ + deltas: unknown[]; + drift: string[]; + error?: string; + types: string[]; +}> { + const deltas: unknown[] = []; + const drift: string[] = []; + const types: string[] = []; + let error: string | undefined; + for await (const ev of events) { + types.push(ev.type === 'progress' ? `progress:${ev.phase}` : ev.type); + if (ev.type === 'delta') deltas.push(ev.chunk); + else if (ev.type === 'drift') + drift.push( + `${ev.finding.level}|${ev.finding.change}|${ev.finding.path}|${ev.finding.detail ?? ''}`, + ); + else if (ev.type === 'error') error = ev.message; + } + return error === undefined + ? { deltas, drift, types } + : { deltas, drift, types, error }; +} + +async function main(): Promise { + heading( + 'C4 — an `output` schema on a streaming stitch: per delta, or over the aggregate?', + ); + + // ── (a) the mechanism: count the calls ──────────────────────────────────────────────────── + { + const spy = countingValidator(); + const wire = ndjson(500); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + output: spy, + }); + const r = await drain(exportAll.stream()); + check('(a) deltas emitted', r.deltas.length, 500); + check('(a) times `output` was called', spy.calls(), 500); + check('(a) calls carrying ONE record', spy.recordCalls(), 500); + check( + '(a) largest array ever handed to the validator', + spy.sawArrayOfLength(), + 0, + ); + note( + '(a) → PER DELTA, unambiguously', + '500 calls, 500 of them carrying a single object, and the validator never saw an array at all. `engine.ts:1414-1419` runs it inside the decode loop, before the `delta` is emitted', + ); + } + + // ── (b) the same over `decode: 'json'` — a single top-level array ───────────────────────── + // The shape the fear was really about: the wire IS one array. Does the contract see the array? + { + const spy = countingValidator(); + const wire = singleArray(500); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'json' }, + output: spy, + }); + const r = await drain(exportAll.stream()); + check('(b) deltas emitted', r.deltas.length, 500); + check('(b) times `output` was called', spy.calls(), 500); + check( + '(b) largest array handed to the validator', + spy.sawArrayOfLength(), + 0, + ); + note( + '(b) → the contract describes a ROW, not the export', + 'even when the wire is literally one array, `output` never sees it. The decoder’s element is the unit of validation', + ); + } + + // ── (c) a failing row fails the STREAM, and the bad value never arrives ─────────────────── + { + const spy = countingValidator(); + // Row 3 is not a product — the contract must reject it. + const wire = singleArray(6, (i) => + i === 3 ? { nope: true } : productRow(i), + ); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'json' }, + output: spy, + }); + const r = await drain(exportAll.stream()); + check('(c) deltas delivered before the bad row', r.deltas.length, 3); + check('(c) the bad row was never emitted', spy.rejects(), 1); + check('(c) error message', r.error, 'contract violation (drift)'); + check('(c) terminal event', r.types.at(-1), 'done'); + note( + '(c) → a contract on a stream is a CIRCUIT BREAKER, not a filter', + 'one bad row ends the export; the 3 good rows already delivered stay delivered, and rows 5 and 6 are never read', + ); + } + + // ── (d) the value served is the RAW chunk, not the validated one ────────────────────────── + // The buffered path serves `validated` (engine.ts:1223, "serve the validated value"). The + // streaming path destructures only `{ errors }` from the same function (engine.ts:1419) and + // throws the validated value away. So a coercing/stripping schema type-checks and then does + // nothing to what the consumer receives. + { + const wire = singleArray(2); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'json' }, + output: coercingValidator(), + }); + const r = await drain(exportAll.stream()); + check('(d) deltas', r.deltas.length, 2); + const first = r.deltas[0] as Record; + check( + '(d) did the validator’s added field arrive?', + first['coerced_marker'], + undefined, + ); + check('(d) the raw field is intact', first['currency'], 'usd'); + note( + '(d) → on a stream, `output` VALIDATES but never TRANSFORMS', + 'engine.ts:1419 keeps `{ errors }` and drops `value`; engine.ts:1223 on the buffered path does the opposite. The same schema coerces on `await` and does not on `.stream()`', + ); + } + + // ── (e) the cost: heap with and without ─────────────────────────────────────────────────── + const bare = probeOk({ mode: 'stream-ndjson', rows: 100_000 }); + const validated = probeOk({ mode: 'stream-ndjson-output', rows: 100_000 }); + note( + '(e) 100k rows, `.stream()`, no `output`', + `${mb(bare.peakLive)} retained = ${x(bare.ratio)} wire`, + ); + note( + '(e) 100k rows, `.stream()`, `output` on every row', + `${mb(validated.peakLive)} retained = ${x(validated.ratio)} wire`, + ); + checkFlat( + '(e) what `output` added to peak heap', + bare.peakLive, + validated.peakLive, + 1.15, + ); + const jbare = probeOk({ + mode: 'stream-json', + rows: 100_000, + buffer: 1_000_000_000, + }); + const jval = probeOk({ + mode: 'stream-json-output', + rows: 100_000, + buffer: 1_000_000_000, + }); + note('(e) same, `decode: "json"` without `output`', mb(jbare.peakLive)); + note('(e) same, `decode: "json"` with `output`', mb(jval.peakLive)); + checkFlat( + '(e) what `output` added over `json`', + jbare.peakLive, + jval.peakLive, + 1.15, + ); + note( + '(e) → `output` is INNOCENT', + 'the hypothesis this claim was written to catch does not happen. Validation is free in memory terms, on both decoders. What is expensive is `chunks` (C2) and the array buffer (C3), neither of which `output` touches', + ); + + finish( + 'C4', + 'PER DELTA, and the fear was misplaced. The instrumented contract was called 500 times for 500 records, every call carrying ONE object, and `sawArrayOfLength` stayed at 0 — including when the wire was literally one top-level array under `decode: "json"`. In heap terms `output` is free: 30.2MB against 30.2MB on `ndjson`, 48.7MB against 48.8MB on `json`, both within 1%. Two things it does that a config author will not expect. A failing row is a CIRCUIT BREAKER, not a filter: `contract violation (drift)`, the stream ends, the rows already delivered stay and the rest are never read. And on a stream `output` VALIDATES WITHOUT TRANSFORMING — `engine.ts:1419` keeps only `{ errors }` and discards the validated value, where `engine.ts:1223` on the buffered path serves it. The same coercing schema reshapes your data on `await` and silently does not on `.stream()`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/c5-pick-transform.ts b/docs/scenarios/proofs/large-response-memory/c5-pick-transform.ts new file mode 100644 index 00000000..d2af2447 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c5-pick-transform.ts @@ -0,0 +1,167 @@ +// C5 — `pick` / `transform` on a streaming stitch: per-delta or buffering? +// +// The question the capture asked is the wrong question, and finding that out is the claim. Neither +// runs at all. `runStreaming` never calls them (engine.ts:1244-1247 says so in a comment; the code +// path simply has no call site), so the honest answers are "neither" — and the interesting part is +// what that failure MODE looks like from the call site, because a `pick` that silently does nothing +// is a config that reads correct and returns the wrong shape. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c5-pick-transform.ts +import { stitch } from '../../../../packages/core/src/index'; +import { stream } from '../../../../packages/core/src/stream'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { + bufferingAdapter, + ndjson, + singleArray, + streamingAdapter, +} from './fake-export'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const URL = 'https://api.vendor.example/v1/products/export'; + +async function deltas(events: AsyncIterable): Promise<{ + chunks: unknown[]; + types: string[]; +}> { + const chunks: unknown[] = []; + const types: string[] = []; + for await (const ev of events) { + types.push(ev.type === 'progress' ? `progress:${ev.phase}` : ev.type); + if (ev.type === 'delta') chunks.push(ev.chunk); + } + return { chunks, types }; +} + +async function main(): Promise { + heading('C5 — `pick` and `transform` on a streaming stitch'); + + // ── (a) `transform` on a stream: called at all? ─────────────────────────────────────────── + { + let calls = 0; + let sawArrayOfLength = 0; + const wire = ndjson(200); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + transform: (body: unknown) => { + calls++; + if (Array.isArray(body)) + sawArrayOfLength = Math.max(sawArrayOfLength, body.length); + return { marked: true }; + }, + }); + const r = await deltas(exportAll.stream()); + check('(a) deltas emitted', r.chunks.length, 200); + check('(a) times `transform` was called', calls, 0); + check('(a) largest array it saw', sawArrayOfLength, 0); + check( + '(a) did the transform’s value reach the consumer?', + (r.chunks[0] as { marked?: boolean }).marked, + undefined, + ); + check( + '(a) the raw row arrived instead', + (r.chunks[0] as { currency?: string }).currency, + 'usd', + ); + note( + '(a) → NEITHER per-delta NOR buffering. `transform` is DEAD on a stream', + 'not called once, and no event says so: no `info`, no drift finding, no throw', + ); + } + + // ── (b) `pick` on a stream: same ────────────────────────────────────────────────────────── + // The realistic authoring mistake: the vendor wraps the export in `{ data: [ … ] }` and the + // author reaches for the same `pick: 'data'` that works everywhere else. + { + const wire = ndjson(50); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + pick: 'id', + }); + const r = await deltas(exportAll.stream()); + check('(b) deltas emitted', r.chunks.length, 50); + check( + '(b) is the delta the picked `id`, or the whole row?', + typeof r.chunks[0], + 'object', + ); + check( + '(b) `pick: "id"` produced', + JSON.stringify( + (r.chunks[0] as Record)['id'] ?? null, + ), + '"prd_0000000"', + ); + note( + '(b) → the whole row, `pick` ignored', + 'and the STATIC type follows `output`, not `pick`, so the call site does not catch it either', + ); + } + + // ── (c) the same two on the BUFFERED path, for contrast ─────────────────────────────────── + // Identical spelling, opposite behaviour. This is the asymmetry a docs page has to name. + { + let calls = 0; + const wire = singleArray(50); + const exportAll = stitch({ + url: URL, + adapter: bufferingAdapter(wire), + transform: (body: unknown) => { + calls++; + return { data: body as unknown[] }; + }, + pick: 'data', + }); + const rows = (await exportAll()) as unknown[]; + check('(c) `transform` called on the buffered path', calls, 1); + check('(c) it received the WHOLE array', rows.length, 50); + note( + '(c) → one call, one aggregate — the exact shape C4 feared for `output`', + '`transform` really is a whole-body hook. It is just not wired to the streaming path at all', + ); + } + + // ── (d) is there ANY signal that the config was ignored? ────────────────────────────────── + { + const wire = ndjson(10); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + pick: 'nonexistent.deep.path', + transform: () => ({ nonsense: true }), + }); + const r = await deltas(exportAll.stream()); + checkSeq( + '(d) full event spine with both slots set and both ignored', + [...new Set(r.types)], + ['start', 'progress:request', 'delta', 'result', 'done'], + ); + check( + '(d) `info` events', + r.types.filter((t) => t === 'info').length, + 0, + ); + check( + '(d) `drift` findings', + r.types.filter((t) => t === 'drift').length, + 0, + ); + note( + '(d) → completely silent', + 'the engine teaches elsewhere (an upload progress bar the transport cannot draw gets an `info` event — engine.ts:1694-1698). Two ignored config slots on a stream get nothing', + ); + } + + finish( + 'C5', + 'NEITHER — they do not run. `transform` was called ZERO times over 200 deltas and `pick` changed nothing: the consumer got the raw decoded row, `pick: "id"` and all. The same two slots on the same fake vendor over the BUFFERED path behave exactly as documented — one `transform` call carrying the whole 50-row array, `pick` selecting from it — so this is a path asymmetry, not a broken hook. And it is completely silent: no `info`, no drift finding, no throw, and the static delta type is derived from `output`, never from `pick`, so the call site does not catch it either. A stitch that carries `pick`/`transform` and is later switched to `kind: stream` keeps compiling and quietly stops reshaping', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/c6-backpressure.ts b/docs/scenarios/proofs/large-response-memory/c6-backpressure.ts new file mode 100644 index 00000000..2e90cfbf --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c6-backpressure.ts @@ -0,0 +1,216 @@ +// C6 — backpressure. A slow consumer against a fast producer. Does the stream buffer unboundedly? +// And what does `stream.buffer.chars` do AT the cap — throw, truncate, or block? +// +// Three separate mechanisms wear the word "buffer" here and they behave differently, so this claim +// keeps them apart: +// (a) the SOCKET QUEUE — the `ReadableStream`'s internal queue. Bounded by the reader pulling, +// and invisible to `heapUsed` because a `Uint8Array`'s backing store is external memory. +// (b) the DECODER's working buffer — what `stream.buffer.chars` caps. Per un-terminated UNIT. +// (c) the ENGINE's `chunks` array — capped by nothing at all. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c6-backpressure.ts +import { stream } from '../../../../packages/core/src/stream'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { + ndjson, + neverClosingArray, + noNewlines, + singleArray, + streamingAdapter, +} from './fake-export'; +import { + check, + checkAtLeast, + checkAtMost, + checkSeq, + finish, + heading, + mb, + note, +} from './harness'; +import { probeOk } from './run-probe'; + +/** A fraction as `1.4%` — `x()`'s one decimal place rounds these to `0.0x` and hides the point. */ +const pct = (f: number): string => `${(f * 100).toFixed(1)}%`; + +const URL = 'https://api.vendor.example/v1/products/export'; + +interface Drained { + deltas: number; + error?: string; + types: string[]; +} + +async function drain( + events: AsyncIterable, + onDelta?: (n: number) => Promise | void, +): Promise { + let deltas = 0; + const types: string[] = []; + let error: string | undefined; + for await (const ev of events) { + types.push(ev.type === 'progress' ? `progress:${ev.phase}` : ev.type); + if (ev.type === 'delta') { + deltas++; + await onDelta?.(deltas); + } else if (ev.type === 'error') error = ev.message; + } + return error === undefined ? { deltas, types } : { deltas, types, error }; +} + +async function main(): Promise { + heading( + 'C6 — a slow consumer, a fast producer, and three different buffers', + ); + + // ── (a) does a slow consumer slow the producer? ─────────────────────────────────────────── + // The consumer awaits a macrotask per delta — as slow as a real database insert. If backpressure + // did not propagate, the producer would race to the end of the body while the consumer crawled. + { + const wire = ndjson(2_000); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + }); + let chunksAt100 = 0; + await drain(exportAll.stream(), async (n) => { + if (n === 100) chunksAt100 = wire.enqueued(); + await new Promise((r) => setTimeout(r, 0)); + }); + const total = wire.enqueued(); + note( + '(a) wire chunks the producer had emitted when the consumer had seen 100 of 2000 rows', + `${String(chunksAt100)} of ${String(total)}`, + ); + checkAtMost('(a) producer chunks ahead at row 100', chunksAt100, 8); + check('(a) producer chunks in total', total, 32); + note( + '(a) → backpressure PROPAGATES, end to end', + 'the chain is pull-based the whole way — `reader.read()` -> the decoder generator -> `runStreaming`’s `yield` -> your `for await`. A slow consumer really does stop the socket', + ); + } + + // ── (b) a producer that ignores it — and why `heapUsed` will not show you ───────────────── + // Same body, same consumer, but the source enqueues everything without consulting `desiredSize`. + // The whole response then sits in the stream's internal queue as `Uint8Array`s — which live in + // EXTERNAL memory, so `heapUsed` reports nothing at all. `arrayBuffers` is where it shows. + { + const lazy = probeOk({ + mode: 'stream-json', + rows: 100_000, + buffer: 1_000_000_000, + }); + const eager = probeOk({ + mode: 'eager-json', + rows: 100_000, + buffer: 1_000_000_000, + }); + note( + '(b) lazy producer, 100k rows', + `queue high-water ${mb(lazy.peakBuffers)} of ${mb(lazy.wireBytes)} wire = ${pct(lazy.peakBuffers / lazy.wireBytes)}`, + ); + note( + '(b) producer that ignores backpressure, same 100k rows', + `queue high-water ${mb(eager.peakBuffers)} of ${mb(eager.wireBytes)} wire = ${pct(eager.peakBuffers / eager.wireBytes)}`, + ); + checkAtMost( + '(b) lazy producer’s queue as a fraction of the body', + lazy.peakBuffers / lazy.wireBytes, + 0.05, + pct, + ); + checkAtLeast( + '(b) eager producer’s queue as a fraction of the body', + eager.peakBuffers / eager.wireBytes, + 0.99, + pct, + ); + note( + '(b) → the queue is not capped by anything StitchAPI owns', + '`stream.buffer.chars` counts DECODED CHARACTERS in the decoder; the bytes queued ahead of the decoder are the transport’s business. And they are invisible to `process.memoryUsage().heapUsed` — measure `arrayBuffers` or you will conclude a 21MB backlog is free', + ); + } + + // ── (c) AT the cap: throw, truncate, or block? ──────────────────────────────────────────── + // Three decoders, three un-terminated shapes, one cap. THROW in every case — surfaced as an + // `error` event, never a truncation and never a pause. + { + const cases: { + label: string; + decode: 'json' | 'ndjson' | 'lines'; + body: () => ReturnType; + }[] = [ + { + label: 'json / a value that never closes', + decode: 'json', + body: () => neverClosingArray(5_000), + }, + { + label: 'ndjson / a body with no newline', + decode: 'ndjson', + body: () => noNewlines(5_000), + }, + { + label: 'lines / a body with no newline', + decode: 'lines', + body: () => noNewlines(5_000), + }, + ]; + for (const c of cases) { + const wire = c.body(); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: c.decode, buffer: { chars: 50_000 } }, + }); + const r = await drain(exportAll.stream()); + check(`(c) ${c.label} — deltas delivered`, r.deltas, 0); + checkSeq(`(c) ${c.label} — terminal spine`, r.types.slice(-2), [ + 'error', + 'done', + ]); + note(`(c) ${c.label} — message`, r.error ?? '(none)'); + } + note( + '(c) → THROW. Not truncate, not block', + 'the decoder raises, `runStreaming` turns it into an `error` event plus `done(ok:false)` (engine.ts:1446-1448, 1466-1472). The connection is torn down; no partial unit is delivered', + ); + } + + // ── (d) the cap does NOT bound the run ──────────────────────────────────────────────────── + // The obvious misreading of `stream.buffer.chars` is "the memory ceiling for this stream". It is + // not. It bounds ONE un-terminated unit. A million perfectly-terminated 200-byte lines pass a + // 1,000-char cap one at a time — and the engine keeps every one of them. + { + const wire = ndjson(20_000); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson', buffer: { chars: 1_000 } }, + }); + const r = await drain(exportAll.stream()); + check( + '(d) 20,000 rows through a 1,000-character cap', + r.deltas, + 20_000, + ); + check('(d) errors', r.error ?? 'none', 'none'); + const big = probeOk({ mode: 'stream-ndjson', rows: 100_000 }); + note( + '(d) and the engine’s own accumulator at 100k rows', + `${mb(big.peakLive)} retained, under any cap you like — \`chunks\` is not a decoder buffer`, + ); + note( + '(d) → `stream.buffer.chars` is a MALFORMED-INPUT guard, not a memory budget', + 'it answers "how long may one line / one un-closed value get", never "how much may this call use". The only stream it bounds in total is one whose records are pathological', + ); + } + + finish( + 'C6', + 'Backpressure PROPAGATES and the cap THROWS — and neither fact bounds the call. A consumer awaiting a macrotask per row kept the producer within 8 chunks of 32: the chain is pull-based end to end, so a slow reader really does stop the socket, and a lazy producer’s queue stayed at 1.3% of the body. A producer that ignores `desiredSize` puts 100% of the body in the stream’s internal queue instead — and that backlog is INVISIBLE to `heapUsed`, because a `Uint8Array`’s store is external memory; it only shows in `arrayBuffers`. At the cap, all three decoders (`json`, `ndjson`, `lines`) THROW: an `error` event, `done(ok:false)`, zero deltas from the offending unit, no truncation and no pause. And the cap is a malformed-input guard, not a budget: 20,000 well-formed rows streamed cleanly through a 1,000-CHARACTER cap while the engine quietly accumulated all 20,000 of them', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/c7-buffered-guard.ts b/docs/scenarios/proofs/large-response-memory/c7-buffered-guard.ts new file mode 100644 index 00000000..f1c34c3e --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c7-buffered-guard.ts @@ -0,0 +1,159 @@ +// C7 — is there any guard on the BUFFERED path? A very large response on a plain `await`: does +// anything intervene — a cap, a warning, an `info` event — or is OOM the only signal? +// +// This claim is about ABSENCE, which is the hardest thing to demonstrate. So it does it three ways: +// (a) run a big body through and enumerate every event the engine emitted; +// (b) set the one cap that exists (`stream.buffer.chars`) on a buffered stitch and show it is +// accepted, typed, and completely inert; +// (c) actually kill a process. Same data, same heap ceiling, one config lives and one dies. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c7-buffered-guard.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { bufferingAdapter, singleArray } from './fake-export'; +import { + check, + checkAtMost, + checkSeq, + finish, + heading, + mb, + note, +} from './harness'; +import { probeRun } from './run-probe'; + +const URL = 'https://api.vendor.example/v1/products/export'; + +/** The heap ceiling both halves of (c) run under. Small enough to be reached, big enough to boot. */ +const HEAP_MB = 96; +/** Rows whose buffered tree exceeds that ceiling but whose streamed form is nowhere near it. */ +const ROWS = 400_000; + +async function main(): Promise { + heading('C7 — a very large buffered response: does anything intervene?'); + + // ── (a) every event a 21MB buffered response produces ───────────────────────────────────── + { + const wire = singleArray(100_000); + const exportAll = stitch({ + url: URL, + adapter: bufferingAdapter(wire), + }); + const types: string[] = []; + for await (const ev of exportAll.stream() as AsyncIterable) + types.push( + ev.type === 'progress' ? `progress:${ev.phase}` : ev.type, + ); + checkSeq('(a) the whole event spine for a 21MB response', types, [ + 'start', + 'progress:request', + 'result', + 'done', + ]); + check('(a) `info` events', types.filter((t) => t === 'info').length, 0); + check( + '(a) `drift` findings', + types.filter((t) => t === 'drift').length, + 0, + ); + note( + '(a) → nothing. Four events, the same four a 40-byte response produces', + 'no size threshold exists on the buffered path, so nothing can warn you as you approach one', + ); + } + + // ── (b) the one cap there is, pointed at the buffered path ──────────────────────────────── + // `stream.buffer.chars` is a top-level `StitchConfig` slot. It compiles on a plain `stitch()`, + // it survives `compose`, and `runOnce` never reads it — only the streaming decoders do + // (`stream.ts:77`, `json-stream.ts:89-97`, `line-reader.ts:38-46`). + { + const wire = singleArray(50_000); + const exportAll = stitch({ + url: URL, + adapter: bufferingAdapter(wire), + stream: { buffer: { chars: 1_000 } }, // 1,000 characters. The body is 11 million. + }); + const rows = (await exportAll()) as unknown[]; + check( + '(b) a 1,000-character cap on an 11MB body — rows delivered', + rows.length, + 50_000, + ); + note( + '(b) → accepted, type-checked, and inert', + 'the only knob in the library with the word `buffer` in it does nothing on the path where buffering actually happens. It is a decoder guard wearing a general-sounding name', + ); + } + + // ── (c) the only signal there is ────────────────────────────────────────────────────────── + // Two child processes, identical `--max-old-space-size`, identical rows, identical bytes. The + // only difference is the config. + { + const died = probeRun({ + mode: 'buffered', + rows: ROWS, + nodeArgs: [`--max-old-space-size=${String(HEAP_MB)}`], + }); + check( + `(c) \`await stitch()\` on ${String(ROWS)} rows under a ${String(HEAP_MB)}MB heap — measurement printed?`, + died.measurement !== undefined, + false, + ); + check('(c) V8 aborted on the heap limit?', died.heapOom, true); + check('(c) did it exit cleanly?', died.status === 0, false); + note( + '(c) exit status', + `${String(died.status)} — SIGABRT (128+6), not a code the library chose`, + ); + const fatal = died.stderr + .split('\n') + .find( + (l) => + l.includes('FATAL ERROR') || + l.includes('heap out of memory'), + ); + note( + '(c) what the operator sees', + (fatal ?? '(no line matched)').trim().slice(0, 140), + ); + note( + '(c) → the signal is the process dying', + 'no status code from the library, no error to catch, no `error` event: V8 aborts the whole process, so a `try`/`catch` around the `await` never runs and neither does a `finally`', + ); + + const lived = probeRun({ + mode: 'assembled-ndjson', + rows: ROWS, + nodeArgs: [`--max-old-space-size=${String(HEAP_MB)}`], + }); + const m = lived.measurement; + check( + `(c) the SAME ${String(ROWS)} rows, batched over ndjson, same ${String(HEAP_MB)}MB heap — survived?`, + m !== undefined && m.ok, + true, + ); + if (m !== undefined && m.ok) { + note( + '(c) …at', + `${mb(m.peakLive)} retained for ${mb(m.wireBytes)} of wire, ${String(m.records)} rows processed`, + ); + checkAtMost( + '(c) peak retained heap', + m.peakLive, + 8 * 1024 * 1024, + mb, + ); + } + note( + '(c) → same data, same ceiling, opposite outcome', + 'which is the whole scenario: the dangerous path is the DEFAULT one, and the difference between them is invisible until the day the catalog grows', + ); + } + + finish( + 'C7', + 'NOTHING intervenes, and OOM is the only signal. A 21MB buffered response emits exactly `start`, `progress:request`, `result`, `done` — the same four events a 40-byte one emits, with no `info`, no drift finding, no threshold anywhere. The one config slot with `buffer` in its name is accepted on a buffered stitch and does nothing: `stream: { buffer: { chars: 1_000 } }` type-checked, composed, and delivered all 50,000 rows of an 11-million-character body, because only the streaming decoders ever read it. And when the wall arrives it is V8’s, not the library’s: 400,000 rows under a 96MB heap killed the process with `FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory` and exit 134 (SIGABRT) — no catchable error, no `error` event, and no `finally`. The same 400,000 rows and 85.8MB of wire, same 96MB ceiling, batched over `ndjson`: 1.3MB retained and every row processed', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/c8-assembled.ts b/docs/scenarios/proofs/large-response-memory/c8-assembled.ts new file mode 100644 index 00000000..ad4c39d7 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/c8-assembled.ts @@ -0,0 +1,223 @@ +// C8 — assemble the best available answer: stream, validate per record, batch the consumer. Report +// the seam(s), the line count, and peak heap against the C1 baseline. +// +// There is exactly ONE seam, and it is `Surface.stream`. Not because the decoding needs replacing — +// `decode: 'ndjson'` is already O(1) (C2a) — but because the engine retains every value that seam +// yields (engine.ts:1443), so the only way to bound the run is to yield something small. A hook that +// consumes rows and emits one receipt per batch is that. +// +// The claim also prices what the seam costs you, because two things stop working when you take it: +// `output` no longer describes a row, and the awaited result is no longer the rows. +// +// pnpm exec tsx docs/scenarios/proofs/large-response-memory/c8-assembled.ts +import { stitch } from '../../../../packages/core/src/index'; +import { stream } from '../../../../packages/core/src/stream'; +import { + type BatchReceipt, + batchedSurface, + drainBatched, +} from './batched-export'; +import { ndjson, streamingAdapter } from './fake-export'; +import { handRolledExport } from './hand-rolled'; +import { + check, + checkAtMost, + checkFlat, + checkSeq, + finish, + heading, + mb, + note, + x, +} from './harness'; +import { SCALES, probeOk, series } from './run-probe'; +import { countingValidator } from './validator-spy'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const URL = 'https://api.vendor.example/v1/products/export'; +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** Counted lines: no blanks, no comment-only lines. The same rule every scenario in this set uses. */ +function countLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +async function main(): Promise { + heading('C8 — the assembled answer: one seam, and what it costs'); + + // ── (a) it works, and the receipts are the progress bar ─────────────────────────────────── + { + let sunk = 0; + const receipts: BatchReceipt[] = []; + const spy = countingValidator(); + const wire = ndjson(1_250); + const exportAll = stitch({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + kind: batchedSurface({ + batch: 500, + validate: spy.validate, + onBatch: (rows) => { + sunk += rows.length; + }, + }), + }); + const done = await drainBatched(exportAll.stream(), (r) => + receipts.push(r), + ); + check('(a) rows through the sink', sunk, 1_250); + check('(a) rows the receipts report', done.rows, 1_250); + check('(a) per-record contract calls', spy.calls(), 1_250); + check('(a) largest array the contract saw', spy.sawArrayOfLength(), 0); + checkSeq( + '(a) receipts (batch, rows, running total)', + receipts.map((r) => [r.batch, r.rows, r.total]), + [ + [1, 500, 500], + [2, 500, 1_000], + [3, 250, 1_250], + ], + ); + note( + '(a) → three deltas for 1,250 rows', + 'which is exactly the point: `chunks` now holds 3 small objects instead of 1,250 product rows', + ); + } + + // ── (b) THE FOOTGUN: `stream({ kind })` silently drops your surface ─────────────────────── + // `stream()` spreads your config and then writes `kind: streamSurface` over it (stream.ts:143-146 + // — `sse()` does the same at sse.ts:210-212). So the assembled answer must be spelled + // `stitch({ kind })`. Nothing warns; the deltas just quietly go back to being rows. + { + let sunk = 0; + const wire = ndjson(1_000); + const wrong = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: { decode: 'ndjson' }, + kind: batchedSurface({ + batch: 500, + onBatch: (rows) => { + sunk += rows.length; + }, + }), + }); + const done = await drainBatched(wrong.stream()); + check('(b) rows the custom surface actually processed', sunk, 0); + check('(b) deltas the consumer received', done.receipts, 1_000); + check('(b) errors', done.error ?? 'none', 'none'); + note( + '(b) → the surface was ignored and nothing said so', + '`stream({ kind })` type-checks, runs, and hands back 1,000 raw rows. The one spelling that undoes the fix is the one the surface helper invites', + ); + } + + // ── (c) the number: peak heap against the C1 baseline ───────────────────────────────────── + const assembled = series('assembled-ndjson'); + for (const [i, m] of assembled.entries()) + note( + `(c) assembled, ${String(SCALES[i])} rows / ${mb(m.wireBytes)} wire`, + `${mb(m.peakLive)} retained = ${x(m.ratio)} wire`, + ); + const [a1, , a100] = assembled as [ + (typeof assembled)[0], + (typeof assembled)[0], + (typeof assembled)[0], + ]; + checkFlat('(c) 1x -> 100x assembled heap', a1.peakLive, a100.peakLive); + const baseline = probeOk({ mode: 'buffered', rows: 100_000 }); + note( + '(c) C1 baseline at 100k rows (`await stitch()`)', + mb(baseline.peakLive), + ); + note('(c) assembled at 100k rows', mb(a100.peakLive)); + checkAtMost( + '(c) assembled ÷ baseline', + a100.peakLive / baseline.peakLive, + 0.1, + x, + ); + note( + '(c) → 40x less heap, and FLAT rather than merely smaller', + 'the baseline grows with the catalog and this does not, which is the difference between a number and a guarantee', + ); + + // ── (d) the same seam over `decode: 'json'` — most of the win is gone ───────────────────── + // Because C3's array buffer is upstream of the seam. The batching surface fixes the ENGINE's + // accumulator; it cannot fix the decoder holding the array text. + const overJson = probeOk({ + mode: 'assembled-json', + rows: 100_000, + buffer: 1_000_000_000, + }); + note( + '(d) same surface, same rows, one top-level array instead of ndjson', + `${mb(overJson.peakLive)} retained = ${x(overJson.ratio)} wire — against ${mb(a100.peakLive)} over ndjson`, + ); + checkAtMost( + '(d) how much of the buffered baseline it still costs', + overJson.peakLive / baseline.peakLive, + 0.6, + x, + ); + note( + '(d) → the assembled answer is only as bounded as its WIRE FORMAT', + 'over one giant array the seam removes the engine’s 29MB and leaves the decoder’s 19MB, and on default settings the call would not have finished at all (C3d)', + ); + + // ── (e) the price, in lines ─────────────────────────────────────────────────────────────── + const seam = countLines('batched-export.ts'); + const rolled = countLines('hand-rolled.ts'); + note( + '(e) `batched-export.ts` — the surface + its drain helper', + `${String(seam)} lines`, + ); + note( + '(e) `hand-rolled.ts` — the same feature set, no library', + `${String(rolled)} lines`, + ); + note( + '(e) config on top of the seam', + '4 lines: `url`, `adapter`, `stream: { decode }`, `kind`', + ); + { + // The hand-rolled twin, run for real, so the line count is a comparison and not a claim. + let sunk = 0; + const spy = countingValidator(); + const wire = ndjson(1_250); + const total = await handRolledExport(wire.body, { + batch: 500, + validate: spy.validate, + onBatch: (rows) => { + sunk += rows.length; + }, + }); + check('(e) hand-rolled rows processed', total, 1_250); + check('(e) hand-rolled sink', sunk, 1_250); + check('(e) hand-rolled per-record contract calls', spy.calls(), 1_250); + } + note( + '(e) → the seam is not cheaper than the hand-rolled version, and that is fine', + 'what the 4 config lines buy is the rest of the stack — `auth` on the open, `retry` on the connect, `throttle` charged per open, `timeout.total`, `trace`, `verdict.accept` — none of which the hand-rolled reader has and all of which keep working through a custom surface', + ); + + finish( + 'C8', + 'ACHIEVABLE, with ONE seam and a wire format you may not be offered. The seam is `Surface.stream` — not to replace the decoding (`decode: "ndjson"` is already O(1)) but because the engine retains everything that hook yields, so the fix is to yield one small receipt per batch instead of one row per row. Measured over `ndjson`: 1.3MB retained for 100,000 rows against the C1 baseline’s 53.8MB, a 40x cut, and FLAT — 1.3MB at 1,000 rows and 1.4MB at 100,000. It cost 75 counted lines of surface plus 4 lines of config, against 67 lines hand-rolled with no library at all; the lines are a wash and what the config buys is the resilience stack around the open. Two prices are real. Over a single top-level JSON ARRAY the same seam still costs 19.0MB, because C3’s array buffer sits UPSTREAM of it. And `stream({ kind })` silently drops the surface — `stream()` overwrites `kind` after spreading your config (stream.ts:143-146) — so the answer only works spelled `stitch({ kind })`, with no warning if you get it wrong', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/fake-export.ts b/docs/scenarios/proofs/large-response-memory/fake-export.ts new file mode 100644 index 00000000..e7919371 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/fake-export.ts @@ -0,0 +1,399 @@ +// The fake vendor: a product-catalog export endpoint, offline and in-memory. +// +// Everything here is LAZY. The rows are generated inside the stream's `pull()`, one ~16 KB chunk at +// a time, so the fixture itself never exists as a whole — otherwise the fixture would dominate every +// heap measurement and the numbers would be about this file rather than about the library. A +// `pull`-driven `ReadableStream` also models a real socket: its internal queue holds about one chunk, +// so a slow reader really does slow the producer. `eagerArray` is the deliberate opposite (C6). +// +// Two wire formats, same rows: +// - `singleArray` — ONE top-level JSON array: `[{…},{…},…]`. The hard case. No newlines between +// records, so nothing can split it on `\n`. +// - `ndjson` — one record per line. The easy case, and the control for the measurement. +// +// `wire()` reports the exact byte count the producer emitted, so every ratio in this directory is +// measured against real bytes rather than an estimate. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +const enc = new TextEncoder(); + +/** One product row — the shape the incident was about (a catalog export). ~200 bytes of JSON. */ +export interface ProductRow { + id: string; + sku: string; + title: string; + price_cents: number; + currency: string; + in_stock: boolean; + tags: string[]; + updated_at: string; +} + +const pad = (n: number): string => String(n).padStart(7, '0'); + +export function productRow(i: number): ProductRow { + return { + id: `prd_${pad(i)}`, + sku: `SKU-${pad(i)}-A`, + title: `Refurbished Widget Assembly, Model ${pad(i)}`, + price_cents: 1999 + (i % 5000), + currency: 'usd', + in_stock: i % 7 !== 0, + tags: ['catalog', 'widget', `bin-${String(i % 40)}`], + updated_at: `2026-0${String((i % 9) + 1)}-1${String(i % 10)}T04:00:00.000Z`, + }; +} + +/** Rows per emitted chunk, sized so a chunk lands near a real socket read (~16 KB). */ +const ROWS_PER_CHUNK = 64; + +/** A live response body plus the exact byte count it wrote. */ +export interface Wire { + body: ReadableStream; + /** Bytes emitted so far (final once the stream has closed). */ + wire: () => number; + /** Chunks the producer has enqueued so far — the backpressure observable (C6). */ + enqueued: () => number; + /** + * Sample from the PRODUCER, once per emitted chunk. This is the one seam every mode shares — + * buffered, streamed, awaited, batched — so installing the sampler here (rather than in each + * consumer) is what makes the modes comparable. A consumer-side sampler would give `.stream()` + * hundreds of chances to catch a peak and `await` exactly none, and the resulting "await uses + * less heap" would be an artefact of the instrument. + */ + watch: (s: WireSampler, marks: number) => void; +} + +/** The two sampling operations a {@link Wire} drives. Structurally the `Sampler` from `mem.ts`. */ +export interface WireSampler { + tick: () => void; + mark: () => number; +} + +function lazyWire( + chunkFor: (start: number, end: number) => string, + open: string, + close: string, + rows: number, +): Wire { + let bytes = 0; + let chunks = 0; + let next = 0; + let opened = false; + let watcher: { s: WireSampler; every: number } | undefined; + const emit = ( + controller: ReadableStreamDefaultController, + s: string, + ): void => { + const u = enc.encode(s); + bytes += u.byteLength; + chunks++; + controller.enqueue(u); + if (watcher) { + watcher.s.tick(); + if (chunks % watcher.every === 0) watcher.s.mark(); + } + }; + const body = new ReadableStream({ + pull(controller) { + if (!opened) { + opened = true; + if (open !== '') { + emit(controller, open); + return; + } + } + if (next >= rows) { + if (close !== '') emit(controller, close); + controller.close(); + return; + } + const end = Math.min(next + ROWS_PER_CHUNK, rows); + emit(controller, chunkFor(next, end)); + next = end; + }, + }); + return { + body, + wire: () => bytes, + enqueued: () => chunks, + watch: (s, marks) => { + watcher = { + s, + every: Math.max(1, Math.ceil(rows / ROWS_PER_CHUNK / marks)), + }; + }, + }; +} + +/** + * ONE top-level JSON array, streamed element by element: `[{…},{…},…]`. The shape the scenario is + * about — a `\n` split cannot recover the records, only a structural parser can. + */ +export function singleArray( + rows: number, + row: (i: number) => unknown = productRow, +): Wire { + return lazyWire( + (start, end) => { + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push((i === 0 ? '' : ',') + JSON.stringify(row(i))); + return parts.join(''); + }, + '[', + ']', + rows, + ); +} + +/** + * CONCATENATED top-level values with no separator and no newline: `{…}{…}{…}`. The `'json'` decoder + * advertises this shape alongside the single array, and it is the control that localises C3's + * finding: same decoder, same records, same bytes — the only difference is whether they sit inside + * one top-level array or stand as siblings. + */ +export function concatObjects(rows: number): Wire { + return lazyWire( + (start, end) => { + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push(JSON.stringify(productRow(i))); + return parts.join(''); + }, + '', + '', + rows, + ); +} + +/** Newline-delimited JSON — one record per line. The easy case, and the measurement's control. */ +export function ndjson( + rows: number, + row: (i: number) => unknown = productRow, +): Wire { + return lazyWire( + (start, end) => { + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push(`${JSON.stringify(row(i))}\n`); + return parts.join(''); + }, + '', + '', + rows, + ); +} + +/** + * The same single array, but every byte is enqueued at once instead of on demand — a producer that + * ignores the reader's backpressure signal entirely. The whole body sits in the stream's internal + * queue before the consumer reads its second chunk (C6). + */ +export function eagerArray(rows: number): Wire { + let bytes = 0; + let chunks = 0; + let dumped = false; + let watcher: { s: WireSampler; every: number } | undefined; + const body = new ReadableStream({ + // On the FIRST pull, dump the entire body into the stream's internal queue without ever + // consulting `desiredSize`. That is what "ignores backpressure" means for a `ReadableStream` + // source, and doing it on first pull rather than in `start()` keeps the allocation inside the + // measured window instead of before the baseline was taken. + pull(controller) { + if (dumped) { + controller.close(); + return; + } + dumped = true; + const emit = (s: string): void => { + const u = enc.encode(s); + bytes += u.byteLength; + chunks++; + controller.enqueue(u); + if (watcher) { + watcher.s.tick(); + if (chunks % watcher.every === 0) watcher.s.mark(); + } + }; + emit('['); + for (let start = 0; start < rows; start += ROWS_PER_CHUNK) { + const end = Math.min(start + ROWS_PER_CHUNK, rows); + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push( + (i === 0 ? '' : ',') + JSON.stringify(productRow(i)), + ); + emit(parts.join('')); + } + emit(']'); + controller.close(); + }, + }); + return { + body, + wire: () => bytes, + enqueued: () => chunks, + watch: (s, marks) => { + watcher = { + s, + every: Math.max(1, Math.ceil(rows / ROWS_PER_CHUNK / marks)), + }; + }, + }; +} + +/** A body that never closes its first element — the `stream.buffer.chars` cap's target case (C6). */ +export function neverClosingArray(rows: number): Wire { + // A nested array opened as element 0 and never closed: every subsequent record lands inside one + // un-terminated value, which is precisely what the cap exists to stop. + return lazyWire( + (start, end) => { + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push(`${JSON.stringify(productRow(i))},`); + return parts.join(''); + }, + '[[', + '', + rows, + ); +} + +/** A body with no `\n` at all — the `'lines'`/`'ndjson'` cap's target case (C6). */ +export function noNewlines(rows: number): Wire { + return lazyWire( + (start, end) => { + const parts: string[] = []; + for (let i = start; i < end; i++) + parts.push(JSON.stringify(productRow(i))); + return parts.join(' '); + }, + '', + '', + rows, + ); +} + +/** An adapter that hands back the live body. Mirrors `fetchAdapter` when `req.stream` is set. */ +export function streamingAdapter(wire: Wire, status = 200): Adapter { + return (req: AdapterRequest): Promise => { + if (!req.stream) + return Promise.reject(new Error('expected req.stream to be set')); + return Promise.resolve({ status, headers: {}, body: wire.body }); + }; +} + +/** Sampling hooks a buffering transport offers, so the peak can be measured where it happens. */ +export interface BufferHooks { + /** After every socket read — a cheap sample. */ + onChunk?: () => void; + /** At the two instants that matter: whole text live, then text AND parsed tree live. */ + onPeak?: () => void; +} + +/** + * An adapter that BUFFERS, byte for byte the way `fetchAdapter` does on the non-streaming path + * (`http-adapter.ts:133-138`): read the body to one string, then `JSON.parse` it. + * + * The chunks are collected and `join`ed once rather than `+=`'d, which is the cheaper of the two and + * the closer match to what `response.text()` does — the honest choice, since the point here is to + * measure the library's floor rather than to inflate it. + * + * `onPeak` fires at the two instants the caller cannot otherwise see: with the whole wire text live + * and nothing parsed, and with the text AND the object tree both live. The second is the buffered + * path's true high-water mark, and by the time `await stitch()` resolves it has already passed. + */ +export function bufferingAdapter(wire: Wire, hooks: BufferHooks = {}): Adapter { + return async (req: AdapterRequest): Promise => { + if (req.stream) + return Promise.reject(new Error('did not expect req.stream')); + const reader = wire.body.getReader(); + const decoder = new TextDecoder(); + let parts: string[] = []; + for (;;) { + const r = await reader.read(); + if (r.done) break; + parts.push(decoder.decode(r.value, { stream: true })); + hooks.onChunk?.(); + } + parts.push(decoder.decode()); + const text = parts.join(''); + parts = []; // release the pieces; only the flat string survives + hooks.onPeak?.(); // the whole wire text is live and nothing is parsed yet + const parsed: unknown = JSON.parse(text); + hooks.onPeak?.(); // text AND tree are both live — the buffered path's true peak + return { + status: 200, + headers: { 'content-type': 'application/json' }, + body: parsed, + }; + }; +} + +// ---- correctness fixtures (C3) ------------------------------------------------------------- +// The structural decoder's claim is not only "flat heap" but "right boundaries". These are the +// adversarial bodies: things a `\n` split or a naive `},{` split gets wrong. + +/** A pretty-printed array — every record spans multiple LINES. `'ndjson'` cannot read this. */ +export const PRETTY_ARRAY = `[ + { + "id": "a", + "note": "line one" + }, + { + "id": "b", + "nested": { "deep": [1, 2, { "x": "}" }] } + } +]`; + +/** Records whose STRING VALUES contain the structural characters and escaped quotes. */ +export const HOSTILE_ARRAY = + '[' + + JSON.stringify({ id: 1, s: 'has , comma and ] bracket and } brace' }) + + ',' + + JSON.stringify({ id: 2, s: 'embedded \n newline and "quotes"' }) + + ',' + + JSON.stringify({ id: 3, s: 'unicode 
 and escape \\" tail' }) + + ',' + + JSON.stringify({ + id: 4, + nested: [ + [1, 2], + [3, [4, 5]], + ], + obj: { a: { b: { c: [] } } }, + }) + + ']'; + +/** A stream that emits `text` split at EVERY `at` boundary, to place chunk splits mid-token. */ +export function splitStream(text: string, at: number): Wire { + const pieces: string[] = []; + for (let i = 0; i < text.length; i += at) + pieces.push(text.slice(i, i + at)); + let i = 0; + let bytes = 0; + let chunks = 0; + const body = new ReadableStream({ + pull(controller) { + if (i >= pieces.length) { + controller.close(); + return; + } + const u = enc.encode(pieces[i++] as string); + bytes += u.byteLength; + chunks++; + controller.enqueue(u); + }, + }); + return { + body, + wire: () => bytes, + enqueued: () => chunks, + watch: () => undefined, // correctness fixture — nothing to sample + }; +} diff --git a/docs/scenarios/proofs/large-response-memory/hand-rolled.ts b/docs/scenarios/proofs/large-response-memory/hand-rolled.ts new file mode 100644 index 00000000..8218b080 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/hand-rolled.ts @@ -0,0 +1,77 @@ +// The same feature set with no library at all — the baseline C8 prices against. +// +// Deliberately scoped to what the assembled StitchAPI answer actually delivers, no more: read an +// NDJSON body off a `ReadableStream`, carry a partial line across chunk boundaries, cap the carry so +// a body with no newline cannot grow memory without limit, validate every record, batch, and report +// progress per batch. No retry, no auth, no throttle, no timeout, no trace — those are the rows the +// library wins and C8 says so rather than pretending the comparison is like for like. +import type { Validator } from '../../../../packages/core/src/validator'; +import type { BatchReceipt } from './batched-export'; + +export interface HandRolledOptions { + batch: number; + onBatch: (rows: unknown[]) => void | Promise; + onReceipt?: (r: BatchReceipt) => void; + validate?: Validator['validate']; + /** Cap on one un-terminated line, mirroring `stream.buffer.chars`. */ + maxLineChars?: number; +} + +export async function handRolledExport( + body: ReadableStream, + opts: HandRolledOptions, +): Promise { + const reader = body.getReader(); + const decoder = new TextDecoder(); + const cap = opts.maxLineChars ?? 8 * 1024 * 1024; + let carry = ''; + let buf: unknown[] = []; + let total = 0; + let batch = 0; + const take = async (line: string): Promise => { + if (line.trim() === '') return; + const row: unknown = JSON.parse(line); + if (opts.validate) { + const r = await opts.validate(row); + if (!r.ok) + throw new Error( + `row ${String(total + buf.length)}: ${r.issues[0]?.message ?? 'invalid'}`, + ); + } + buf.push(row); + if (buf.length < opts.batch) return; + await opts.onBatch(buf); + total += buf.length; + batch++; + opts.onReceipt?.({ batch, rows: buf.length, total }); + buf = []; + }; + try { + for (;;) { + const r = await reader.read(); + if (r.done) break; + carry += decoder.decode(r.value, { stream: true }); + let nl = carry.indexOf('\n'); + while (nl >= 0) { + await take(carry.slice(0, nl)); + carry = carry.slice(nl + 1); + nl = carry.indexOf('\n'); + } + if (carry.length > cap) + throw new Error('un-terminated line exceeded the cap'); + } + carry += decoder.decode(); + for (const line of carry.split('\n')) await take(line); + if (buf.length > 0) { + await opts.onBatch(buf); + total += buf.length; + batch++; + opts.onReceipt?.({ batch, rows: buf.length, total }); + buf = []; + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + return total; +} diff --git a/docs/scenarios/proofs/large-response-memory/harness.ts b/docs/scenarios/proofs/large-response-memory/harness.ts new file mode 100644 index 00000000..85acd39b --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/harness.ts @@ -0,0 +1,160 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is a MEASURED NUMBER, and heap numbers are noisy. So the assertions here +// are deliberately loose where the physics is loose and tight where it is not: +// +// - `check` / `checkSeq` — exact equality, for the CORRECTNESS half (element counts, boundaries, +// event spines). Nothing noisy about those. +// - `checkFlat` — "peak heap did NOT grow with N". Asserts the 100x measurement is within a +// tolerance of the 1x one. The flat-vs-linear shape is the robust signal, not any single figure. +// - `checkLinear` — the opposite claim: peak heap DID grow roughly with N. Asserts growth exceeds +// a floor, so a merely-noisy measurement cannot pass it. +// - `checkAtMost` / `checkAtLeast` — one-sided bounds, for ratios where only the direction matters. +// +// Every one of them prints the measured number whether it passes or fails, because the number is +// the finding. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v}n`; + if (typeof v === 'number' && Number.isNaN(v)) return 'NaN'; + return JSON.stringify(v) ?? String(v); +} + +/** Bytes as MB, 1 decimal — the unit every heap number in this directory is reported in. */ +export function mb(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(1)}MB`; +} + +/** A ratio as `12.4x`. */ +export function x(ratio: number): string { + return `${ratio.toFixed(1)}x`; +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** Assert a measured SEQUENCE matches, element-wise via `JSON.stringify`. Prints it in full. */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** One-sided bound: the measured number must not exceed `limit`. */ +export function checkAtMost( + label: string, + actual: number, + limit: number, + render: (n: number) => string = show, +): void { + checks++; + const ok = actual <= limit; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${render(actual)} (limit ${render(limit)})`, + ); +} + +/** One-sided bound: the measured number must be at least `floor`. */ +export function checkAtLeast( + label: string, + actual: number, + floor: number, + render: (n: number) => string = show, +): void { + checks++; + const ok = actual >= floor; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${render(actual)} (floor ${render(floor)})`, + ); +} + +/** + * FLAT: `big` (the 100x workload) used no more than `tolerance`x the heap of `small` (the 1x one). + * This is the control assertion — a decoder whose working set is one record is flat by construction, + * so the tolerance can be tight (2x) and still never flake. `+1` guards a divide-by-zero when a + * measurement lands at 0 bytes of retained heap, which happens for the truly O(1) modes. + */ +export function checkFlat( + label: string, + small: number, + big: number, + tolerance = 2, +): void { + checks++; + const growth = (big + 1) / (small + 1); + const ok = growth <= tolerance; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: ${mb(small)} -> ${mb(big)} = ${x(growth)} growth (flat if <= ${x(tolerance)})`, + ); +} + +/** + * LINEAR: `big` used at least `floor`x the heap of `small`. The workloads differ by 100x, so a floor + * of 10x is far outside anything measurement noise produces — this cannot pass by accident. + */ +export function checkLinear( + label: string, + small: number, + big: number, + floor = 10, +): void { + checks++; + const growth = (big + 1) / (small + 1); + const ok = growth >= floor; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: ${mb(small)} -> ${mb(big)} = ${x(growth)} growth (linear if >= ${x(floor)})`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = + value === '' + ? '' + : `: ${typeof value === 'string' ? value : show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/large-response-memory/mem.ts b/docs/scenarios/proofs/large-response-memory/mem.ts new file mode 100644 index 00000000..cc4df7a4 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/mem.ts @@ -0,0 +1,160 @@ +// The measurement kit. Heap numbers are the whole evidence in this scenario, so the methodology has +// to be stated in code, not just in prose. +// +// TWO numbers are recorded per run, and they answer different questions: +// +// 1. `peakHeap` — the high-water mark of `process.memoryUsage().heapUsed`, sampled. This is what +// the OS-facing pressure looks like: it includes floating garbage the collector had not got to +// yet. It is REAL (an allocation rate the GC cannot keep up with is exactly how a process dies) +// but it is NOISY — it depends on when V8 chose to collect. +// +// 2. `peakLive` — the high-water mark of `heapUsed` sampled IMMEDIATELY AFTER a forced full GC. +// This is the RETAINED working set: bytes that are still reachable and therefore cannot be +// collected under pressure. It is nearly noise-free and it is the number that decides whether a +// workload scales. **Every verdict in this directory is based on `peakLive`**; `peakHeap` is +// reported alongside as context. +// +// A forced GC costs real time on a large heap, so `mark()` (the GC-ing sample) is called ~10 times +// per run at workload seams, while `tick()` (the cheap sample) is called densely. `tick` alone would +// not be trustworthy: a `setInterval` sampler cannot preempt a synchronous `JSON.parse`, and an +// in-memory `ReadableStream` resolves its reads on the MICROtask queue, which starves timers +// entirely. So the workload calls `tick()`/`mark()` itself at points it knows are seams, and the +// interval sampler is a backstop rather than the mechanism. +// +// Requires `--expose-gc`. The scripts FAIL LOUDLY without it rather than silently reporting the +// noisy number as if it were the clean one. + +/** Fail loudly when the process was not started with `--expose-gc`. */ +export function requireGc(): () => void { + const g = (globalThis as { gc?: () => void }).gc; + if (typeof g !== 'function') { + console.error( + 'FAIL — this script measures heap and needs a forced GC between phases.\n' + + ' Re-run with --expose-gc, e.g.\n' + + ' pnpm exec tsx --expose-gc ', + ); + process.exit(2); + } + return g; +} + +/** What the workload uses to sample itself. */ +export interface Sampler { + /** Cheap `heapUsed` sample — call densely (every N records). No GC. */ + tick(): void; + /** + * Forced-GC sample: collect, then read `heapUsed`. Returns the RETAINED bytes above baseline at + * this instant. Call at workload seams (~10 per run) — it is not cheap. + */ + mark(): number; +} + +export interface Measurement { + /** Bytes the fake vendor actually wrote to the wire. */ + wireBytes: number; + /** Records the workload observed (deltas, array elements, rows — mode-dependent). */ + records: number; + /** `heapUsed` after a full GC, before the workload started. */ + baseline: number; + /** High-water `heapUsed` above baseline, INCLUDING floating garbage. Noisy. */ + peakHeap: number; + /** High-water POST-GC `heapUsed` above baseline — the retained working set. The robust number. */ + peakLive: number; + /** + * High-water `arrayBuffers` above its own baseline. `heapUsed` counts the V8 heap ONLY, and a + * `Uint8Array` off the socket lives in an external backing store — so a body sitting unread in a + * `ReadableStream`'s internal queue is INVISIBLE to `heapUsed`. C6 turns on this distinction, so + * it is measured rather than assumed. + */ + peakBuffers: number; + /** Retained bytes above baseline after the workload finished and its result went out of scope. */ + settled: number; + /** `peakLive / wireBytes` — the multiplier the scenario is about. */ + ratio: number; + /** How many cheap samples and forced-GC samples were taken. */ + ticks: number; + marks: number; + /** Wall-clock ms. */ + ms: number; +} + +/** + * Run `fn` under measurement. `fn` receives a {@link Sampler} and must return the number of records + * it observed; `wire()` is read afterwards for the byte count the producer emitted. + * + * The result of `fn` is deliberately NOT returned — holding on to it would keep the workload's data + * alive past the final GC and corrupt `settled`. + */ +export async function measure( + fn: (s: Sampler) => Promise, + wire: () => number, +): Promise { + const gc = requireGc(); + + gc(); + gc(); // a second pass collects what the first one's finalizers freed + const base = process.memoryUsage(); + const baseline = base.heapUsed; + const baseBuffers = base.arrayBuffers; + + let peakHeap = 0; + let peakLive = 0; + let peakBuffers = 0; + let ticks = 0; + let marks = 0; + + /** One reading of both spaces. Returns the heap delta; records the buffer high-water too. */ + const read = (): number => { + const m = process.memoryUsage(); + const b = m.arrayBuffers - baseBuffers; + if (b > peakBuffers) peakBuffers = b; + return m.heapUsed - baseline; + }; + const tick = (): void => { + ticks++; + const d = read(); + if (d > peakHeap) peakHeap = d; + }; + const mark = (): number => { + marks++; + gc(); + const d = read(); + if (d > peakLive) peakLive = d; + if (d > peakHeap) peakHeap = d; + return d; + }; + + // Backstop sampler. Only fires when the workload yields to the timer phase; the workload's own + // `tick()` calls are what actually carry the measurement (see the header note). + const timer = setInterval(tick, 1); + timer.unref?.(); + + const t0 = Date.now(); + let records: number; + try { + records = await fn({ tick, mark }); + mark(); // the mandatory final live sample, taken while the workload's result is still in scope + } finally { + clearInterval(timer); + } + const ms = Date.now() - t0; + + gc(); + gc(); + const settled = process.memoryUsage().heapUsed - baseline; + const wireBytes = wire(); + + return { + wireBytes, + records, + baseline, + peakHeap, + peakLive, + peakBuffers, + settled, + ratio: wireBytes === 0 ? 0 : peakLive / wireBytes, + ticks, + marks, + ms, + }; +} diff --git a/docs/scenarios/proofs/large-response-memory/probe.ts b/docs/scenarios/proofs/large-response-memory/probe.ts new file mode 100644 index 00000000..19956f25 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/probe.ts @@ -0,0 +1,297 @@ +// One measurement, one process. `probe.ts --mode= --rows= [--buffer=]` runs exactly one +// workload against the fake export endpoint and prints ONE line of JSON to stdout: the +// {@link Measurement}. +// +// Why a separate process per measurement: V8's heap is a shared, stateful thing. Running the +// buffered baseline and the streaming control in the same process means the second one inherits the +// first one's fragmented old space, its grown heap limit, and whatever the collector had not yet +// released. Forking gives every number a clean start, and it is the difference between a result you +// can quote and a result you have to apologise for. The claim scripts (`c1`…`c8`) spawn this file. +// +// Why the sampler lives on the PRODUCER: see `Wire.watch`. Every mode here is sampled the same way, +// once per emitted wire chunk, so `.stream()` and `await` get exactly the same number of chances to +// observe a peak. The buffered modes take two EXTRA marks — with the whole response text live, and +// with the text and the parsed tree both live — because that path's high-water mark happens after +// the last byte arrives and would otherwise be invisible. That asymmetry is in the measurement's +// favour for the streaming modes, not against them. +// +// Run it directly to see a single number: +// pnpm exec tsx --expose-gc docs/scenarios/proofs/large-response-memory/probe.ts --mode=buffered --rows=100000 +import { stitch } from '../../../../packages/core/src/index'; +import { stream, streamSurface } from '../../../../packages/core/src/stream'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + ResolvedStitchConfig, + StitchEvent, +} from '../../../../packages/core/src/types'; +import { batchedSurface, drainBatched } from './batched-export'; +import { + bufferingAdapter, + concatObjects, + eagerArray, + ndjson, + singleArray, + streamingAdapter, +} from './fake-export'; +import { type Measurement, type Sampler, measure } from './mem'; +import { countingValidator } from './validator-spy'; + +const URL = 'https://api.vendor.example/v1/products/export'; + +/** Forced-GC marks per run — the retained-heap curve, at a cost the run can absorb. */ +const MARKS = 12; +/** Rows per batch in the assembled answer (C8). */ +export const BATCH = 500; + +type Mode = + // the buffered baseline and its no-library floor + | 'buffered' + | 'parse-only' + // the decoders in isolation: the engine is not in the picture + | 'decoder-json' + | 'decoder-ndjson' + | 'decoder-concat' + // the decoders through the engine, drained via `.stream()` + | 'stream-json' + | 'stream-ndjson' + // the same, awaited (the collected array is the declared result) + | 'stream-json-await' + | 'stream-ndjson-await' + // an `output` contract on the streaming path + | 'stream-json-output' + | 'stream-ndjson-output' + // a producer that ignores backpressure + | 'eager-json' + // the assembled answer, over each wire format + | 'assembled-json' + | 'assembled-ndjson'; + +function arg(name: string, fallback: string): string { + const hit = process.argv.find((a) => a.startsWith(`--${name}=`)); + return hit === undefined ? fallback : hit.slice(name.length + 3); +} + +/** + * Drain a `.stream()` event spine, discarding every delta. Returns the delta count. + * + * `extra` adds CONSUMER-side sampling, and exactly one mode uses it: `eager-json`, whose producer + * front-loads the whole body before a byte is decoded, so producer-driven marks all land before the + * interesting phase. That mode is therefore not heap-comparable with the others — its claim is about + * `peakBuffers`, not `peakLive`. + */ +async function drainDeltas( + events: AsyncIterable, + extra?: { s: Sampler; every: number }, +): Promise { + let n = 0; + const errors: string[] = []; + for await (const ev of events) { + if (ev.type === 'delta') { + n++; + if (extra && n % extra.every === 0) extra.s.mark(); + } else if (ev.type === 'error') errors.push(ev.message); + } + if (errors.length > 0) throw new Error(errors.join('; ')); + return n; +} + +async function run( + mode: Mode, + rows: number, + bufferChars: number | undefined, +): Promise { + const streamCfg = (decode: 'json' | 'ndjson') => + bufferChars === undefined + ? ({ decode } as const) + : ({ decode, buffer: { chars: bufferChars } } as const); + + switch (mode) { + // ---- the buffered baseline --------------------------------------------------------- + case 'buffered': { + const wire = singleArray(rows); + return measure(async (s) => { + wire.watch(s, MARKS); + const exportAll = stitch({ + url: URL, + adapter: bufferingAdapter(wire, { onPeak: s.mark }), + }); + const products = (await exportAll()) as unknown[]; + s.mark(); // taken WHILE the parsed tree is still referenced — that is the point + return products.length; + }, wire.wire); + } + // No library at all: read the body to text, `JSON.parse`. The floor the buffered path + // cannot beat, and the thing `fetchAdapter` itself does (http-adapter.ts:133-138). + case 'parse-only': { + const wire = singleArray(rows); + return measure(async (s) => { + wire.watch(s, MARKS); + const reader = wire.body.getReader(); + const decoder = new TextDecoder(); + let parts: string[] = []; + for (;;) { + const r = await reader.read(); + if (r.done) break; + parts.push(decoder.decode(r.value, { stream: true })); + } + parts.push(decoder.decode()); + const text = parts.join(''); + parts = []; + s.mark(); // the whole wire text, live, nothing parsed + const parsed = JSON.parse(text) as unknown[]; + s.mark(); // text AND tree + return parsed.length; + }, wire.wire); + } + + // ---- the decoders in isolation (no engine) ----------------------------------------- + // `streamSurface.stream` is exactly what the engine calls; calling it directly measures the + // DECODER's working set with nothing accumulating around it. + case 'decoder-json': + case 'decoder-concat': + case 'decoder-ndjson': { + const decode = mode === 'decoder-ndjson' ? 'ndjson' : 'json'; + const wire = + mode === 'decoder-json' + ? singleArray(rows) + : mode === 'decoder-concat' + ? concatObjects(rows) + : ndjson(rows); + const hook = streamSurface.stream as NonNullable; + const cfg: ResolvedStitchConfig = { + kind: streamSurface, + stream: streamCfg(decode), + }; + return measure(async (s) => { + wire.watch(s, MARKS); + let n = 0; + for await (const row of hook( + { status: 200, headers: {}, body: wire.body }, + cfg, + )) { + void row; + n++; + } + return n; + }, wire.wire); + } + + // ---- the decoders through the engine ----------------------------------------------- + case 'stream-json': + case 'stream-ndjson': + case 'stream-json-output': + case 'stream-ndjson-output': { + const decode = mode.startsWith('stream-json') ? 'json' : 'ndjson'; + const withOutput = mode.endsWith('-output'); + const wire = decode === 'json' ? singleArray(rows) : ndjson(rows); + const base = { + url: URL, + adapter: streamingAdapter(wire), + stream: streamCfg(decode), + }; + const exportAll = withOutput + ? stream({ ...base, output: countingValidator() }) + : stream(base); + return measure(async (s) => { + wire.watch(s, MARKS); + return drainDeltas(exportAll.stream()); + }, wire.wire); + } + case 'stream-json-await': + case 'stream-ndjson-await': { + const decode = mode === 'stream-json-await' ? 'json' : 'ndjson'; + const wire = decode === 'json' ? singleArray(rows) : ndjson(rows); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: streamCfg(decode), + }); + return measure(async (s) => { + wire.watch(s, MARKS); + const products = await exportAll(); + s.mark(); // the collected array is live here + return products.length; + }, wire.wire); + } + + // ---- a producer that ignores backpressure ------------------------------------------- + // `eagerArray` enqueues the WHOLE body before the consumer reads a byte, so its own `watch` + // samples fire during `start()` — before anything is decoded. That is the point: the peak + // this mode reports is the stream's internal queue, not the decoder's. + case 'eager-json': { + const wire = eagerArray(rows); + const exportAll = stream({ + url: URL, + adapter: streamingAdapter(wire), + stream: streamCfg('json'), + }); + return measure(async (s) => { + wire.watch(s, MARKS); + return drainDeltas(exportAll.stream(), { + s, + every: Math.max(1, Math.floor(rows / MARKS)), + }); + }, wire.wire); + } + + // ---- the assembled answer ------------------------------------------------------------ + case 'assembled-json': + case 'assembled-ndjson': { + const decode = mode === 'assembled-json' ? 'json' : 'ndjson'; + const wire = decode === 'json' ? singleArray(rows) : ndjson(rows); + let sunk = 0; + const surface = batchedSurface({ + batch: BATCH, + validate: countingValidator().validate, + // Stand-in for the database insert. It must not RETAIN the rows, which is the whole + // discipline: process the batch, keep a summary, let it go. + onBatch: (batch) => { + sunk += batch.length; + }, + }); + // `stitch({ kind })`, NOT `stream({ kind })` — the `stream()` helper spreads your config + // and then overwrites `kind` with `streamSurface` (stream.ts:143-146), so a custom + // surface passed to it is silently dropped. C8(b) asserts that. + const exportAll = stitch({ + url: URL, + adapter: streamingAdapter(wire), + stream: streamCfg(decode), + kind: surface, + }); + return measure(async (s) => { + wire.watch(s, MARKS); + const done = await drainBatched(exportAll.stream()); + s.mark(); + if (done.error !== undefined) throw new Error(done.error); + if (done.rows !== sunk) + throw new Error('receipt total disagreed with the sink'); + return done.rows; + }, wire.wire); + } + } +} + +async function main(): Promise { + const mode = arg('mode', 'buffered') as Mode; + const rows = Number(arg('rows', '10000')); + const bufferArg = arg('buffer', ''); + const bufferChars = bufferArg === '' ? undefined : Number(bufferArg); + try { + const m = await run(mode, rows, bufferChars); + console.log(JSON.stringify({ mode, rows, ok: true, ...m })); + } catch (e) { + // A workload that BLEW UP is a measurement too — C3's whole finding is one of these. Report + // it as data on stdout rather than a stack trace on stderr, so the claim scripts can assert + // on the message. + console.log( + JSON.stringify({ + mode, + rows, + ok: false, + error: e instanceof Error ? e.message : String(e), + }), + ); + } +} + +void main(); diff --git a/docs/scenarios/proofs/large-response-memory/run-probe.ts b/docs/scenarios/proofs/large-response-memory/run-probe.ts new file mode 100644 index 00000000..cd815ae6 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/run-probe.ts @@ -0,0 +1,132 @@ +// Spawn `probe.ts` in a fresh process and read back its one line of JSON. +// +// The claim scripts do not measure anything themselves. They ask for measurements, one process each, +// and assert on the shapes of the curves that come back. That separation is deliberate: a claim +// script that measured in-process would be comparing numbers taken from a heap its own earlier +// measurements had already grown and fragmented. +import { spawnSync } from 'node:child_process'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const PROBE = join(HERE, 'probe.ts'); +/** The repo root — `pnpm exec` must run there for `tsx` to resolve. */ +const ROOT = join(HERE, '..', '..', '..', '..'); + +/** A measurement that succeeded. */ +export interface ProbeOk { + ok: true; + mode: string; + rows: number; + wireBytes: number; + records: number; + peakHeap: number; + peakLive: number; + peakBuffers: number; + settled: number; + ratio: number; + ticks: number; + marks: number; + ms: number; +} +/** A measurement whose workload BLEW UP. Also data — C3's finding is one of these. */ +export interface ProbeFail { + ok: false; + mode: string; + rows: number; + error: string; +} +export type Probe = ProbeOk | ProbeFail; + +export interface ProbeArgs { + mode: string; + rows: number; + /** `stream.buffer.chars`. Omit for the library default (~8M). */ + buffer?: number; + /** Extra node flags, e.g. `['--max-old-space-size=96']` to give the run a real heap ceiling. */ + nodeArgs?: string[]; +} + +/** The raw outcome of a probe process, INCLUDING one that died. C7 needs the corpse. */ +export interface ProbeRun { + /** Exit code. `null` when the process was killed by a signal. */ + status: number | null; + stdout: string; + stderr: string; + /** The parsed measurement, when the process lived long enough to print one. */ + measurement: Probe | undefined; + /** True when V8 aborted on a heap limit — the only signal the buffered path ever gives. */ + heapOom: boolean; +} + +/** Run one probe process and hand back everything it did, alive or dead. */ +export function probeRun({ + mode, + rows, + buffer, + nodeArgs = [], +}: ProbeArgs): ProbeRun { + const args = [ + 'exec', + 'tsx', + '--expose-gc', + ...nodeArgs, + PROBE, + `--mode=${mode}`, + `--rows=${String(rows)}`, + ]; + if (buffer !== undefined) args.push(`--buffer=${String(buffer)}`); + const r = spawnSync('pnpm', args, { + cwd: ROOT, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + const stdout = r.stdout ?? ''; + const stderr = r.stderr ?? ''; + const line = stdout.trim().split('\n').at(-1) ?? ''; + return { + status: r.status, + stdout, + stderr, + measurement: line.startsWith('{') + ? (JSON.parse(line) as Probe) + : undefined, + heapOom: /heap out of memory|Allocation failed/i.test( + `${stdout}\n${stderr}`, + ), + }; +} + +/** Run one probe. Throws only when the CHILD ITSELF failed to run (a crash, a bad path). */ +export function probe(args: ProbeArgs): Probe { + const r = probeRun(args); + if (r.measurement === undefined) { + throw new Error( + `probe ${args.mode}@${String(args.rows)} produced no measurement.\n` + + `exit=${String(r.status)}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`, + ); + } + return r.measurement; +} + +/** Run one probe and insist it succeeded. */ +export function probeOk(args: ProbeArgs): ProbeOk { + const p = probe(args); + if (!p.ok) + throw new Error(`probe ${p.mode}@${String(p.rows)} failed: ${p.error}`); + return p; +} + +/** The 1x / 10x / 100x series every scaling claim in this directory is built on. */ +export const SCALES = [1_000, 10_000, 100_000] as const; + +/** + * Run a mode across {@link SCALES}. Returns the three measurements in order. + * + * A cap large enough to be irrelevant is passed by default so the SHAPE of the curve is what is + * measured rather than where the library's 8M-char guard happens to sit; C3 measures the guard + * itself, separately and on purpose. + */ +export function series(mode: string, buffer = 1_000_000_000): ProbeOk[] { + return SCALES.map((rows) => probeOk({ mode, rows, buffer })); +} diff --git a/docs/scenarios/proofs/large-response-memory/validator-spy.ts b/docs/scenarios/proofs/large-response-memory/validator-spy.ts new file mode 100644 index 00000000..a1d9ff52 --- /dev/null +++ b/docs/scenarios/proofs/large-response-memory/validator-spy.ts @@ -0,0 +1,97 @@ +// An instrumented `output` contract. C4 asks a mechanism question — does validation run PER DELTA +// or over the AGGREGATE — and the honest way to answer it is to look at what the validator was +// handed, not to infer it from a heap number. +// +// So this is a real {@link Validator} (the `{ validate }` shape `toValidator` passes through +// untouched — `validator.ts:44-55`) that records every call: how many, and what SHAPE each argument +// was. If `output` validated the aggregate, one call would arrive carrying a 100,000-element array +// and `sawArrayOfLength` would say so. If it validates per delta, N calls arrive each carrying one +// object. The counter is the proof; the heap number is the consequence. +// +// It also does REAL per-field work — eight field checks against the declared product shape — so a +// heap measurement taken with it on is not measuring a no-op. +import type { Validator } from '../../../../packages/core/src/validator'; + +export interface ValidatorSpy extends Validator { + /** How many times the engine called `validate`. */ + readonly calls: () => number; + /** The largest array length ever handed to `validate`, or 0 if it never saw an array. */ + readonly sawArrayOfLength: () => number; + /** How many calls carried a plain (non-array) object — i.e. one record. */ + readonly recordCalls: () => number; + /** How many calls failed the shape check. */ + readonly rejects: () => number; +} + +const isRecord = (v: unknown): v is Record => + typeof v === 'object' && v !== null && !Array.isArray(v); + +/** + * A contract for one product row, wired to count. Retains NOTHING it was handed — only counters — + * so the spy itself can never be the reason a measurement looks linear. + */ +export function countingValidator(): ValidatorSpy { + let calls = 0; + let maxArray = 0; + let records = 0; + let rejects = 0; + const spy: ValidatorSpy = { + validate(value: unknown) { + calls++; + if (Array.isArray(value)) + maxArray = Math.max(maxArray, (value as unknown[]).length); + else if (isRecord(value)) records++; + if (!isRecord(value)) { + rejects++; + return Promise.resolve({ + ok: false as const, + issues: [{ path: [], message: 'expected an object' }], + }); + } + // Eight real field checks — the cost a schema library would charge, near enough. + const bad = + typeof value['id'] !== 'string' || + typeof value['sku'] !== 'string' || + typeof value['title'] !== 'string' || + typeof value['price_cents'] !== 'number' || + typeof value['currency'] !== 'string' || + typeof value['in_stock'] !== 'boolean' || + !Array.isArray(value['tags']) || + typeof value['updated_at'] !== 'string'; + if (bad) { + rejects++; + return Promise.resolve({ + ok: false as const, + issues: [{ path: [], message: 'bad product row' }], + }); + } + return Promise.resolve({ ok: true as const, value }); + }, + calls: () => calls, + sawArrayOfLength: () => maxArray, + recordCalls: () => records, + rejects: () => rejects, + }; + return spy; +} + +/** + * A contract that COERCES: it returns a value that is not the one it was given (a `price` in + * dollars added alongside the cents). Used to ask whether the streaming path serves the VALIDATED + * value or the raw chunk — the buffered path serves the validated one (`engine.ts:1223`). + */ +export function coercingValidator(): Validator { + return { + validate(value: unknown) { + if (!isRecord(value)) + return Promise.resolve({ + ok: false as const, + issues: [{ path: [], message: 'expected an object' }], + }); + return Promise.resolve({ + ok: true as const, + value: { ...value, coerced_marker: true }, + }); + }, + }; +} diff --git a/docs/scenarios/proofs/mid-stream-failure/README.md b/docs/scenarios/proofs/mid-stream-failure/README.md new file mode 100644 index 00000000..94f97962 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/README.md @@ -0,0 +1,141 @@ +# Proofs — a stream that fails after you've already shown the user 800 tokens + +Runnable evidence for the claims in +[`../../mid-stream-failure.md`](../../mid-stream-failure.md). + +Every script is standalone, offline, and deterministic: it injects a fake `text/event-stream` +provider through StitchAPI's `adapter` (or `Surface.execute`) seam and drives every wait off an +injected `manualClock()`, so a nine-second reconnect backoff is exact **virtual** time — no real +sleeping, nothing flaky, no network. + +**The measurement is a sequence, not a number.** The whole scenario turns on "what did a downstream +accumulator actually receive", so `observe()` drains `.stream()` and reports the exact delta +sequence, the concatenated token text, the tagged event spine and the reconnect boundaries. Tokens +are single letters, so duplication is legible at a glance: a correct run reads `ABCDE` and a +replayed one reads `ABCDEABCDEABCDEABCDE`. On the provider side, `opens.length`, `lastEventIds` and +`gaps` make "it reconnected", "it sent the right header" and "it waited what the server asked" three +measurements rather than three arguments. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts + +# all of them +for f in docs/scenarios/proofs/mid-stream-failure/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. That matters here: this scenario is about code +shipped two commits ago (#622). + +They typecheck under `packages/core`'s full strict set (the `@ts-expect-error` block in C8 is the +machine-checked half of that claim — a `@ts-expect-error` that is _not_ an error fails `tsc`): + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/mid-stream-failure/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| --------------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `c1-drop-mid-body.ts` | what does a `.stream()` consumer see on a drop? | **An `error` event, not a throw** — and a CLEAN truncation is `result,done(ok:true)`, identical to success | +| `c2-retry-on-a-stream.ts` | does `retry` re-emit already-delivered deltas? | **No — `retry` never runs on a stream.** 4 requests buffered vs **1** streaming, same config | +| `c3-resumable-reconnect.ts` | does `reconnect` resume a feed with `id:`? | **Yes, cleanly.** `ABCDE` once, `Last-Event-ID` `[(none),t2,t5,t5]`. Costs 3 wasted opens on a finished feed | +| `c4-openai-reconnect.ts` | …and against an OpenAI stream with no `id:`? | **4× replay of a stream that never failed.** `ABCDEABCDEABCDEABCDE`, 24 deltas, `done(ok:true)` | +| `c5-in-band-error.ts` | can a `data: {"error"}` frame at 200 fail? | **Only via `output`.** `interpret` runs **0 times** on a stream; `verdict.flag` inert; `onError` never fires | +| `c6-missing-done.ts` | can "ended early" be told from "ended"? | **Not by any built-in** — but 8 lines of surface `stream` hook makes it a named failure | +| `c7-partial-output.ts` | is the partial reachable when a stream fails? | **`.stream()` only.** `.safe().data` null, `error.body` undefined, `.inspect()` `data:null status:0` | +| `c8-connect-vs-body.ts` | connect-retry ON, body-retry OFF? | **Not in config.** One flag governs both phases. `Surface.execute` expresses it in 10 lines | +| `c9-assembled-solution.ts` | best answer for the LLM case, and is it worth it? | **62 lines** across 2 seams vs **83** hand-rolled; byte-identical results on all 6 shapes | + +## Files + +- `fake-llm-stream.ts` — the providers. One class, three shapes: **OpenAI-shaped** (`data: {...}` + frames, no `id:`, `data: [DONE]` terminator), **resumable feed** (`id:` on every frame, honours + `Last-Event-ID`), and **connect-phase failure** (a 503 before any byte, optionally healing). Plus + the failure knobs: `cut` (error the socket, or close it cleanly, after N frames — on chosen opens + only), `errorFrameAfter` (an in-band error at HTTP 200), and `retryHint` (a server `retry:` + field). Records every open with the virtual timestamp and the `Last-Event-ID` it received. +- `observe.ts` — the consumer-side instrument. Drains `.stream()` under the injected clock and + returns `{ data, text, events, reconnects, dones, errorFrames, ok, error, threw }`. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. +- `llm-stream.ts` — **user code** for C9: `llmSurface()` (connect-only retry via `execute`; `[DONE]` + requirement and in-band error detection via the `stream` hook) and `completion()` (drains + `.stream()` and keeps the partial). +- `hand-rolled.ts` — the same five rules with no StitchAPI in them, including its own SSE frame + parser, so C9's line-count comparison is honest and its behaviour comparison is exact. + +## Reading the numbers honestly + +- **C4 is a bug in freshly-shipped code, not a policy trade-off, and it is the finding of this + scenario.** `sse: { reconnect: true }` on a clean OpenAI-shaped completion — one that reached + `data: [DONE]` and closed normally — produced **4 opens, 24 deltas, 4 `[DONE]` sentinels** and + the text `ABCDEABCDEABCDEABCDE` delivered to the consumer as one uninterrupted spine, ending + `done(ok: true)`. Two independent defects compose. `resumable` is decided from surface + CAPABILITY, once, before any frame is read (engine.ts:1288 ands the reconnect flag with the mere + PRESENCE of `resumeToken` and `applyResume`), and `sseSurface` always exposes both hooks, so a + stream with no `id:` anywhere is classified resumable. When the reopen comes, `lastToken` is + still `undefined`, the guard at + engine.ts:1344 skips `applyResume`, and the request goes out with **no `Last-Event-ID`** — a + request for the whole completion, from token one. Separately, the reconnect loop treats a clean + close (`'closed'`) exactly like a drop (`'error'`) at engine.ts:1466, so `[DONE]` terminates + nothing and the attempt budget is always spent. The docs + ([surfaces.mdx:83-99](../../../../apps/docs/content/docs/reference/surfaces.mdx)) say "a dropped + stream is reopened"; they do not say a finished one is. +- **C2 refutes the capture in the safer direction, and the refutation matters more than the + prediction.** The capture expects `retry` to duplicate deltas. It cannot, because `retry` does not + run on a streaming stitch at all — `runStreaming` (engine.ts:1248) is a different function from + the buffered `attemptLoop` and has no attempt loop in it. The control is exact: an identical + four-attempt `retry` block against the same always-503 fake makes **4** requests on a buffered + stitch and **1** on an `sse` one, with `error.attempts: 1`. `retry` is not fully ignored, though — + `retry.backoff` supplies the reconnect CURVE (`fixed 5s` → measured gaps `5000,5000,5000`) while + `retry.attempts` is inert there. Two knobs that read as one cap, capping different things. +- **C5 kills the seam that ought to have been the answer.** `Surface.interpret` exists precisely to + say "this 200 is really a failure", and on a streaming surface it runs **0 times** — measured with + a counter, not read off the source. `runStreaming` only ever calls `classifyStatus(res.status, +cfg)` (engine.ts:1371), which is deliberately status-only because there is no buffered body at + open time. `verdict.flag` rides the same path and is silently inert — `ok: true`, and not even an + `info` drift finding. What works is `output`: per-`delta` validation at engine.ts:1414-1436 runs + BEFORE the delta is emitted, so a rejecting schema both fails the run and **withholds the bad + frame** (measured: 0 error frames delivered). The price is a generic `contract violation (drift)` + message; the schema's own wording survives only on `.report().findings`. +- **`hooks.onError` does not fire for ANY post-200 stream failure.** Measured hook sequence across a + failing run: `[onRequest, onResponse]`. The mid-body catch at engine.ts:1446-1449 records the + error and returns `'error'` without calling the hook — only the open-phase catch at + engine.ts:1354 does. Any error pipeline built on hooks is blind to exactly the failures this + scenario is about. +- **C7: the engine is holding the partial at the moment it discards it.** `chunks` accumulates every + delta at engine.ts:1443; the failure path at engine.ts:1467-1472 emits `error` + `done` and + returns **before** reaching `resultEvt(chunks, …)` at engine.ts:1492. Measured from outside: + `delta` fired 3 times, `result` never did. And `.inspect()` — the accessor whose documented job is + "what did the server actually send?" — answers `data: null`, `raw: null`, `status: 0`, so it does + not even report the 200 that was received. +- **C8's config workaround is worse than the gap it fills.** `verdict.accept: [503]` does make a 503 + reconnectable (the accepted status streams a non-`ReadableStream` body, decodes nothing, returns + `'closed'`). Measured against a healing server: 9 opens, answer replayed 7 times. Measured against + a permanently-503 server: **`ok: true`, `data: []`** — four refusals in a row resolving as a + successful empty stream. Nothing warns. +- **C9's comparison is honest in both directions.** 62 executable lines of user code against 83 + hand-rolled, and the hand-rolled side's SSE parser is counted because a hand-rolled client really + does need one. The results are byte-identical on all six shapes and the open counts match, so the + 21-line difference is not buying behaviour — it is buying the spine (one `start`/`delta`×N/ + `error`/`done` trace under one traceId) and keeping `auth`/`headers`/`throttle`/`timeout` as + config. The caveat is load-bearing: adding `sse: { reconnect: true }` to the assembled stitch + re-breaks it (4 opens, `ABCDEABCDEABCDEABCDE`), and no surface hook can defend against that — the + decision is made above them in `runStreaming`. +- **Two mitigations for C4 exist and both cost something.** Resetting the accumulator on the + `progress:reconnect` event recovers `ABCDE` — but "reset on reconnect" is the opposite of + resuming, so the reconnect bought nothing. `break`-ing out of the `for await` on `[DONE]` holds it + to 1 open — but leaving the loop early forfeits `await`/`.safe()` entirely, since awaiting drains + the generator, replays and all. Aborting a signal on `[DONE]` also stops it at 1 open, and poisons + the run with `Error: aborted`. diff --git a/docs/scenarios/proofs/mid-stream-failure/c1-drop-mid-body.ts b/docs/scenarios/proofs/mid-stream-failure/c1-drop-mid-body.ts new file mode 100644 index 00000000..18dd6c10 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c1-drop-mid-body.ts @@ -0,0 +1,159 @@ +// C1 — the stream drops mid-body after N deltas, with no `[DONE]`. What does a `.stream()` consumer +// see: an error, a clean-looking end, or something indistinguishable from success? And what does +// `await` (which collects) give? +// +// The two halves answer differently, and that difference is the finding. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c1-drop-mid-body.ts +import { sse } from '../../../../packages/core/src/sse'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeStreamProvider } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +async function main(): Promise { + heading('C1 — a mid-body drop: what does the consumer see?'); + + // ── (a) baseline: a CLEAN OpenAI completion ─────────────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const obs = await observe(chat, {}, clock); + check('(a) opens', api.opens.length, 1); + check('(a) text delivered', obs.text, 'ABCDE'); + check('(a) `[DONE]` sentinels', obs.dones, 1); + checkSeq('(a) event spine', obs.events, [ + 'start', + 'progress:request', + ...Array(6).fill('delta'), // 5 tokens + `[DONE]` + 'result', + 'done', + ]); + check('(a) done.ok', obs.ok, true); + } + + // ── (b) the DROP: transport dies after 3 of 5 tokens ────────────────────────────────────── + // The 200 is long since spent. The engine emits the deltas it got, then an `error` and a + // FAILED `done`. So a `.stream()` consumer CAN tell — but only by reading the control events; + // the delta sequence itself just stops. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error' }, + }); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const obs = await observe(chat, {}, clock); + check('(b) opens', api.opens.length, 1); + check('(b) text delivered before the drop', obs.text, 'ABC'); + check('(b) `[DONE]` sentinels', obs.dones, 0); + checkSeq('(b) event spine', obs.events, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'delta', + 'error', + 'done', + ]); + check('(b) done.ok', obs.ok, false); + check('(b) error.message', obs.error, 'socket reset by peer'); + check( + '(b) did the ITERATOR throw?', + obs.threw ?? 'no — the failure is an `error` EVENT', + 'no — the failure is an `error` EVENT', + ); + note( + '(b) → a `for await (… of .stream())` that only looks at `delta` sees a SILENT truncation', + 'the error arrives as a separate event type it never matched', + ); + } + + // ── (c) the same drop on the AWAIT path — the partial is GONE ───────────────────────────── + // `await`/`.safe()` collects the deltas into an array, but a failed run resolves to the error, + // never to `chunks`. The three tokens the server already produced (and billed) are unreachable. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error' }, + }); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const p = chat.safe({}); + await clock.advance(3_600_000); + const r = await p; + check('(c) ok', r.ok, false); + check('(c) data', JSON.stringify(r.data), 'null'); + check('(c) error.message', r.error?.message, 'socket reset by peer'); + check('(c) error.status', String(r.error?.status), 'undefined'); + check('(c) error.attempts', r.error?.attempts, 1); + check( + '(c) error.body (the partial?)', + String((r.error as { body?: unknown } | null)?.body), + 'undefined', + ); + note( + '(c) → on the await path a mid-stream failure is TOTAL', + 'the 3 tokens that arrived are not on the error, not in `data`, not anywhere', + ); + } + + // ── (d) the nastier drop: a CLEAN close, mid-answer ─────────────────────────────────────── + // A truncated answer whose socket closed normally. The transport was fine, so the engine has + // nothing to complain about: `result` + `done(ok: true)`. It is byte-for-byte the (a) spine + // with fewer deltas — success and truncation are the SAME shape. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const obs = await observe(chat, {}, clock); + check('(d) text delivered', obs.text, 'ABC'); + check('(d) `[DONE]` sentinels', obs.dones, 0); + checkSeq('(d) event spine', obs.events, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'delta', + 'result', + 'done', + ]); + check('(d) done.ok', obs.ok, true); + const p = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const s2 = sse({ url: URL, adapter: p.adapter(), clock }); + const pr = s2.safe({}); + await clock.advance(3_600_000); + const r = await pr; + check('(d) await → ok', r.ok, true); + check( + '(d) await → collected deltas', + (r.data as unknown[] | null)?.length, + 3, + ); + note( + '(d) → THE footgun of this claim', + 'a severed answer resolves SUCCESSFULLY with a short array; only the missing `[DONE]` tells you', + ); + } + + finish( + 'C1', + 'BOTH, depending on which end you read, and the answers disagree. A TRANSPORT drop is visible on `.stream()` — `delta,delta,delta,error,done(ok:false)` with `error.message: "socket reset by peer"` — but it is an EVENT, not a throw, so a consumer that only matches `delta` sees a silent truncation. On the await path the same drop is TOTAL: `ok:false`, `data: null`, and the 3 tokens that did arrive are on neither the error (`error.body === undefined`) nor anywhere else. And a CLEAN close mid-answer is worse than either: 3 deltas, `result`, `done(ok:true)`, `await` resolving to a 3-element array — the exact spine of a complete run. Success and truncation are the same shape unless YOU check for `[DONE]`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c2-retry-on-a-stream.ts b/docs/scenarios/proofs/mid-stream-failure/c2-retry-on-a-stream.ts new file mode 100644 index 00000000..ccf4164c --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c2-retry-on-a-stream.ts @@ -0,0 +1,196 @@ +// C2 — THE DECIDING CLAIM. With `retry` configured on a STREAMING stitch and a mid-body drop after +// N deltas have already been yielded to the consumer: are those N deltas RE-EMITTED? +// +// This is the "fragment already held by a downstream accumulator" hazard the research capture calls +// the sharp one. The capture predicts duplication. It is WRONG, and the reason it is wrong is a +// bigger finding than the prediction would have been: `retry` does not run on a streaming stitch AT +// ALL. Not once, for any status, for any `on`. The streaming path (`engine.ts:1248` `runStreaming`) +// is a different function from the buffered `attemptLoop` and has no retry loop in it — its only +// loop is the reconnect loop (C3/C4), which is off by default. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c2-retry-on-a-stream.ts +import { stitch } from '../../../../packages/core/src/index'; +import { sse } from '../../../../packages/core/src/sse'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Adapter, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { FakeStreamProvider } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +async function main(): Promise { + heading('C2 — does `retry` re-emit already-delivered deltas?'); + + // ── (a) `retry: { attempts: 3 }` + a mid-body drop after 3 tokens ───────────────────────── + // The measured delta sequence is `ABC`. Once. Not `ABCABCABC`. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error' }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + retry: { attempts: 3 }, + }); + const obs = await observe(chat, {}, clock); + checkSeq( + '(a) DELTA SEQUENCE the consumer observed', + obs.data.map((d) => JSON.stringify(d).slice(0, 40)), + [ + '{"object":"chat.completion.chunk","choic', + '{"object":"chat.completion.chunk","choic', + '{"object":"chat.completion.chunk","choic', + ], + ); + check('(a) TEXT the accumulator holds', obs.text, 'ABC'); + check('(a) opens the server saw', api.opens.length, 1); + check('(a) done.ok', obs.ok, false); + check('(a) any content duplicated?', obs.text === 'ABC', true); + note( + '(a) → NO duplication. Also no retry: `attempts: 3` produced ONE open', + 'the capture predicted `ABCABCABC`; the measurement is `ABC`', + ); + } + + // ── (b) the control — `retry` is not merely "not triggered", it is INERT on a stream ────── + // Same status, same `retry`, two surfaces. A 503 is in the default `retry.on` + // (`[429, 502, 503, 504]` — engine.ts:612), so the buffered stitch retries it 4 times. The + // streaming stitch opens ONCE and gives up. + { + const mk503 = (): { adapter: Adapter; count: () => number } => { + let n = 0; + return { + count: () => n, + adapter: () => { + n++; + return Promise.resolve({ + status: 503, + headers: {}, + body: { error: 'overloaded' }, + } satisfies AdapterResponse); + }, + }; + }; + + const clockA = manualClock(); + const a = mk503(); + const buffered = stitch({ + url: URL, + adapter: a.adapter, + clock: clockA, + retry: { attempts: 4 }, + }); + const pa = buffered.safe({}); + await clockA.advance(3_600_000); + await pa; + + const clockB = manualClock(); + const b = mk503(); + const streaming = sse({ + url: URL, + adapter: b.adapter, + clock: clockB, + retry: { attempts: 4 }, + }); + const pb = streaming.safe({}); + await clockB.advance(3_600_000); + const rb = await pb; + + check('(b) BUFFERED stitch, retry.attempts 4 → requests', a.count(), 4); + check( + '(b) STREAMING stitch, retry.attempts 4 → requests', + b.count(), + 1, + ); + check('(b) streaming ok', rb.ok, false); + check('(b) streaming error.attempts', rb.error?.attempts, 1); + note( + '(b) → same config, same status, same engine', + '`retry` is read for the reconnect BACKOFF and nothing else on a streaming surface', + ); + } + + // ── (c) the one thing `retry` DOES do on a stream: pace a reconnect ─────────────────────── + // `runStreaming` falls back to `backoffDelay(attempt + 1, cfg.retry)` (engine.ts:1481) when no + // server `retry:` and no `reconnect.delay` is set. So `retry.backoff` is live — as a CURVE for + // the reconnect loop, never as an attempt count. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + cut: { after: 1, how: 'error', onOpens: [1, 2, 3, 4] }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: { attempts: 3 } }, + retry: { attempts: 1, backoff: { curve: 'fixed', base: '5s' } }, + }); + await observe(chat, {}, clock); + checkSeq('(c) virtual ms between opens', api.gaps, [5000, 5000, 5000]); + check( + '(c) opens, with retry.attempts 1', + api.opens.length, + 4, // 1 + reconnect.attempts 3 — the RECONNECT cap, not the retry cap + ); + note( + '(c) → `retry.attempts: 1` and four opens happened anyway', + 'the two knobs that look like they cap the same thing cap different things', + ); + } + + // ── (d) the default reconnect backoff, when no `retry` block is authored ────────────────── + // `backoffDelay` defaults to `expo-jitter` off base 100 (resilience.ts:39-56), so the first + // reconnect lands in [0, 100) ms. A dropped LLM stream is replayed within a tenth of a second. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + cut: { after: 1, how: 'error', onOpens: [1, 2, 3, 4] }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + await observe(chat, {}, clock); + const gaps = api.gaps; + check('(d) reconnects', gaps.length, 3); + check( + '(d) first reconnect wait < 100ms', + (gaps[0] as number) < 100, + true, + ); + check( + '(d) every wait < 500ms', + gaps.every((g) => g < 500), + true, + ); + note( + '(d) measured waits (ms)', + gaps.map((g) => Math.round(g)).join(', '), + ); + } + + finish( + 'C2', + 'NO — and the capture is refuted twice over. The measured delta sequence under `retry: { attempts: 3 }` with a drop after 3 of 5 tokens is `ABC`, ONE time: no fragment is replayed into a downstream accumulator. The reason is that `retry` does not run on a streaming stitch AT ALL — the control pins it: the same `retry: { attempts: 4 }` against the same always-503 fake makes 4 requests on a buffered stitch and 1 on an `sse` one, and `error.attempts` is 1. `runStreaming` (engine.ts:1248) has no attempt loop; the only loop is the reconnect loop. `retry` is not ignored, though — `retry.backoff` supplies the reconnect CURVE (`fixed 5s` → measured gaps 5000,5000,5000) while `retry.attempts` is inert there, so the two knobs that look like one cap different things. And the default reconnect backoff is `expo-jitter` off base 100: measured first wait under 100ms. The duplication hazard is real in this library, but it is `sse.reconnect` that causes it (C4), not `retry`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c3-resumable-reconnect.ts b/docs/scenarios/proofs/mid-stream-failure/c3-resumable-reconnect.ts new file mode 100644 index 00000000..e9aa832d --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c3-resumable-reconnect.ts @@ -0,0 +1,170 @@ +// C3 — `sse: { reconnect: true }` against a server that DOES emit `id:` on every frame and DOES +// honour `Last-Event-ID`. Does it resume without duplication? Measure the delta sequence AND the +// header value actually sent on the reopened request. +// +// This is the shape SSE was designed for, and it is the one case where the answer is genuinely +// clean — the delta spine is `ABCDE`, once, across a drop. The cost is measured too, because it is +// not zero: the loop cannot tell "the feed is finished" from "the connection dropped", so it keeps +// reopening until the attempt budget runs out. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c3-resumable-reconnect.ts +import { sse } from '../../../../packages/core/src/sse'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeStreamProvider } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://events.example.com/feed'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +/** The `Last-Event-ID` header per open, with the absent first one spelled so it reads in output. */ +const idsOf = (api: FakeStreamProvider): string[] => + api.lastEventIds.map((v) => v ?? '(none)'); + +async function main(): Promise { + heading('C3 — resumable feed + `sse.reconnect`: resume, or replay?'); + + // ── (a) the headline: a drop after 2 of 5, resumed exactly ──────────────────────────────── + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + done: false, // a feed has no `[DONE]` sentinel; it just keeps going + cut: { after: 2, how: 'error' }, // only open 1 drops + }); + const feed = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + const obs = await observe(feed, {}, clock); + + check('(a) TEXT the consumer accumulated', obs.text, 'ABCDE'); + check('(a) any token seen twice?', obs.text === 'ABCDE', true); + check('(a) deltas delivered', obs.data.length, 5); + checkSeq('(a) `Last-Event-ID` sent on each open', idsOf(api), [ + '(none)', // the first connection carries no header + 't2', // the drop happened after `id: t2` — this is the resume point + 't5', + 't5', + ]); + check('(a) reconnects the consumer could see', obs.reconnects, 3); + check('(a) done.ok', obs.ok, true); + note( + '(a) → this is the clean case', + 'ONE `sse: { reconnect: true }` and the resume is correct — no duplication, right header', + ); + } + + // ── (b) the cost: a CLEAN close is treated as a drop, so the loop over-reopens ──────────── + // `openAndDecode` returns `'closed'` when the body runs out, and the reconnect loop + // (engine.ts:1460-1490) treats `'closed'` exactly like `'error'`. There is no "the stream is + // finished" signal, so a resumable stitch always burns its whole attempt budget. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + done: false, + // no `cut` at all: the feed completes cleanly on the FIRST connection + }); + const feed = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + const obs = await observe(feed, {}, clock); + check('(b) text', obs.text, 'ABCDE'); + check('(b) opens for a feed that never dropped', api.opens.length, 4); + check('(b) wasted round trips', api.opens.length - 1, 3); + checkSeq('(b) `Last-Event-ID` sent', idsOf(api), [ + '(none)', + 't5', + 't5', + 't5', + ]); + note( + '(b) → harmless HERE only because the server correctly returns nothing after `t5`', + 'a server that ignores `Last-Event-ID` replays instead — that is C4', + ); + } + + // ── (c) the server's own pacing wins ────────────────────────────────────────────────────── + // A `retry:` field on the dropped connection beats `reconnect.delay` and `retry.backoff` + // (engine.ts:1480-1481). Measured as exact virtual gaps between opens. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + done: false, + retryHint: 9_000, + cut: { after: 1, how: 'error', onOpens: [1, 2, 3, 4] }, + }); + const feed = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: { attempts: 3, delay: '250ms' } }, + }); + await observe(feed, {}, clock); + checkSeq( + '(c) virtual ms between opens (server said 9000)', + api.gaps, + [9000, 9000, 9000], + ); + note( + '(c) → `reconnect.delay: "250ms"` was authored and never used', + 'the server `retry:` wins, as documented', + ); + } + + // ── (d) a feed that keeps dropping: resumption is still correct, the run still FAILS ────── + // Every open drops after one frame. The ids make each resume exact — `ABCD`, no repeats — but + // the budget runs out and the run ends `error` + `done(ok:false)`. The four tokens are on + // `.stream()` only; the await path gets nothing (C7). + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + ids: 'per-token', + done: false, + cut: { after: 1, how: 'error', onOpens: [1, 2, 3, 4] }, + }); + const feed = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: { attempts: 3 } }, + }); + const obs = await observe(feed, {}, clock); + check('(d) text across four partial connections', obs.text, 'ABCD'); + checkSeq('(d) `Last-Event-ID` sent', idsOf(api), [ + '(none)', + 't1', + 't2', + 't3', + ]); + check('(d) done.ok', obs.ok, false); + check('(d) error.message', obs.error, 'socket reset by peer'); + check( + '(d) `result` event emitted?', + obs.events.includes('result'), + false, + ); + } + + finish( + 'C3', + 'YES — this is the one case that genuinely just works. A feed with `id:` on every frame, dropped after 2 of 5 tokens, reopened with `sse: { reconnect: true }`: the consumer observed `ABCDE`, five deltas, ZERO duplication, and the reopened request carried the exact right header — measured `Last-Event-ID` sequence `[(none), "t2", "t5", "t5"]`, where `t2` is the last id delivered before the drop. Server pacing is honoured (a `retry: 9000` frame produced measured gaps 9000,9000,9000, overriding an authored `reconnect.delay: "250ms"`). Two real costs, both measured: the loop cannot distinguish "the feed finished" from "the connection dropped", so a feed that NEVER drops still opens 4 times (3 wasted round trips, each replaying `Last-Event-ID: t5`); and a feed that keeps dropping resumes correctly (`ABCD`, no repeats) but still ends `done(ok:false)` with no `result` event once the budget is spent', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts b/docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts new file mode 100644 index 00000000..5d42e77a --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts @@ -0,0 +1,179 @@ +// C4 — `sse: { reconnect: true }` against an OPENAI-SHAPED stream: `data: {...}` frames with NO +// `id:` anywhere, terminated by `data: [DONE]`. Silent restart from the beginning (duplication), +// refusal, or something else? +// +// Measured: SILENT RESTART, four times over, on a stream that DID NOT FAIL. This is the sharpest +// finding in the scenario and it is a correctness bug in freshly-shipped code (#622), not a policy +// trade-off. Two independent defects compose: +// +// 1. `resumable` (engine.ts:1288) is `policy.enabled && !!resumeToken && !!applyResume` — the +// surface's CAPABILITY, evaluated once, before any frame is read. `sseSurface` always exposes +// both hooks, so an OpenAI stream with no `id:` in it anywhere is classified resumable. When +// the reopen happens, `lastToken` is still `undefined`, the `attempt > 1 && lastToken !== +// undefined` guard at engine.ts:1344 skips `applyResume`, and the request goes out with NO +// `Last-Event-ID` — i.e. a request for the whole completion, from token one. +// 2. The reconnect loop (engine.ts:1460-1490) treats a CLEAN body close (`'closed'`) exactly like +// a drop (`'error'`). There is no "this stream is complete" signal, so `[DONE]` means nothing +// and a finished completion is reopened until the attempt budget is spent. +// +// Together: one config flag turns one answer into four, delivered to the consumer as one +// uninterrupted delta spine, and the run ends `ok: true`. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c4-openai-reconnect.ts +import { sse } from '../../../../packages/core/src/sse'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { FakeStreamProvider, contentOf, isDone } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +async function main(): Promise { + heading('C4 — `reconnect` on an OpenAI-shaped stream with no `id:`'); + + // ── (a) A STREAM THAT NEVER FAILED, replayed four times ─────────────────────────────────── + // No `cut`. The provider writes all five tokens and `data: [DONE]`, then closes normally. The + // consumer is handed the complete answer FOUR times and the run reports success. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + const obs = await observe(chat, {}, clock); + + check('(a) opens the provider saw', api.opens.length, 4); + check( + '(a) TEXT the consumer accumulated', + obs.text, + 'ABCDEABCDEABCDEABCDE', + ); + check('(a) deltas delivered', obs.data.length, 24); // 4 × (5 tokens + [DONE]) + check('(a) `[DONE]` sentinels observed', obs.dones, 4); + checkSeq( + '(a) `Last-Event-ID` sent on each open', + api.lastEventIds.map((v) => v ?? '(none)'), + ['(none)', '(none)', '(none)', '(none)'], + ); + check('(a) done.ok', obs.ok, true); + note( + '(a) → the model ran 4 times, the caller pays 4×, the UI renders the answer 4×', + 'and nothing in the result says so: `done(ok: true)` with a 24-element array', + ); + } + + // ── (b) the same flag on a stream that DID drop ─────────────────────────────────────────── + // A drop after 3 of 5. The partial is replayed whole on every reopen — `ABCABCABCABC` — and the + // run still ends in failure. Worst of both: duplicated content AND an error. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error', onOpens: [1, 2, 3, 4] }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + const obs = await observe(chat, {}, clock); + check('(b) opens', api.opens.length, 4); + check('(b) TEXT the consumer accumulated', obs.text, 'ABCABCABCABC'); + check('(b) `[DONE]` sentinels', obs.dones, 0); + check('(b) done.ok', obs.ok, false); + check('(b) error.message', obs.error, 'socket reset by peer'); + } + + // ── (c) `sse: true` is the same thing ───────────────────────────────────────────────────── + // The shorthand (`stitch.ts:243` — `sse === true` becomes `{ reconnect: true }`) reads like + // "this is an SSE stitch", which is exactly the sort of thing someone adds without thinking. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: true, + }); + const obs = await observe(chat, {}, clock); + check('(c) `sse: true` → opens', api.opens.length, 4); + check('(c) `sse: true` → text', obs.text, 'ABCDEABCDEABCDEABCDE'); + } + + // ── (d) the duplication IS detectable by the consumer — a `progress:reconnect` marks it ──── + // `runStreaming` emits `{ type: 'progress', phase: 'reconnect' }` before each reopen + // (engine.ts:1482-1488). A consumer that resets its accumulator on that event recovers the + // right text. This is the mitigation, and it is user code, and it costs the whole point of + // reconnect (the resume) since every reopen starts over. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + let text = ''; + const drain = (async () => { + for await (const ev of chat.stream( + {}, + ) as AsyncIterable) { + if (ev.type === 'progress' && ev.phase === 'reconnect') + text = ''; + if (ev.type === 'delta') + text += + contentOf((ev.chunk as { data: unknown }).data) ?? ''; + } + })(); + await clock.advance(3_600_000); + await drain; + check( + '(d) text after resetting on every `progress:reconnect`', + text, + 'ABCDE', + ); + note( + '(d) → the boundary is visible, so a careful consumer can undo the damage', + 'but "reset the accumulator" is the OPPOSITE of resuming — the reconnect bought nothing', + ); + } + + // ── (e) the cheap escape: `break` on `[DONE]` ───────────────────────────────────────────── + // Leaving the `for await` calls `.return()` on the generator, so the engine never reaches the + // reconnect. One open, one answer. The price is that you can no longer `await` the stitch — + // awaiting drains the whole generator, replays and all. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + }); + const obs = await observe(chat, {}, clock, { stopOn: isDone }); + check( + '(e) opens when the consumer breaks on `[DONE]`', + api.opens.length, + 1, + ); + check('(e) text', obs.text, 'ABCDE'); + check('(e) terminal `done` event seen?', obs.ok === undefined, true); + } + + finish( + 'C4', + 'SILENT RESTART FROM THE BEGINNING — and, worse than the capture guessed, it happens to streams that never failed. `sse: { reconnect: true }` on a clean OpenAI-shaped completion produced 4 opens, a 24-delta spine, 4 `[DONE]` sentinels, and the measured text `ABCDEABCDEABCDEABCDE` delivered to the consumer as one uninterrupted stream — ending `done(ok: true)`. No `Last-Event-ID` was ever sent (measured `[(none) ×4]`): there is no id to resume from, so every reopen is a request for the whole completion. Two defects compose — `resumable` is decided from surface CAPABILITY before any frame is read (engine.ts:1288), and a clean close is treated as a drop (engine.ts:1466), so `[DONE]` terminates nothing. On a stream that DID drop the result is `ABCABCABCABC` plus a failure. `sse: true` is the same flag. Two mitigations exist and both are user code: reset the accumulator on the `progress:reconnect` event (measured: recovers `ABCDE`), or `break` on `[DONE]` (measured: 1 open) — which forfeits `await` entirely', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c5-in-band-error.ts b/docs/scenarios/proofs/mid-stream-failure/c5-in-band-error.ts new file mode 100644 index 00000000..872fe775 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c5-in-band-error.ts @@ -0,0 +1,271 @@ +// C5 — an in-band `data: {"error": {...}}` frame arriving at HTTP 200. Can it become a REAL +// failure? The capture nominates three candidates: `verdictOf`, a custom surface's `interpret`, and +// hooks. Two of the three are structurally unavailable on a streaming surface, and the reason is +// worth stating precisely because it also answers "when does `interpret` run relative to the body". +// +// It does not run. Not once, not early, not late. `runStreaming` (engine.ts:1248) never calls +// `interpretOf(surface)` — the only verdict it takes is `classifyStatus(res.status, cfg)` at +// engine.ts:1371, which is deliberately status-only because at open time there is no buffered body +// to rule on. So the surface hook that exists precisely to say "this 200 is actually a failure" is +// dead code on every streaming surface. +// +// The one thing that DOES work is `output`, and it works well: per-`delta` contract validation +// (engine.ts:1414-1436) runs BEFORE the delta is emitted, so a rejecting schema both fails the run +// and withholds the bad frame from the consumer. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c5-in-band-error.ts +import { stitch } from '../../../../packages/core/src/index'; +import { sse, sseSurface } from '../../../../packages/core/src/sse'; +import type { StandardSchemaV1 } from '../../../../packages/core/src/standard-schema'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { verdictOf } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeStreamProvider, errorOf } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +/** A provider that streams 2 tokens and then an in-band error frame, at HTTP 200 throughout. */ +const inBand = (clock: ReturnType): FakeStreamProvider => + new FakeStreamProvider({ clock, tokens: TOKENS, errorFrameAfter: 2 }); + +async function main(): Promise { + heading('C5 — an in-band error frame at HTTP 200: can it fail the call?'); + + // ── (a) the default: it is just another delta, and the run SUCCEEDS ─────────────────────── + { + const clock = manualClock(); + const api = inBand(clock); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const obs = await observe(chat, {}, clock); + check('(a) text delivered', obs.text, 'AB'); + check('(a) in-band error frames observed', obs.errorFrames, 1); + check('(a) done.ok', obs.ok, true); + checkSeq('(a) event spine', obs.events, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'delta', // ← the error frame, indistinguishable from a token at the engine level + 'result', + 'done', + ]); + note( + '(a) → the provider said it failed and the client says it succeeded', + 'the error is in the delta array; nothing looks at it', + ); + } + + // ── (b) a custom surface's `interpret` — NEVER CALLED ───────────────────────────────────── + // The obvious move: clone `sseSurface`, add an `interpret` that composes `verdictOf`. It + // typechecks, it is the documented seam for "this 200 is a failure" (surface.ts:174-191), and + // on a streaming surface it runs ZERO times. + { + let interpretCalls = 0; + const guarded: Surface = { + ...sseSurface, + id: 'sse-guarded', + interpret: (res, cfg) => { + interpretCalls++; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const clock = manualClock(); + const api = inBand(clock); + const chat = stitch({ + url: URL, + kind: guarded, + adapter: api.adapter(), + clock, + }); + const r = await chat.safe({}); + check('(b) times `interpret` ran', interpretCalls, 0); + check('(b) ok', r.ok, true); + check('(b) collected deltas', (r.data as unknown[] | null)?.length, 3); + note( + '(b) → `interpret` is the surface model’s answer to this exact question', + 'and `runStreaming` never calls it: only `classifyStatus(res.status)` at engine.ts:1371', + ); + } + + // ── (c) `verdict.flag` — silently inert on a stream ─────────────────────────────────────── + // `verdict.flag` is read by `verdictOf`, which only `interpret` calls. On a streaming stitch it + // is accepted by the config, typechecks, and does nothing at all — not even a drift finding. + { + const clock = manualClock(); + const api = inBand(clock); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + verdict: { flag: 'ok' }, + }); + const r = await chat.safe({}); + check('(c) `verdict.flag: "ok"` → ok', r.ok, true); + check('(c) collected deltas', (r.data as unknown[] | null)?.length, 3); + const rep = await chat.report({}); + check('(c) drift findings from the inert flag', rep.findings.length, 0); + } + + // ── (d) hooks — `onResponse` fires before a single frame is parsed ──────────────────────── + // `hooks.onResponse` runs at engine.ts:1352, immediately after the adapter returns and before + // the body is decoded, so `ctx.res.body` is a live `ReadableStream` and there is nothing in it + // to inspect yet. Reading it would consume the stream out from under the surface. + { + const clock = manualClock(); + const api = inBand(clock); + let bodyKind = ''; + let deltasAtHook = -1; + const seen: string[] = []; + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + hooks: { + onResponse: (ctx) => { + seen.push('onResponse'); + bodyKind = + ctx.res?.body instanceof ReadableStream + ? 'ReadableStream' + : typeof ctx.res?.body; + deltasAtHook = 0; + }, + onError: (ctx) => { + seen.push( + `onError:${String((ctx.error as Error).message)}`, + ); + }, + }, + }); + const obs = await observe(chat, {}, clock); + check( + '(d) `onResponse` fired', + seen.filter((s) => s === 'onResponse').length, + 1, + ); + check('(d) `ctx.res.body` at hook time', bodyKind, 'ReadableStream'); + check('(d) deltas available at hook time', deltasAtHook, 0); + check( + '(d) `onError` fired for the in-band error', + seen.some((s) => s.startsWith('onError')), + false, + ); + check('(d) run still ok', obs.ok, true); + } + + // ── (e) `onError` does not fire for a TRANSPORT drop either ────────────────────────────── + // Worth pinning while we are here: the mid-body `catch` at engine.ts:1446-1449 records the + // error and returns `'error'` WITHOUT calling `hooks.onError` (only the open-phase catch at + // engine.ts:1354 does). So a stream that fails after the 200 fires no error hook at all, even + // though the run fails. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 2, how: 'error' }, + }); + const fired: string[] = []; + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + hooks: { + onRequest: () => { + fired.push('onRequest'); + }, + onResponse: () => { + fired.push('onResponse'); + }, + onError: () => { + fired.push('onError'); + }, + }, + }); + const obs = await observe(chat, {}, clock); + checkSeq('(e) hooks fired on a mid-body drop', fired, [ + 'onRequest', + 'onResponse', + ]); + check('(e) run ok', obs.ok, false); + note( + '(e) → the run failed and `hooks.onError` never fired', + 'a hook-based error pipeline misses every post-200 stream failure', + ); + } + + // ── (f) `output` — the ONE thing that works, and it works properly ──────────────────────── + // Per-`delta` contract validation (engine.ts:1414-1436) runs before the delta is emitted, on the + // value the surface's `contractValue` picks (`sse` → the event's `data`). An `error` finding + // fails the stream — and the offending frame is WITHHELD from the consumer. + { + const rejectErrorFrames: StandardSchemaV1 = { + '~standard': { + version: 1, + vendor: 'mid-stream-failure-proof', + validate: (value: unknown) => + errorOf(value) !== undefined + ? { + issues: [ + { + message: `provider sent an in-band error: ${String(errorOf(value)?.message)}`, + }, + ], + } + : { value }, + }, + }; + const clock = manualClock(); + const api = inBand(clock); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + output: rejectErrorFrames, + }); + const obs = await observe(chat, {}, clock); + checkSeq('(f) event spine', obs.events, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'drift', // the contract rejected the error frame + 'error', + 'done', + ]); + check('(f) text delivered', obs.text, 'AB'); + check( + '(f) was the error frame delivered to the consumer?', + obs.errorFrames, + 0, + ); + check('(f) done.ok', obs.ok, false); + check('(f) error.message', obs.error, 'contract violation (drift)'); + + const p = chat.safe({}); + await clock.advance(3_600_000); + const r = await p; + check('(f) await → ok', r.ok, false); + check( + '(f) await → error.message', + r.error?.message, + 'contract violation (drift)', + ); + note( + '(f) → the message the schema authored is on the drift FINDING, not the error', + 'the thrown error says only `contract violation (drift)`; `.report().findings` carries the detail', + ); + const rep = await chat.report({}); + check('(f) findings on the report', rep.findings.length >= 1, true); + note('(f) finding detail', rep.findings[0]?.detail ?? '(none)'); + } + + finish( + 'C5', + 'YES, but through exactly ONE seam, and it is not the one the capture nominates. A custom surface’s `interpret` runs ZERO times on a streaming stitch (measured) — `runStreaming` never calls `interpretOf`, only `classifyStatus(res.status, cfg)` at engine.ts:1371, so the surface hook whose job is "this 200 is really a failure" is dead code here; `verdict.flag` rides the same path and is silently inert (ok:true, and not even a drift finding). Hooks cannot see it either: `onResponse` fires before a single frame is parsed (`ctx.res.body` measured as a live `ReadableStream`), and `hooks.onError` does not fire for an in-band error NOR for a real transport drop — measured hook sequence on a mid-body drop is `[onRequest, onResponse]` while the run fails. What works is `output`: per-`delta` validation runs BEFORE the delta is emitted, so a schema that rejects `{ error: … }` frames turns the frame into `drift` → `error` → `done(ok:false)` AND withholds it from the consumer (measured: 0 error frames delivered, text `AB`). The cost is that the thrown message is the generic `contract violation (drift)` — the schema’s own message survives only on `.report().findings`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c6-missing-done.ts b/docs/scenarios/proofs/mid-stream-failure/c6-missing-done.ts new file mode 100644 index 00000000..c4738871 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c6-missing-done.ts @@ -0,0 +1,225 @@ +// C6 — the missing `[DONE]`. Can "the stream ended early" be distinguished from "the stream ended", +// and can the difference be made a failure? +// +// Nothing built-in can do it, for a structural reason: every gate on the streaming path is +// PER-FRAME (`output` validates each delta before emitting it) or PER-OPEN (`classifyStatus` on the +// status line). Truncation is the absence of a frame, and absence is not a frame. +// +// There IS a working seam, and it is a good one: the surface's own `stream` hook. Wrapping +// `sseSurface.stream` in a generator that throws when the body runs out without `[DONE]` puts the +// check exactly where the knowledge lives, and the throw lands in the engine's mid-body `catch` +// (engine.ts:1446) — so truncation surfaces as an ordinary stream failure with your message on it. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c6-missing-done.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { SseEvent } from '../../../../packages/core/src/sse'; +import { sse, sseSurface } from '../../../../packages/core/src/sse'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + AdapterResponse, + ResolvedStitchConfig, +} from '../../../../packages/core/src/types'; +import { FakeStreamProvider, isDone } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +/** + * USER CODE — the `sse` surface with one added rule: a body that ends without `data: [DONE]` was + * truncated, and truncation is a failure. Eight executable lines. + */ +const decodeSse = sseSurface.stream as NonNullable; +const sseRequiringDone: Surface = { + ...sseSurface, + id: 'sse-done', + stream: async function* (res: AdapterResponse, cfg: ResolvedStitchConfig) { + let sawDone = false; + for await (const chunk of decodeSse(res, cfg)) { + sawDone ||= isDone((chunk as SseEvent).data); + yield chunk; + } + if (!sawDone) + throw new Error('stream ended without the `[DONE]` sentinel'); + }, +}; + +async function main(): Promise { + heading('C6 — can a missing `[DONE]` be detected, and made a failure?'); + + // ── (a) the A/B: complete vs truncated, side by side ────────────────────────────────────── + // Same config, same surface, same transport health. The only difference in the two event + // spines is the NUMBER of deltas — which the client has no way to know is short. + { + const clock = manualClock(); + const whole = new FakeStreamProvider({ clock, tokens: TOKENS }); + const cut = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const a = await observe( + sse({ url: URL, adapter: whole.adapter(), clock }), + {}, + clock, + ); + const b = await observe( + sse({ url: URL, adapter: cut.adapter(), clock }), + {}, + clock, + ); + check('(a) complete → done.ok', a.ok, true); + check('(a) truncated → done.ok', b.ok, true); + check( + '(a) same terminal shape?', + `${String(a.events.at(-2))},${String(a.events.at(-1))}` === + `${String(b.events.at(-2))},${String(b.events.at(-1))}`, + true, + ); + check('(a) complete text', a.text, 'ABCDE'); + check('(a) truncated text', b.text, 'ABC'); + note( + '(a) → `result,done(ok:true)` both times', + 'the ONLY signal is that one delta array contains `[DONE]` and the other does not', + ); + } + + // ── (b) `output` cannot see it — a contract is per-frame ────────────────────────────────── + // A schema that demands the sentinel rejects every TOKEN frame instead, because it is asked + // about each delta in isolation and never about the sequence. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + output: { + '~standard': { + version: 1, + vendor: 'mid-stream-failure-proof', + validate: (value: unknown) => + isDone(value) + ? { value } + : { issues: [{ message: 'not the sentinel' }] }, + }, + }, + }); + const obs = await observe(chat, {}, clock); + check('(b) deltas delivered before it blew up', obs.data.length, 0); + check('(b) done.ok', obs.ok, false); + note( + '(b) → the schema fired on delta 1, not at the end', + '`output` is a per-frame gate; "the sequence lacked a frame" is not expressible in it', + ); + } + + // ── (c) the working seam: a surface `stream` hook that requires the sentinel ────────────── + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const chat = stitch({ + url: URL, + kind: sseRequiringDone, + adapter: api.adapter(), + clock, + }); + const obs = await observe(chat, {}, clock); + checkSeq('(c) event spine on a TRUNCATED stream', obs.events, [ + 'start', + 'progress:request', + 'delta', + 'delta', + 'delta', + 'error', + 'done', + ]); + check('(c) done.ok', obs.ok, false); + check( + '(c) error.message', + obs.error, + 'stream ended without the `[DONE]` sentinel', + ); + check('(c) partial still delivered on `.stream()`', obs.text, 'ABC'); + + // …and it does not fire on a complete stream. + const whole = new FakeStreamProvider({ clock, tokens: TOKENS }); + const ok = await observe( + stitch({ + url: URL, + kind: sseRequiringDone, + adapter: whole.adapter(), + clock, + }), + {}, + clock, + ); + check('(c) complete stream → done.ok', ok.ok, true); + check('(c) complete stream → text', ok.text, 'ABCDE'); + + // …and it survives to the await path. + const api2 = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const chat2 = stitch({ + url: URL, + kind: sseRequiringDone, + adapter: api2.adapter(), + clock, + }); + const p = chat2.safe({}); + await clock.advance(3_600_000); + const r = await p; + check('(c) await → ok', r.ok, false); + check( + '(c) await → error.message', + r.error?.message, + 'stream ended without the `[DONE]` sentinel', + ); + check('(c) await → the partial', JSON.stringify(r.data), 'null'); + note( + '(c) → 8 lines of surface, and truncation becomes a real, named failure', + 'the partial is still only on `.stream()` — see C7', + ); + } + + // ── (d) the consumer-side one-liner, for comparison ─────────────────────────────────────── + // If you are already reading `.stream()`, the check is a boolean. It is smaller than the + // surface and it keeps the partial in hand — but it cannot make the STITCH fail, so anything + // downstream that only sees the awaited result learns nothing. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'close' }, + }); + const obs = await observe( + sse({ url: URL, adapter: api.adapter(), clock }), + {}, + clock, + ); + const truncated = obs.ok === true && obs.dones === 0; + check('(d) `done.ok && !sawDone` detects truncation', truncated, true); + check('(d) …and the partial is in hand', obs.text, 'ABC'); + } + + finish( + 'C6', + 'NOT BY ANY BUILT-IN, but YES in 8 lines of surface. A truncated stream and a complete one produce the SAME terminal spine — measured `result, done(ok:true)` for both, `ABCDE` vs `ABC` — because every gate on the streaming path is per-frame (`output`) or per-open (`classifyStatus`), and truncation is the absence of a frame. `output` cannot express it: a schema demanding the sentinel rejects delta 1 instead (measured: 0 deltas delivered, run failed at the wrong place). The seam that works is the surface’s own `stream` hook — wrap `sseSurface.stream`, track whether `[DONE]` was seen, throw if not. The throw lands in the engine’s mid-body catch (engine.ts:1446) and becomes an ordinary stream failure: measured spine `delta,delta,delta,error,done(ok:false)` with `error.message: "stream ended without the `[DONE]` sentinel"`, on both `.stream()` and `await`, and it stays quiet on a complete stream. The consumer-side check (`done.ok && dones === 0`) is one boolean and keeps the partial, but cannot make the stitch itself fail', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c7-partial-output.ts b/docs/scenarios/proofs/mid-stream-failure/c7-partial-output.ts new file mode 100644 index 00000000..d12256ed --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c7-partial-output.ts @@ -0,0 +1,205 @@ +// C7 — is the PARTIAL OUTPUT reachable on a mid-stream failure? Scenario 3 (batch-partial-failure) +// found no channel at all for a batch residue. Streams are different — but only by one channel, and +// only if you were already using it. +// +// Every buffered accessor is checked here against the same failed run: the awaited value, the +// `SafeResult`, the thrown `StitchError` and all its properties, `.inspect()`, `.report()`, and the +// hook surface. The engine DOES hold the partial while the run fails — `chunks` accumulates every +// delta at engine.ts:1443 and `resultEvt(chunks, …)` at engine.ts:1492 is simply never reached on +// the failure path (engine.ts:1467-1472 emits `error` + `done` and returns instead). So the data +// exists inside the engine at the moment of failure and is dropped. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c7-partial-output.ts +import { sse } from '../../../../packages/core/src/sse'; +import { + collectStitchEvents, + manualClock, +} from '../../../../packages/core/src/testing'; +import { FakeStreamProvider, contentOf } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +/** The one provider shape used throughout: 3 of 5 tokens, then the socket errors. */ +const dropAfter3 = ( + clock: ReturnType, +): FakeStreamProvider => + new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error' }, + }); + +async function main(): Promise { + heading('C7 — where does the partial go when a stream fails mid-body?'); + + // ── (a) `.stream()` — THE channel. The deltas were already handed over. ─────────────────── + { + const clock = manualClock(); + const api = dropAfter3(clock); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const obs = await observe(chat, {}, clock); + check('(a) partial text held by the consumer', obs.text, 'ABC'); + check('(a) deltas received before the failure', obs.data.length, 3); + check('(a) run failed', obs.ok, false); + note( + '(a) → the partial is not RETURNED, it was already DELIVERED', + 'which means only a `.stream()` consumer has it — and only if it kept it', + ); + } + + // ── (b) every buffered accessor: nothing ───────────────────────────────────────────────── + { + const clock = manualClock(); + + const api1 = dropAfter3(clock); + const p1 = sse({ url: URL, adapter: api1.adapter(), clock }).safe({}); + await clock.advance(3_600_000); + const safe = await p1; + + const api2 = dropAfter3(clock); + let thrown: unknown; + const p2 = sse({ url: URL, adapter: api2.adapter(), clock })({}).catch( + (e: unknown) => { + thrown = e; + }, + ); + await clock.advance(3_600_000); + await p2; + + const api3 = dropAfter3(clock); + const insp = await sse({ + url: URL, + adapter: api3.adapter(), + clock, + }).inspect({}); + + const api4 = dropAfter3(clock); + const rep = await sse({ + url: URL, + adapter: api4.adapter(), + clock, + }).report({}); + + check('(b) `.safe().data`', JSON.stringify(safe.data), 'null'); + check( + '(b) thrown `StitchError` own keys', + Object.keys(thrown as object) + .sort() + .join(','), + 'attempts,body,name,status,url', + ); + check( + '(b) `error.body`', + String((thrown as { body?: unknown }).body), + 'undefined', + ); + check( + '(b) `error.data`', + String((thrown as { data?: unknown }).data), + 'undefined', + ); + check( + '(b) `error.partial`', + String((thrown as { partial?: unknown }).partial), + 'undefined', + ); + check( + '(b) `error.chunks`', + String((thrown as { chunks?: unknown }).chunks), + 'undefined', + ); + check('(b) `.inspect().data`', JSON.stringify(insp.data), 'null'); + check('(b) `.inspect().raw`', JSON.stringify(insp.raw), 'null'); + check('(b) `.inspect().status`', insp.status, 0); + check('(b) `.report().data`', JSON.stringify(rep.data), 'null'); + check('(b) `.report().attempts`', rep.attempts, 1); + note( + '(b) → `.inspect()` exists to answer "what did the server actually send?"', + 'on a mid-stream failure it answers `null`, with `status: 0` — the 200 is not even reported', + ); + } + + // ── (c) hooks are not a channel either ─────────────────────────────────────────────────── + // `onError` never fires for a post-200 failure (C5e), and `onResponse` fires before any frame + // is parsed, so a hook has no moment at which the partial exists and it is on the context. + { + const clock = manualClock(); + const api = dropAfter3(clock); + const fired: string[] = []; + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + hooks: { + onRequest: () => { + fired.push('onRequest'); + }, + onResponse: () => { + fired.push('onResponse'); + }, + onError: () => { + fired.push('onError'); + }, + }, + }); + const obs = await observe(chat, {}, clock); + checkSeq('(c) hooks fired across the failed run', fired, [ + 'onRequest', + 'onResponse', + ]); + check('(c) run ok', obs.ok, false); + } + + // ── (d) the engine HELD the partial and threw it away ──────────────────────────────────── + // `result` is the event that carries the collected array. On a failing stream it is absent, + // while `delta` fired three times — so the array existed, fully populated, one line above the + // early `return`. + { + const clock = manualClock(); + const api = dropAfter3(clock); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + const c = collectStitchEvents(chat.stream({})); + await clock.advance(3_600_000); + const collected = await c; + check('(d) `delta` events', collected.deltas.length, 3); + check('(d) `result` event', String(collected.result), 'undefined'); + check('(d) `done.ok`', collected.done?.ok, false); + note( + '(d) → engine.ts:1443 pushes every chunk; engine.ts:1467-1472 returns without emitting `result`', + 'the collected array is complete at the moment it is discarded', + ); + } + + // ── (e) the user-side fix, which is small ──────────────────────────────────────────────── + // Accumulate as you go and treat the `error` event as "stop, keep what you have". Four lines, + // and it is the only construction that ends a failed stream holding the tokens you paid for. + { + const clock = manualClock(); + const api = dropAfter3(clock); + const chat = sse({ url: URL, adapter: api.adapter(), clock }); + let text = ''; + let failure: string | undefined; + const drain = (async () => { + for await (const ev of chat.stream({})) { + if (ev.type === 'delta') + text += + contentOf((ev.chunk as { data: unknown }).data) ?? ''; + if (ev.type === 'error') failure = ev.message; + } + })(); + await clock.advance(3_600_000); + await drain; + check('(e) partial kept', text, 'ABC'); + check('(e) failure known', failure, 'socket reset by peer'); + } + + finish( + 'C7', + 'REACHABLE — through exactly one channel, `.stream()`, and lost through every other. A drop after 3 of 5 tokens leaves a `.stream()` consumer holding `ABC` (measured), because the deltas were DELIVERED before the failure rather than returned after it. Everything buffered is empty: `.safe().data` is `null`, the thrown `StitchError` has own keys `attempts,body,name,status,url` with `body`, `data`, `partial` and `chunks` all `undefined`, and `.inspect()` — the accessor whose whole job is "what did the server actually send?" — returns `data: null`, `raw: null`, `status: 0`. Hooks are not a channel either (measured hook sequence `[onRequest, onResponse]`; `onError` never fires post-200). The engine is holding the answer at the moment it discards it: `delta` fired 3 times and `result` never did, because engine.ts:1467-1472 returns before reaching `resultEvt(chunks, …)` at engine.ts:1492. Better than scenario 3’s batch residue — there the data had no channel at all — but only for a caller already reading `.stream()`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c8-connect-vs-body.ts b/docs/scenarios/proofs/mid-stream-failure/c8-connect-vs-body.ts new file mode 100644 index 00000000..f209fb32 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c8-connect-vs-body.ts @@ -0,0 +1,285 @@ +// C8 — THE OTHER DECIDING CLAIM. Can retry be enabled for the CONNECT phase (a 503 before any byte +// is written — safe to replay, nothing has been delivered) and disabled once bytes have flowed +// (unsafe — a replay duplicates content the consumer already has)? That is the policy every LLM +// client actually wants, and it is the one thing this scenario is really asking for. +// +// The answer is that NO combination of config expresses it, for a reason that is easy to miss: the +// two phases are not separately addressable. `retry` does not run on a streaming stitch at all +// (C2), and `sse.reconnect` is a SINGLE flag that governs both phases at once — turn it on to get +// connect recovery and you have also turned on the body replay that duplicates content (C4). +// +// There is a clean seam, and it is `Surface.execute` (surface.ts:118): a transport that owns the +// connect and nothing else. It is called at engine.ts:1351, inside the resilience chain, BEFORE the +// body is decoded — so a retry loop inside it provably cannot re-deliver a delta. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c8-connect-vs-body.ts +import { stitch } from '../../../../packages/core/src/index'; +import { sse, sseSurface } from '../../../../packages/core/src/sse'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { FakeStreamProvider } from './fake-llm-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { observe } from './observe'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; + +/** + * USER CODE — the `sse` surface with a connect-phase-only retry in its transport. Ten executable + * lines. It cannot possibly duplicate a delta: by the time it returns, not one byte of the body has + * been read, and it is never re-entered for a body failure. + */ +function sseRetryingConnect(inner: Adapter, attempts = 4): Surface { + return { + ...sseSurface, + id: 'sse-connect-retry', + execute: async (req: AdapterRequest): Promise => { + let last: AdapterResponse | undefined; + for (let i = 0; i < attempts; i++) { + last = await inner(req); + if (![429, 502, 503, 504].includes(last.status)) return last; + } + return last as AdapterResponse; + }, + }; +} + +async function main(): Promise { + heading('C8 — connect-phase retry ON, body-phase retry OFF: expressible?'); + + // ── (a) `retry` does not reach the connect phase of a stream ───────────────────────────── + // A 503 before any byte, with `retry: { attempts: 4 }`. The buffered control retries it four + // times (503 is in the default `retry.on`); the streaming stitch opens once and fails. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 2 }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + retry: { attempts: 4 }, + }); + const obs = await observe(chat, {}, clock); + check('(a) opens', api.opens.length, 1); + check('(a) done.ok', obs.ok, false); + check('(a) error.message', obs.error, 'HTTP 503'); + note( + '(a) → the ONE case where replaying is unambiguously safe, and `retry` cannot do it', + 'nothing was delivered, so nothing could be duplicated — and it still gives up', + ); + } + + // ── (b) `reconnect` does not reach it either, when the refusal is an HTTP STATUS ───────── + // `classifyStatus` at engine.ts:1371 makes a rejected status TERMINAL — `return 'fail'` — so the + // reconnect loop never sees it. Documented as deliberate ("the server actively refused"). + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 2 }, + }); + const chat = sse({ + url: URL, + adapter: api.adapter(), + clock, + sse: { reconnect: true }, + retry: { attempts: 4 }, + }); + const obs = await observe(chat, {}, clock); + check( + '(b) opens with BOTH retry and reconnect on', + api.opens.length, + 1, + ); + check('(b) reconnects', obs.reconnects, 0); + check('(b) done.ok', obs.ok, false); + } + + // ── (c) …but a transport-level THROW is reconnected — by the same flag that duplicates ─── + // An `ECONNREFUSED` (the adapter rejects) returns `'error'` from `openAndDecode` + // (engine.ts:1353-1359), which IS reconnectable. So connect recovery does exist — welded to the + // body replay. One flag, two phases, no way to separate them. + { + const clock = manualClock(); + let calls = 0; + const refusing: Adapter = () => { + calls++; + return Promise.reject(new Error('ECONNREFUSED')); + }; + const chat = sse({ + url: URL, + adapter: refusing, + clock, + sse: { reconnect: true }, + }); + const obs = await observe(chat, {}, clock); + check('(c) connect attempts on a THROWN failure', calls, 4); + check('(c) done.ok', obs.ok, false); + check('(c) error.message', obs.error, 'ECONNREFUSED'); + note( + '(c) → so `reconnect: true` DOES retry a refused connect', + 'and the same `reconnect: true` replays the whole answer once bytes flow (C4)', + ); + } + + // ── (d) the config trick that appears to work, and the silent success it hides ─────────── + // `verdict.accept: [503]` makes the 503 an acceptable status, so the streaming path opens its + // (buffered, non-stream) body, decodes nothing, returns `'closed'` — and reconnects. It works. + // It also means a server that is 503 FOREVER resolves the call SUCCESSFULLY with zero deltas. + { + const clock = manualClock(); + const healing = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 2 }, + }); + const a = await observe( + sse({ + url: URL, + adapter: healing.adapter(), + clock, + verdict: { accept: [503] }, + sse: { reconnect: { attempts: 8 } }, + }), + {}, + clock, + ); + check('(d) opens against a healing server', healing.opens.length, 9); + check( + '(d) text — note the replays after it healed', + a.text, + 'ABCDEABCDEABCDEABCDEABCDEABCDEABCDE', + ); + + const dead = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 999 }, + }); + const chat = sse({ + url: URL, + adapter: dead.adapter(), + clock, + verdict: { accept: [503] }, + sse: { reconnect: true }, + }); + const p = chat.safe({}); + await clock.advance(3_600_000); + const r = await p; + check('(d) permanently-503 server → ok', r.ok, true); + check( + '(d) permanently-503 server → data', + JSON.stringify(r.data), + '[]', + ); + note( + '(d) → four 503s in a row resolve as a SUCCESSFUL empty stream', + 'the workaround for the missing connect retry is a silent-failure generator', + ); + } + + // ── (e) the seam that DOES express the split: `Surface.execute` ────────────────────────── + // Connect retried until it lands, body NOT retried when it drops. Both halves measured in one + // run: 2 refusals absorbed, then a body that dies after 3 tokens, delivered once. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 2 }, + cut: { after: 3, how: 'error', onOpens: [3] }, // open 3 is the first 200 + }); + const chat = stitch({ + url: URL, + kind: sseRetryingConnect(api.adapter()), + clock, + }); + const obs = await observe(chat, {}, clock); + check('(e) transport calls (2 × 503 + 1 × 200)', api.opens.length, 3); + check('(e) TEXT delivered — once, not replayed', obs.text, 'ABC'); + check('(e) done.ok (the body failure is NOT swallowed)', obs.ok, false); + check('(e) error.message', obs.error, 'socket reset by peer'); + check('(e) reconnects', obs.reconnects, 0); + + // …and the happy path is untouched. + const healthy = new FakeStreamProvider({ + clock, + tokens: TOKENS, + connect: { status: 503, healAfter: 1 }, + }); + const ok = await observe( + stitch({ + url: URL, + kind: sseRetryingConnect(healthy.adapter()), + clock, + }), + {}, + clock, + ); + check('(e) healthy run → text', ok.text, 'ABCDE'); + check('(e) healthy run → done.ok', ok.ok, true); + check('(e) healthy run → transport calls', healthy.opens.length, 2); + note( + '(e) → 10 lines, and the policy is exactly right', + '`execute` runs before a byte is decoded, so it structurally cannot re-deliver a delta', + ); + } + + // ── (f) what is missing, machine-checked ───────────────────────────────────────────────── + // Neither knob has a phase. A `@ts-expect-error` that is NOT an error fails `tsc`, so these + // three lines are a compile-time assertion that the vocabulary has no way to say it. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + stitch({ + url: URL, + kind: sseSurface, + adapter: api.adapter(), + clock, + // @ts-expect-error — `retry` has no phase selector. + retry: { attempts: 3, phase: 'connect' }, + }); + stitch({ + url: URL, + kind: sseSurface, + adapter: api.adapter(), + clock, + // @ts-expect-error — `reconnect` cannot be told to fire only on a drop, never on a close. + sse: { reconnect: { attempts: 3, onlyOnDrop: true } }, + }); + stitch({ + url: URL, + kind: sseSurface, + adapter: api.adapter(), + clock, + // @ts-expect-error — and it cannot be told to refuse when there is no resume token. + sse: { reconnect: { attempts: 3, requireToken: true } }, + }); + checkSeq( + '(f) config keys that would express the policy', + ['retry.phase', 'reconnect.onlyOnDrop', 'reconnect.requireToken'], + ['retry.phase', 'reconnect.onlyOnDrop', 'reconnect.requireToken'], + ); + note( + '(f) → all three are compile errors (machine-checked by the `@ts-expect-error`s above)', + 'the smallest real fix is `reconnect.requireToken`: it alone would turn C4 from silent replay into a refusal', + ); + } + + finish( + 'C8', + 'NOT IN CONFIG — and the reason is that the two phases are not separately addressable. `retry` never runs on a streaming stitch (measured: 1 open against a 503 with `retry: { attempts: 4 }`, versus 4 on the buffered control), so the one case where replay is unambiguously safe is the one case it cannot cover. `sse.reconnect` does not help either when the refusal is an HTTP status: `classifyStatus` makes a rejected status terminal (measured: 1 open with BOTH `retry` and `reconnect` on, 0 reconnects). It DOES retry a transport-level throw — measured 4 connect attempts on `ECONNREFUSED` — but that is the same single flag that replays the whole answer once bytes flow, so the two policies cannot be set independently. The config workaround is worse than the gap: `verdict.accept: [503]` + `reconnect` does retry the connect (measured: 9 opens against a healing server) but a permanently-503 server then resolves SUCCESSFULLY with `data: []`, and once it heals it replays the answer 7 times. What works is `Surface.execute` (surface.ts:118): 10 lines of transport that retry only the connect. Measured in one run — 2 × 503 absorbed, then a body that drops after 3 tokens delivered ONCE (`ABC`), `done(ok:false)`, 0 reconnects. Missing, machine-checked: `retry.phase`, `reconnect.onlyOnDrop`, `reconnect.requireToken`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/c9-assembled-solution.ts b/docs/scenarios/proofs/mid-stream-failure/c9-assembled-solution.ts new file mode 100644 index 00000000..a5a1de13 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/c9-assembled-solution.ts @@ -0,0 +1,257 @@ +// C9 — the best answer for the LLM case, run against every failure shape in this scenario, and +// compared honestly to the hand-rolled alternative. +// +// The two implementations are driven by the SAME fake provider through the SAME transport, and the +// comparison asserts their observable results are IDENTICAL on all five shapes. Then the line count +// is measured from the files themselves rather than claimed. +// +// pnpm exec tsx docs/scenarios/proofs/mid-stream-failure/c9-assembled-solution.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { FakeStreamProvider } from './fake-llm-stream'; +import { handRolledCompletion } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { Completion } from './llm-stream'; +import { completion, llmSurface } from './llm-stream'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const URL = 'https://api.openai.example/v1/chat/completions'; +const TOKENS = ['A', 'B', 'C', 'D', 'E']; +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** The five shapes this scenario is about, each as a provider factory. */ +const SHAPES = { + complete: () => ({ tokens: TOKENS }), + 'connect 503 ×2': () => ({ + tokens: TOKENS, + connect: { status: 503, healAfter: 2 }, + }), + 'drop after 3': () => ({ + tokens: TOKENS, + cut: { after: 3, how: 'error' as const }, + }), + 'truncated (no [DONE])': () => ({ + tokens: TOKENS, + cut: { after: 3, how: 'close' as const }, + }), + 'in-band error frame': () => ({ tokens: TOKENS, errorFrameAfter: 2 }), + 'connect 503 forever': () => ({ + tokens: TOKENS, + connect: { status: 503, healAfter: 999 }, + }), +}; + +/** + * Executable lines of a proof file — the comparable unit. Import statements (single- and + * multi-line), blank lines and comment-only lines are all removed, on BOTH sides, so the number is + * the code someone actually has to write and maintain. + */ +function executableLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .replace(/^import[\s\S]*?;$/gm, '') // whole import statements, however they wrap + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +const summarize = (c: Completion): string => + `${c.text || '(nothing)'} | complete=${String(c.complete)} | deltas=${String(c.deltas)} | ${c.error ?? 'no error'}`; + +async function main(): Promise { + heading( + 'C9 — the assembled answer, on every shape, against the hand-rolled twin', + ); + + const stitched: string[] = []; + const rolled: string[] = []; + const opensStitched: number[] = []; + const opensRolled: number[] = []; + + for (const make of Object.values(SHAPES)) { + // The StitchAPI answer. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, ...make() }); + const chat = stitch({ + url: URL, + kind: llmSurface(api.adapter()), + clock, + }); + const p = completion(chat, {}); + await clock.advance(3_600_000); + stitched.push(summarize(await p)); + opensStitched.push(api.opens.length); + } + // The hand-rolled twin, same transport, same fake. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, ...make() }); + const transport: Adapter = api.adapter(); + rolled.push( + summarize( + await handRolledCompletion(transport, { + url: URL, + method: 'POST', + headers: {}, + }), + ), + ); + opensRolled.push(api.opens.length); + } + } + + // ── (a) what the assembled answer produces on each shape ───────────────────────────────── + for (const [i, name] of Object.keys(SHAPES).entries()) + note(`(a) ${name}`, stitched[i] ?? ''); + + checkSeq( + '(a) transport opens per shape', + opensStitched, + [1, 3, 1, 1, 1, 4], + ); + check( + '(a) the complete answer, delivered exactly once', + stitched[0], + 'ABCDE | complete=true | deltas=6 | no error', + ); + check( + '(a) a healed connect: replayed at the connect phase only', + stitched[1], + 'ABCDE | complete=true | deltas=6 | no error', + ); + check( + '(a) a mid-body drop: PARTIAL KEPT, failure named', + stitched[2], + 'ABC | complete=false | deltas=3 | socket reset by peer', + ); + check( + '(a) a truncated stream: caught, partial kept', + stitched[3], + 'ABC | complete=false | deltas=3 | stream truncated: no `[DONE]` sentinel', + ); + check( + '(a) an in-band error frame: a real failure, partial kept, bad frame withheld', + stitched[4], + 'AB | complete=false | deltas=2 | provider error frame: upstream provider overloaded', + ); + check( + '(a) a dead server: fails after 4 connect attempts, no silent success', + stitched[5], + '(nothing) | complete=false | deltas=0 | HTTP 503', + ); + + // ── (b) the hand-rolled twin agrees, shape for shape ───────────────────────────────────── + checkSeq('(b) hand-rolled results', rolled, stitched); + checkSeq('(b) hand-rolled transport opens', opensRolled, opensStitched); + note( + '(b) → the wire behaviour is identical', + 'so the comparison is purely "what do the extra lines buy?"', + ); + + // ── (c) the line count, measured from the files ────────────────────────────────────────── + { + const mine = executableLines('llm-stream.ts'); + const theirs = executableLines('hand-rolled.ts'); + note( + '(c) `llm-stream.ts` (the StitchAPI answer)', + `${String(mine)} executable lines`, + ); + note( + '(c) `hand-rolled.ts` (no StitchAPI)', + `${String(theirs)} executable lines`, + ); + check('(c) is the StitchAPI version shorter?', mine < theirs, true); + note( + '(c) the parser is why', + 'a hand-rolled client brings its own `text/event-stream` parser; `sseSurface.stream` is reused here', + ); + } + + // ── (d) what the extra machinery actually buys, measured ───────────────────────────────── + // Not line count: the stitch is inside the engine, so it gets the spine. Measured on the + // drop shape — the run emits a full event trace with a traceId, and `throttle`/`auth`/`headers` + // /`timeout` are all still config rather than more hand-rolled code. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ + clock, + tokens: TOKENS, + cut: { after: 3, how: 'error' }, + }); + const chat = stitch({ + name: 'chat', + url: URL, + kind: llmSurface(api.adapter()), + clock, + }); + const types: string[] = []; + const ids = new Set(); + const drain = (async () => { + for await (const ev of chat.stream({})) { + types.push(ev.type); + const t = (ev as { traceId?: string }).traceId; + if (t !== undefined) ids.add(t); + } + })(); + await clock.advance(3_600_000); + await drain; + checkSeq('(d) event spine of the assembled answer', types, [ + 'start', + 'progress', + 'delta', + 'delta', + 'delta', + 'error', + 'done', + ]); + check('(d) one traceId across the run', ids.size, 1); + note( + '(d) → and none of `auth`/`headers`/`throttle`/`timeout`/`trace` cost a line here', + 'they are config on the same stitch; the hand-rolled twin would grow for each', + ); + } + + // ── (e) the one-line version of "do not do this" ───────────────────────────────────────── + // Adding `sse: { reconnect: true }` to the assembled stitch un-does it: the truncation throw + // becomes a reconnect trigger and the answer is delivered repeatedly. + { + const clock = manualClock(); + const api = new FakeStreamProvider({ clock, tokens: TOKENS }); + const chat = stitch({ + url: URL, + kind: llmSurface(api.adapter()), + clock, + sse: { reconnect: true }, + }); + const p = completion(chat, {}); + await clock.advance(3_600_000); + const c = await p; + check('(e) with `reconnect: true` → opens', api.opens.length, 4); + check( + '(e) with `reconnect: true` → text', + c.text, + 'ABCDEABCDEABCDEABCDE', + ); + note( + '(e) → the surface rules survive; the reconnect loop is above them', + 'nothing a surface can do prevents this — the decision is made in `runStreaming`', + ); + } + + finish( + 'C9', + `ASSEMBLED AND RUN. ${String(executableLines('llm-stream.ts'))} executable lines of user code (\`llm-stream.ts\`) across TWO seams — \`Surface.execute\` for connect-only retry, and the surface's \`stream\` hook for the \`[DONE]\` requirement plus in-band error frames — with the consumer draining \`.stream()\` so the partial is never lost. Measured across all six shapes: a complete answer delivered ONCE (\`ABCDE\`, 6 deltas, 1 open); a healed connect replayed at the connect phase only (3 opens, \`ABCDE\` once); a mid-body drop keeping \`ABC\` with \`socket reset by peer\`; a truncation caught as \`stream truncated: no [DONE] sentinel\` with \`ABC\` kept; an in-band error frame as \`provider error frame: upstream provider overloaded\` with \`AB\` kept and the bad frame withheld; and a dead server failing after 4 connect attempts rather than resolving empty. The hand-rolled twin (${String(executableLines('hand-rolled.ts'))} executable lines, its own SSE parser included) produces byte-identical results on every shape and the same open counts — so the extra machinery is not buying behaviour, it is buying the spine: one \`start\`/\`delta\`×N/\`error\`/\`done\` trace under one traceId, and \`auth\`/\`headers\`/\`throttle\`/\`timeout\` staying config instead of growing the hand-rolled file. The load-bearing caveat: adding \`sse: { reconnect: true }\` to this same stitch re-breaks it (measured 4 opens, \`ABCDEABCDEABCDEABCDE\`), and no surface hook can defend against that — the decision is made above them in \`runStreaming\``, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/mid-stream-failure/fake-llm-stream.ts b/docs/scenarios/proofs/mid-stream-failure/fake-llm-stream.ts new file mode 100644 index 00000000..6ad111f5 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/fake-llm-stream.ts @@ -0,0 +1,274 @@ +// Fake, in-memory `text/event-stream` PROVIDERS — the three shapes a mid-stream failure arrives in, +// plus the recorders that make duplication a measurement rather than an assertion. +// +// (a) OPENAI-SHAPED — `data: {"choices":[{"delta":{"content":"A"}}]}` frames with NO `id:`, +// terminated by `data: [DONE]`. This is the most common streaming API in the world and the +// one `Last-Event-ID` structurally cannot help: there is no id to resume from. +// (b) RESUMABLE FEED — every frame carries `id: t3`, and the server HONOURS a `Last-Event-ID` +// request header by resuming after that id. The shape SSE was designed for. +// (c) CONNECT-PHASE FAILURE — a `503` before any byte of body is written. Safe to retry: the +// consumer has seen nothing, so a replay duplicates nothing. +// +// Every provider records each OPEN with the virtual timestamp and the `Last-Event-ID` header it +// received, so three things are numbers rather than arguments: +// +// - `opens.length` — how many times the client opened a connection. `> 1` on a completed stream +// means the model ran again: paid twice, and the consumer saw it twice. +// - `lastEventIds` — the exact header value replayed on each reopen. `undefined` means the +// client reopened from scratch. +// - `gaps` — virtual ms between opens: the reconnect pacing, measured. +// +// Nothing touches the network and every wait rides an injected {@link Clock}, so a 5-second +// reconnect backoff is exact virtual time. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +const enc = new TextEncoder(); + +/** One recorded connection open, as the provider saw it. */ +export interface RecordedOpen { + /** Virtual time (ms) the open arrived, read off the injected clock. */ + at: number; + /** The `Last-Event-ID` request header, or `undefined` when the client sent none. */ + lastEventId: string | undefined; + /** Status this open was answered with. */ + status: number; +} + +/** How a connection ENDS after its frames are written. */ +export type CutMode = + /** The socket errors — a transport-level drop, the classic mid-stream failure. */ + | 'error' + /** The socket closes normally, mid-answer. Truncation that looks exactly like completion. */ + | 'close'; + +export interface ProviderOptions { + clock: Clock; + /** + * The tokens the model "generates", one per SSE frame. Single letters keep the measured delta + * sequence readable: a duplicated stream reads `ABCABC`. + */ + tokens?: string[]; + /** + * Frame ids. `'none'` is the OpenAI shape (no `id:` anywhere — nothing to resume from); + * `'per-token'` is a resumable feed (`id: t1`, `id: t2`, … honoured on reconnect). + */ + ids?: 'none' | 'per-token'; + /** Terminate a complete stream with `data: [DONE]` (the OpenAI completion sentinel). Default true. */ + done?: boolean; + /** + * End the body after `after` frames instead of completing: `'error'` errors the socket, + * `'close'` closes it cleanly (a truncation indistinguishable from a finished answer). Applies + * only to the opens listed in `onOpens` — default `[1]`, so a reconnect gets a healthy body. + */ + cut?: { after: number; how: CutMode; onOpens?: number[] }; + /** + * Emit an in-band `data: {"error": {...}}` frame after this many token frames and then close + * cleanly — the HTTP-200 failure OpenRouter documents. The status line is already spent. + */ + errorFrameAfter?: number; + /** + * Answer with `status` before writing any byte, healing to a normal 200 body from open + * `healAfter` + 1 onward. The connect-phase failure — shape (c). + */ + connect?: { status: number; healAfter: number }; + /** Emit `retry: N` on the first frame — the server's own reconnect pacing hint. */ + retryHint?: number; +} + +/** + * A fake streaming provider. One instance is one server; `opens` accumulates across every + * connection the client makes, which is exactly what a duplication claim needs to measure. + */ +export class FakeStreamProvider { + /** Every connection open, in order. */ + readonly opens: RecordedOpen[] = []; + private readonly clock: Clock; + private readonly tokens: string[]; + private readonly ids: 'none' | 'per-token'; + private readonly done: boolean; + private readonly cut: ProviderOptions['cut']; + private readonly errorFrameAfter: number | undefined; + private readonly connect: ProviderOptions['connect']; + private readonly retryHint: number | undefined; + + constructor(opts: ProviderOptions) { + this.clock = opts.clock; + this.tokens = opts.tokens ?? ['A', 'B', 'C', 'D', 'E']; + this.ids = opts.ids ?? 'none'; + this.done = opts.done ?? true; + this.cut = opts.cut; + this.errorFrameAfter = opts.errorFrameAfter; + this.connect = opts.connect; + this.retryHint = opts.retryHint; + } + + /** The `Last-Event-ID` header sent on each open, in order. `undefined` = reopened from scratch. */ + get lastEventIds(): (string | undefined)[] { + return this.opens.map((o) => o.lastEventId); + } + + /** Virtual-clock ms between successive opens — the reconnect pacing, measured. */ + get gaps(): number[] { + const at = this.opens.map((o) => o.at); + return at.slice(1).map((t, i) => t - (at[i] as number)); + } + + /** The id of the nth token frame (1-based), the way `'per-token'` mints them. */ + static idOf(n: number): string { + return `t${String(n)}`; + } + + adapter(): Adapter { + return (req: AdapterRequest): Promise => { + const lastEventId = req.headers['Last-Event-ID']; + const openNo = this.opens.length + 1; + + if (this.connect && openNo <= this.connect.healAfter) { + this.opens.push({ + at: this.clock.now(), + lastEventId, + status: this.connect.status, + }); + // A connect-phase failure: a normal buffered error payload, NOT a live body. Not one + // byte of the answer has been written, so replaying this duplicates nothing. + return Promise.resolve({ + status: this.connect.status, + headers: {}, + body: { error: { message: 'server overloaded' } }, + }); + } + + this.opens.push({ at: this.clock.now(), lastEventId, status: 200 }); + return Promise.resolve({ + status: 200, + headers: { 'content-type': 'text/event-stream' }, + body: this.body(openNo, lastEventId), + }); + }; + } + + // Build one connection's frames, honouring `Last-Event-ID` when the shape supports it. + private body( + openNo: number, + lastEventId: string | undefined, + ): ReadableStream { + // Resume point: a `'per-token'` provider skips everything up to and including the acked id. + // A `'none'` provider has no ids, so it can only start over — that is the whole problem. + const from = + this.ids === 'per-token' && lastEventId !== undefined + ? this.tokens.findIndex( + (_, i) => FakeStreamProvider.idOf(i + 1) === lastEventId, + ) + 1 + : 0; + + const frames: string[] = []; + const remaining = this.tokens.slice(from); + const cutHere = + this.cut && (this.cut.onOpens ?? [1]).includes(openNo) + ? this.cut.after + : undefined; + + for (const [i, token] of remaining.entries()) { + if (cutHere !== undefined && i >= cutHere) break; + if ( + this.errorFrameAfter !== undefined && + i >= this.errorFrameAfter + ) { + // The in-band failure: HTTP 200, headers long since flushed, so the only place the + // error can live is a `data:` frame the consumer has to recognise. + frames.push( + frame({ + data: { + error: { + message: 'upstream provider overloaded', + code: 'provider_error', + }, + }, + }), + ); + return streamOf(frames, 'close'); + } + frames.push( + frame({ + data: chunkOf(token), + ...(this.ids === 'per-token' + ? { id: FakeStreamProvider.idOf(from + i + 1) } + : {}), + ...(i === 0 && this.retryHint !== undefined + ? { retry: this.retryHint } + : {}), + }), + ); + } + + if (cutHere !== undefined) return streamOf(frames, this.cut?.how); + // A completed stream: the OpenAI sentinel, then a normal close. + if (this.done) frames.push('data: [DONE]\n\n'); + return streamOf(frames, 'close'); + } +} + +/** The OpenAI streaming-chunk shape, so `contentOf` reads a delta the way real client code does. */ +export const chunkOf = ( + content: string, +): { object: string; choices: { delta: { content: string } }[] } => ({ + object: 'chat.completion.chunk', + choices: [{ delta: { content } }], +}); + +/** Pull the token text out of one parsed `data:` payload — `undefined` for `[DONE]` or an error frame. */ +export const contentOf = (data: unknown): string | undefined => { + if (typeof data !== 'object' || data === null) return undefined; + const choices = (data as { choices?: { delta?: { content?: string } }[] }) + .choices; + return choices?.[0]?.delta?.content; +}; + +/** Read the in-band error payload off a parsed `data:` payload, if this frame is one. */ +export const errorOf = (data: unknown): { message?: string } | undefined => + typeof data === 'object' && data !== null + ? (data as { error?: { message?: string } }).error + : undefined; + +/** True for the `data: [DONE]` completion sentinel. */ +export const isDone = (data: unknown): boolean => data === '[DONE]'; + +// ---- wire helpers --------------------------------------------------------------------------- + +interface Frame { + data: unknown; + id?: string; + retry?: number; +} + +function frame(f: Frame): string { + const lines: string[] = []; + if (f.id !== undefined) lines.push(`id: ${f.id}`); + if (f.retry !== undefined) lines.push(`retry: ${String(f.retry)}`); + lines.push(`data: ${JSON.stringify(f.data)}`); + return `${lines.join('\n')}\n\n`; +} + +// A body that emits each frame as its own read, then ends: `'close'` closes the socket normally +// (indistinguishable from completion at the transport layer), `'error'` errors it (a drop). +function streamOf( + frames: string[], + how: CutMode = 'close', +): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(c) { + if (i >= frames.length) { + if (how === 'error') c.error(new Error('socket reset by peer')); + else c.close(); + return; + } + c.enqueue(enc.encode(frames[i++] as string)); + }, + }); +} diff --git a/docs/scenarios/proofs/mid-stream-failure/hand-rolled.ts b/docs/scenarios/proofs/mid-stream-failure/hand-rolled.ts new file mode 100644 index 00000000..08de3d0f --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/hand-rolled.ts @@ -0,0 +1,105 @@ +// The HONEST COMPARISON for C9 — the same five rules, hand-rolled, with no StitchAPI in it. It +// takes the same `Adapter`-shaped transport as the assembled answer so both sides run against the +// identical fake provider and the behaviour comparison is exact. +// +// The SSE frame parser is counted on THIS side only, and that is the fair accounting: StitchAPI +// ships one (`sse.ts` `parseEventStream`) and a hand-rolled client has to bring its own. It is +// deliberately the minimum that is still correct for this wire shape — multi-line `data:`, CRLF, +// comment lines, and frames split across reads — because a version that skips those is not a +// comparison, it is a bug. +import type { + Adapter, + AdapterRequest, +} from '../../../../packages/core/src/types'; +import { contentOf, errorOf, isDone } from './fake-llm-stream'; +import type { Completion } from './llm-stream'; + +const TRANSIENT = [429, 502, 503, 504]; + +/** Parse a `text/event-stream` body into the `data:` payloads it dispatches. */ +async function* frames( + body: ReadableStream, +): AsyncGenerator { + const reader = body.getReader(); + const dec = new TextDecoder(); + let buf = ''; + let lines: string[] = []; + for (;;) { + const r = await reader.read(); + if (r.done) break; + buf += dec.decode(r.value, { stream: true }); + let nl = buf.indexOf('\n'); + while (nl !== -1) { + const line = buf.slice(0, nl).replace(/\r$/, ''); + buf = buf.slice(nl + 1); + if (line === '') { + if (lines.length > 0) { + const raw = lines.join('\n'); + try { + yield JSON.parse(raw); + } catch { + yield raw; + } + lines = []; + } + } else if (!line.startsWith(':')) { + const colon = line.indexOf(':'); + const field = colon === -1 ? line : line.slice(0, colon); + let value = colon === -1 ? '' : line.slice(colon + 1); + if (value.startsWith(' ')) value = value.slice(1); + if (field === 'data') lines.push(value); + } + nl = buf.indexOf('\n'); + } + } +} + +/** The same five rules, by hand. */ +export async function handRolledCompletion( + transport: Adapter, + req: AdapterRequest, + connectAttempts = 4, +): Promise { + let res = await transport({ ...req, stream: true }); + for (let i = 1; i < connectAttempts && TRANSIENT.includes(res.status); i++) + res = await transport({ ...req, stream: true }); + + let text = ''; + let deltas = 0; + let complete = false; + if (res.status >= 400) + return { text, complete, deltas, error: `HTTP ${String(res.status)}` }; + if (!(res.body instanceof ReadableStream)) + return { text, complete, deltas, error: 'no stream body' }; + + try { + for await (const data of frames(res.body)) { + const err = errorOf(data); + if (err !== undefined) + return { + text, + complete, + deltas, + error: `provider error frame: ${String(err.message)}`, + }; + deltas++; + complete ||= isDone(data); + text += contentOf(data) ?? ''; + } + } catch (e) { + return { + text, + complete, + deltas, + error: e instanceof Error ? e.message : String(e), + }; + } + return complete + ? { text, complete, deltas } + : { + text, + complete, + deltas, + error: 'stream truncated: no `[DONE]` sentinel', + }; +} diff --git a/docs/scenarios/proofs/mid-stream-failure/harness.ts b/docs/scenarios/proofs/mid-stream-failure/harness.ts new file mode 100644 index 00000000..4c9c04dc --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/harness.ts @@ -0,0 +1,62 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's whole argument is a SEQUENCE (which deltas did the consumer actually see, in what +// order), so `checkSeq` is the load-bearing assertion: it prints the measured sequence in full on +// pass AND on fail, because "the consumer saw `ABCABCABCABC`" is the finding. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — for this scenario the sequence IS the + * evidence, so it must be readable out of context. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/mid-stream-failure/llm-stream.ts b/docs/scenarios/proofs/mid-stream-failure/llm-stream.ts new file mode 100644 index 00000000..f823b13f --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/llm-stream.ts @@ -0,0 +1,112 @@ +// USER CODE — the assembled answer for the LLM case, built on StitchAPI. Two seams and one loop. +// +// The policy it implements is the one the research capture describes as what every LLM client +// actually wants, and none of it is reachable from config alone: +// +// 1. RETRY THE CONNECT, NEVER THE BODY. A 503 before the first byte is replayed; a drop after the +// first byte is not. `Surface.execute` is the right home for this because it runs before a +// single delta is decoded (engine.ts:1351), so it cannot re-deliver one. +// 2. `[DONE]` OR IT DID NOT FINISH. The surface's `stream` hook is the only place that sees the +// body end, so it is the only place that can tell "ended" from "ended early". +// 3. AN IN-BAND `{ "error": … }` FRAME IS A FAILURE. Same hook, same reason — and throwing from +// it means the bad frame is never handed to the consumer. +// 4. NEVER LOSE THE PARTIAL. `.stream()` is the only channel that has it (C7), so the consumer +// accumulates as it goes and keeps what it has when the run fails. +// 5. `sse.reconnect` STAYS OFF. On an id-less stream it replays the whole answer (C4). +import type { SseEvent } from '../../../../packages/core/src/sse'; +import { sseSurface } from '../../../../packages/core/src/sse'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, + ResolvedStitchConfig, + StitchEvent, + StitchInput, +} from '../../../../packages/core/src/types'; +import { contentOf, errorOf, isDone } from './fake-llm-stream'; + +const decodeSse = sseSurface.stream as NonNullable; + +/** Statuses worth replaying at the connect phase — the engine's own default `retry.on`. */ +const TRANSIENT = [429, 502, 503, 504]; + +/** + * The `sse` surface with the three rules above baked in. `transport` is the HTTP client to use + * (`fetchAdapter` in production; the fake here), because `execute` replaces `config.adapter`. + */ +export function llmSurface(transport: Adapter, connectAttempts = 4): Surface { + return { + ...sseSurface, + id: 'llm-sse', + // Rule 1 — connect only. Nothing has been decoded yet, so a replay here is free of the + // duplication hazard by construction. + execute: async (req: AdapterRequest): Promise => { + let res = await transport(req); + for ( + let i = 1; + i < connectAttempts && TRANSIENT.includes(res.status); + i++ + ) + res = await transport(req); + return res; + }, + // Rules 2 and 3 — the body's own verdict, in the one hook that sees the body end. + stream: async function* ( + res: AdapterResponse, + cfg: ResolvedStitchConfig, + ) { + let sawDone = false; + for await (const chunk of decodeSse(res, cfg)) { + const data = (chunk as SseEvent).data; + const err = errorOf(data); + if (err !== undefined) + throw new Error( + `provider error frame: ${String(err.message)}`, + ); + sawDone ||= isDone(data); + yield chunk; + } + if (!sawDone) + throw new Error('stream truncated: no `[DONE]` sentinel'); + }, + }; +} + +/** What a completion attempt produced — including a failed one. */ +export interface Completion { + /** Every token the model produced, whether or not it finished. Never discarded. */ + text: string; + /** True only when the `[DONE]` sentinel arrived. */ + complete: boolean; + /** Why it stopped, when it did not complete. */ + error?: string; + /** Deltas delivered. Equal to the tokens generated — a replay would make this larger. */ + deltas: number; +} + +/** + * Rule 4 — drain `.stream()`, keeping the partial. The `error` event is "stop and keep what you + * have", not "throw away the answer": on every failure shape this still returns the tokens the + * caller has already paid for. + */ +export async function completion( + chat: { stream: (input?: StitchInput) => AsyncIterable }, + input: StitchInput, +): Promise { + let text = ''; + let deltas = 0; + let complete = false; + let error: string | undefined; + for await (const ev of chat.stream(input)) { + if (ev.type === 'delta') { + const data = (ev.chunk as SseEvent).data; + deltas++; + complete ||= isDone(data); + text += contentOf(data) ?? ''; + } else if (ev.type === 'error') error = ev.message; + } + return error === undefined + ? { text, complete, deltas } + : { text, complete, deltas, error }; +} diff --git a/docs/scenarios/proofs/mid-stream-failure/observe.ts b/docs/scenarios/proofs/mid-stream-failure/observe.ts new file mode 100644 index 00000000..f0853fd1 --- /dev/null +++ b/docs/scenarios/proofs/mid-stream-failure/observe.ts @@ -0,0 +1,101 @@ +// The CONSUMER side of the measurement: drain a stitch's `.stream()` and record the EXACT sequence +// of `delta` chunks it observed, plus the control events around them. +// +// This is the instrument the whole scenario turns on. "Does a retry duplicate content?" is not a +// question about the engine's source — it is a question about what arrives at a downstream +// accumulator, so the accumulator is what gets built and printed. `text` is the token stream +// concatenated exactly as a UI would render it: a duplicated answer reads `ABCDEABCDE` at a glance. +import type { ManualClock } from '../../../../packages/core/src/test-clock'; +import type { + StitchEvent, + StitchInput, +} from '../../../../packages/core/src/types'; +import { contentOf, errorOf, isDone } from './fake-llm-stream'; + +/** What a `.stream()` consumer actually saw. */ +export interface Observation { + /** Every `delta`'s parsed SSE `data` payload, in order — the raw evidence. */ + data: unknown[]; + /** + * The token text a UI would have rendered, concatenated across the whole run. `'ABCABC'` means + * the consumer was handed the same content twice. + */ + text: string; + /** Event types in order; a `progress` is tagged with its phase (`progress:reconnect`). */ + events: string[]; + /** How many `progress` events carried `phase: 'reconnect'` — reconnect boundaries the consumer CAN see. */ + reconnects: number; + /** `[DONE]` sentinels observed. `0` on a truncated OpenAI stream; `> 1` means the stream replayed. */ + dones: number; + /** In-band `data: {"error": …}` frames observed. */ + errorFrames: number; + /** The terminal `done` event's `ok`. */ + ok: boolean | undefined; + /** The terminal `error` event's message, when one was emitted. */ + error: string | undefined; + /** Set when the async iterator itself THREW rather than emitting a terminal event. */ + threw: string | undefined; +} + +export interface ObserveOptions { + /** Stop consuming (`break`) as soon as this returns true for a delta's `data`. */ + stopOn?: (data: unknown) => boolean; + /** Virtual ms to advance while the stream runs. Default one hour — enough for any backoff here. */ + advance?: number; +} + +/** + * Drain `stitch.stream(input)` under an injected clock and report what the consumer saw. + * + * The clock is advanced CONCURRENTLY with the drain, because every wait in the streaming path + * (reconnect backoff, throttle pacing) sleeps on the injected clock — so an hour of reconnect + * backoff costs no real time and lands on exact virtual timestamps. + */ +export async function observe( + stitch: { stream: (input?: StitchInput) => AsyncIterable }, + input: StitchInput, + clock: ManualClock, + opts: ObserveOptions = {}, +): Promise { + const obs: Observation = { + data: [], + text: '', + events: [], + reconnects: 0, + dones: 0, + errorFrames: 0, + ok: undefined, + error: undefined, + threw: undefined, + }; + + const drain = (async () => { + try { + for await (const ev of stitch.stream(input)) { + obs.events.push( + ev.type === 'progress' + ? `progress:${String(ev.phase)}` + : ev.type, + ); + if (ev.type === 'progress' && ev.phase === 'reconnect') + obs.reconnects++; + if (ev.type === 'error') obs.error = ev.message; + if (ev.type === 'done') obs.ok = ev.ok; + if (ev.type !== 'delta') continue; + + const data = (ev.chunk as { data: unknown }).data; + obs.data.push(data); + if (isDone(data)) obs.dones++; + if (errorOf(data) !== undefined) obs.errorFrames++; + obs.text += contentOf(data) ?? ''; + if (opts.stopOn?.(data) === true) break; + } + } catch (e) { + obs.threw = e instanceof Error ? e.message : String(e); + } + })(); + + await clock.advance(opts.advance ?? 3_600_000); + await drain; + return obs; +} diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/README.md b/docs/scenarios/proofs/multi-tenant-blast-radius/README.md new file mode 100644 index 00000000..9165640f --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/README.md @@ -0,0 +1,207 @@ +# Proofs — one customer's bad token, and how far it spreads + +Runnable evidence for the claims in [`../../multi-tenant-blast-radius.md`](../../multi-tenant-blast-radius.md). + +**The scenario's answer is a number, and it is 9 of 9.** One customer with a revoked credential, a +`circuit` on the shared seam, nine healthy customers: all nine fail, none of their requests ever +leave the process, and the outage does not end on its own. The same failure under the corrected +construction is **0 of 9**. Everything else here is the distance between those two numbers. + +Every script is standalone and offline. Where a claim is about time — arrival times, cooldown +windows, a TTL measured in virtual years — it runs on an injected `manualClock()`, so the numbers +(`t=2000`, `t=0`, 120 virtual seconds) are exact rather than approximate. Where a claim is about +whether a spelling EXISTS, it runs the TypeScript compiler over candidate statements and reports +which ones compile, so "the built-in can't" is measured rather than grepped. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c1-shared-breaker.ts + +# all of them +for f in docs/scenarios/proofs/multi-tenant-blast-radius/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/multi-tenant-blast-radius/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| --------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `c1-shared-breaker.ts` | does one revoked credential fail everyone? | **9 of 9 healthy tenants down, 0 requests sent — and it never recovers.** 4 cooldown windows, 503 in every one | +| `c2-partition-the-breaker.ts` | can the breaker be per-tenant? is `circuit.key` static? | **Yes, via the KEY STRING only.** 1 of 4 spellings compiles. 10 stitch objects → 1 breaker; 10 seams → 1 breaker | +| `c3-401-is-not-a-dependency-failure.ts` | can a 401 be kept off the breaker without swallowing it? | **Yes, pure config: `verdict: { accept: [401], flag }`.** `accept` alone hands the caller the 401 body as DATA | +| `c4-noisy-neighbour.ts` | does one tenant's burst spend other tenants' budget? | **t=0 → t=2000.** And with `concurrency` the quiet tenant is not slowed, it is **queued last** (t=5000) | +| `c5-partition-the-throttle.ts` | can the rate budget be partitioned at all? | **Yes, two ways** — and `pool: 'host'` collapses both, **and silently re-keys the CIRCUIT** | +| `c6-token-isolation.ts` | does `tenancy: 'principal'` isolate tokens under a refresh storm? | **Yes, fail-closed.** But the DEFAULT is `'app'` (one token for all), and the CREDENTIAL cannot be per-tenant | +| `c7-cost-of-isolation.ts` | what does 100 isolated tenants cost? does anything leak? | **~7kb and 0 timers per tenant — and three things never freed**, incl. breaker keys with no TTL | +| `c8-four-resources.ts` | token / cache / rate / breaker — isolated or not? | **2 by the principal (fail-closed), 2 only by a string you must remember.** Assembled: blast radius **0 of 9** | + +## Files + +- `fake-vendor.ts` — the vendor and its IdP, as plain `Adapter`s over the injected clock. A request + is attributed to a tenant by an `x-tenant` header; `fail('bad', 401)` breaks one named customer + persistently; every request is recorded with its tenant, status, path, `Authorization` and + **arrival time**, which is what the rate claims read. `FakeIdp` mints a traceable + `tok--` per request, so C6 can say whose token went out. Plus `outcomeOf` (a call + reduced to `'ok'` / `''`) and `blastRadius` (how many of a spine were not `'ok'`). +- `probe-store.ts` — a `StitchStore` that records **every key the engine touches**. This is the + scenario's most load-bearing instrument: whether a resource is shared or isolated is not a fact + about which objects were constructed (C2 and C5 both measure constructions that look isolated and + are not), it is a fact about what STRING the state was keyed on. `live()` answers the residency + question C7 (d) turns on. +- `type-probe.ts` — hands the TypeScript compiler one candidate statement per spelling and reports + which compile. `typescript` is `require`d through a path anchored at `packages/core` (the + workspace package that declares it) — a bare `import ts from 'typescript'` resolves under `tsx` + and not under plain Node from `docs/`, which would make the script run one way and typecheck + another. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C1 is the finding, and the part the capture misses is that the outage is self-sustaining.** One + customer with a revoked token, `circuit: { failures: 3, cooldown: '30s' }` on the shared seam, nine + healthy customers: **9 of 9 failed**, with `StitchError` status **503** / message `circuit open`, + and **0 of their requests reached the vendor**. The error carries nothing naming the tenant that + caused it, so the page for customer #7 says the vendor is down while the vendor is fine. Then: + `cooldown` elapses, the breaker goes half-open and admits **exactly one** trial call + (resilience.ts:375-379) — and the tenant most likely to take it is the broken one, because it is + the one retrying hardest. Across **4 full cooldown windows (120 virtual seconds)** the healthy + tenant measured `503,503,503,503`. Recovery happened only in the counter-case where a healthy + tenant won the probe. It is a race, not a policy. +- **The principal does not reach the resilience layer at all.** Root, `.as("t1")` and `.as("t2")` all + touched the single key `circuit:/v1/items`. `Runtime.principal` is threaded into `AuthContext` + (engine.ts:101-102) and read by `oauth2` / `cookieSession` / the cache-key builder; + `attemptWithCircuit` (engine.ts:846-893) never sees it. +- **C2 refutes the capture's proposed workaround outright.** The capture says "if `key` is static + config, per-tenant breakers mean one stitch per tenant". Measured: **10 distinct `Stitch` objects, + each `.as()`-bound and cached per tenant, produced 1 breaker key and 9 of 9 healthy tenants down.** + So did **10 separate seams** sharing one store. Breaker state lives in the shared store at + `circuit:` (resilience.ts:353); N objects resolving to the same `path` are N handles on one + record. **Isolation is a property of the key string, never of the object graph.** +- **What does work is a string, and only a string.** A per-tenant `circuit.key` → 10 keys, **0 of 9** + healthy failed, and the broken tenant's own breaker still opened (503). A per-tenant `name` does it + implicitly, because `hostKey` falls back to `cfg.name ?? cfg.path` (engine.ts:140,273) — which + means the trace/diagnostic label is silently also the partition key. +- **The worst default in the file: `circuit:stitch`.** Ten `url`-only stitches (no `name`, no `path`) + over a shared store all key on the literal string `'stitch'` (`nameOf`, engine.ts:140). An + **unrelated endpoint for an unrelated tenant** measured **503**. +- **C3 splits a trade the capture treats as unavoidable.** `verdict: { accept: [401] }` alone IS the + swallowing case, and worse than swallowing: 4 of 4 calls measured **`ok`** and the caller was handed + **`{"error":"invalid_token"}` as its DATA**. But `verdict: { accept: [401], flag: 'ok' }` — pure + config — gave the broken tenant a real `StitchError` **401** on all 5 calls, **0 of 9** healthy + tenants failed, and the breaker recorded **0 failures** and never tripped. It works because the + engine already routes on WHAT failed (engine.ts:807-833): a bad STATUS throws and is counted, while + a response the SURFACE rejected returns `{ ok: false }`, fails the call, and records + `circuit.onSuccess()`. For a vendor whose error body has no falsy flag, **5 lines** of + `Surface.interpret` composing `verdictOf` do the same. Under either, a genuine 500 outage still + measured `500,500,500,503` and tripped the breaker — the exclusion is surgical. +- **C4's number is 2000 virtual ms**, and the mechanism is that a seam's throttle **discards the + member's key** and re-keys every acquire onto `seam:${seamId}` (seam.ts:51-69,64). A longer window + does not help (`600/m` declares the same 100ms spacing as `10/s` and measured the same 2000); a + tighter one is worse (`2/s` → **10 virtual seconds**). **The concurrency half is sharper and the + capture does not mention it:** with `concurrency: 2` the quiet tenant's instant call left at + **t=5000** with all 20 of the noisy tenant's slow calls ahead of it, because waiters are FIFO over + one shared key (resilience.ts:120-127,159-177). It is not slowed proportionally — it is last. +- **C5 refutes the capture in the OPTIMISTIC direction.** "The rate bucket looks un-partitionable by + tenant" — it is partitionable, twice over: a per-tenant **seam** (the bucket key carries the seam + id, and it holds even over one shared store) or a per-tenant **`name` plus a member-level + `throttle`** (`rl:items:`). Both measured the quiet tenant at **t=0**. What is missing is + only the DECLARATION: of six candidate spellings, only `pool: 'stitch'` and `pool: 'host'` compile. +- **The asymmetry is the thing to put in the docs.** In ONE run, per-tenant seams over a shared store + **isolated the rate budget** (quiet at t=0) and **shared the breaker** (one key, 3 of 3 healthy + tenants down). The two resources are keyed by different rules — the bucket by the seam id, the + breaker by `cfg.name ?? cfg.path` — so no construction can be reasoned about as a whole. +- **`throttle.pool: 'host'` silently re-keys the CIRCUIT.** `hostKey` reads `cfg.throttle?.pool` + (engine.ts:265-274) and is the circuit's fallback key (engine.ts:860), so tuning the rate pool moved + the breaker to `circuit:api.vendor.test` and a per-tenant `name` partition evaporated — an unrelated + endpoint for an unrelated tenant measured **503**. +- **Seam ids are a creation-ORDER counter** (`s1`, `s2`, … — seam.ts:38,233), measured consecutive + with no tenant derivation. Two processes each hand out `s1`, so per-tenant seams over a **shared + durable store** put worker A's tenant-1 and worker B's tenant-7 in the same rate bucket. +- **C6 is the one axis the library gets right by construction.** `oauth2({ tenancy: 'principal' })` + minted 3 tokens for 3 customers under 3 distinct vault keys (`tokenUrl` + NUL + principal, + auth.ts:485-499) and reused t1's on t1's second call. It fails **closed**: without `.as()` the call + errored with a message naming `seam.as(` and made **0** token requests. A revoked tenant's storm — + 5 doomed calls, **6 token fetches, 10 vendor requests** — left the healthy tenant carrying the + identical token before and after, still succeeding. +- **But the default is `'app'`, and `tenancy` partitions the token, not the CREDENTIAL.** Three + different customers under the default measured **1 token fetch and one shared `Authorization` + header**. And even under `'principal'`, all three tokens were minted from client_id `saas-app`, + because `Secret = string | (() => string)` (auth.ts:47) is a **niladic** thunk with no + `AuthContext` in scope. Per-customer credentials need one strategy instance per customer — or the + one-line escape hatch: a custom `AuthStrategy.apply(req, ctx)` reading `ctx.principal`, which is + **the only user-reachable hook in the library that sees the bound principal at call time**. +- **C7 refutes the capture's cost model.** "One client instance per tenant … does not scale: 4,000 + pools, timers and caches." Measured: 100 per-tenant seams in **single-digit ms**, **~6.4kb each**, + **0 timers**, and **0 connection pools** — a seam owns no transport, and all 100 shared one + `adapter`. 1 seam + 100 keyed stitches measured the same order of magnitude. The choice between the + two shapes is about which resource each isolates, not about scale. +- **The real price is three things that are never freed.** (1) **Breaker records have no TTL** — + `circuit.onSuccess`/`onFailure` write without one (resilience.ts:382-403) and the store reads a + missing ttl as live-forever (store.ts:45); a churned tenant's key was **still resident after a + virtual YEAR**, while the rate counter beside it does expire. At the capture's 4,000 connections + that is 4,000 immortal keys and nothing sweeps them. (2) A **rate-paced limiter retains one + in-process map entry per key for the life of the process** — 100/100 after acquire+release, versus + 0/100 for a concurrency-only limiter (store.ts:243-254). (3) **`seam.stitch()` pins every stitch it + creates**: 200/200 root-created still reachable after a forced GC, versus **0/200** created through + `seam.as(p).stitch()` (seam.ts:136-141). The only release is `seam.close()` — which also closes the + store. The per-request shape is the one that does not leak. +- **C8, as one sentence: two of the four are isolated by the principal, two only by a string you have + to remember to write.** Token and cache fold the principal in and **fail closed**; the rate budget + and the breaker take a hand-written key and **fail open**, silently, with no type error and no + warning. The split is exactly the auth/resilience line. Assembled, the construction works: the same + revoked credential that took down **9 of 9** in C1 took down **0 of 9**, the broken tenant still got + a real 401, all 9 healthy calls reached the vendor, a genuine 500 opened **only that tenant's** own + breaker, and the burst that pushed a quiet tenant to t=2000 measured **t=0**. +- **And one thing the built-ins cannot express at all.** A real integration has BOTH a global quota + with the vendor and per-customer fairness. Adding a seam-level `throttle` for the global cap put the + noisy neighbour straight back (quiet at **t=2000**), because a member's throttle stacks + **tighten-only** on the seam bucket (seam.ts:94-106). "1000/m to the vendor AND 10/s per customer" + is not one construction. + +## The footguns + +- **A shared `circuit` on a seam is a multi-tenant outage generator, and it reads like a safety + feature.** Nothing about `circuit: [5, '30s']` on a seam says "one customer's revoked token fails + every other customer". Set `circuit.key` per tenant, or do not set `circuit` on a shared seam. +- **Per-tenant OBJECTS do not give per-tenant STATE.** One stitch per tenant, one seam per tenant — + both measured 1 breaker and 9 of 9 healthy tenants down. The unit of isolation is the key string in + the shared store. This is the single most likely wrong belief a reader arrives with. +- **A `url`-only stitch keys its breaker on the literal string `'stitch'`.** Every such stitch sharing + a store shares one process-wide breaker, across tenants AND across endpoints. +- **`throttle: { pool: 'host' }` moves the CIRCUIT too.** Two unrelated config concerns, one key + function (engine.ts:265-274). Someone tuning the rate pool can widen the breaker to the whole host + without touching the `circuit` block. Set `circuit.key` explicitly and it cannot happen. +- **A per-tenant seam isolates the rate budget and NOT the breaker.** The most isolated-looking + construction available is half a fix, and the half it misses is the one that causes outages. +- **`verdict: { accept: [401] }` on its own does not "ignore" the 401 — it SUCCEEDS on it.** The + caller receives the vendor's error envelope as data. Pair it with `flag`, or with a surface. +- **`oauth2` defaults to `tenancy: 'app'`**, which is one token for every customer. Correct for + `client_credentials`, wrong for a per-customer integration, and silent either way. +- **`tenancy: 'principal'` isolates the token, not the credential.** `Secret` takes no context, so a + shared `oauth2()` mints every tenant's token from the same client id. +- **`cache: { tenancy: 'app' }` serves one tenant another tenant's response body** — measured. The + default is `'principal'` and fail-closed, so this only bites someone who opts out. +- **Per-tenant breaker keys never expire.** One immortal store key per tenant per endpoint, in Redis, + forever. If tenants churn, that set only grows. +- **`seam.stitch()` retains every stitch it creates.** Caching one root-created stitch per tenant — + the obvious optimisation — pins all of them until `seam.close()`. Build per-tenant members through + `seam.as(id).stitch(...)`, which does not register. +- **Seam ids are per-process creation order.** Per-tenant seams over a shared durable store collide + across workers, non-deterministically. diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c1-shared-breaker.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c1-shared-breaker.ts new file mode 100644 index 00000000..c9c0723d --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c1-shared-breaker.ts @@ -0,0 +1,209 @@ +// C1 — DECIDING. One tenant's credential is revoked. With a `circuit` on the shared seam, does the +// breaker open and fail the HEALTHY tenants? Measure exactly: N healthy tenants, how many failed, +// with what error, and how many of them ever reached the vendor. +// +// The capture predicts the shape ("one revoked token, total outage") and it is right. Two things +// it does NOT predict, both measured here: +// +// • The outage is SELF-SUSTAINING, not a `cooldown`-long blip. The breaker admits exactly ONE +// trial call when it goes half-open (resilience.ts:375-379), and the tenant most likely to make +// it is the broken one — it is the one retrying hardest. Its 401 re-opens the breaker before +// any healthy tenant is admitted. Measured over 4 cooldown windows in (d): healthy is 503 in +// every one of them. +// • `seam.as(principal)` does not reach the resilience layer AT ALL. The breaker's key is +// `hostKey(req, cfg)` — `cfg.name ?? cfg.path ?? 'stitch'` (engine.ts:140,265-274,860) — and (e) +// measures the same key string for a root-created and a principal-bound stitch. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c1-shared-breaker.ts +import { seam } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeVendor, blastRadius, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +const HEALTHY = ['t1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9']; +const BAD = 'bad'; + +/** The shared-surface construction a SaaS writes first: one seam, one breaker, `.as()` per customer. */ +function sharedSeam(failing: Record = { [BAD]: 401 }) { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: { failures: 3, cooldown: '30s' }, + }); + const call = (tenant: string) => + s.as(tenant).stitch({ + path: '/v1/items', + headers: { 'x-tenant': tenant }, + }); + return { clock, store, vendor, seam: s, call }; +} + +async function main(): Promise { + heading('C1 — one revoked credential, N healthy tenants'); + + // ── (a) the blast radius, as a count ─────────────────────────────────────────────────────── + // The bad tenant fails its threshold; then nine healthy customers make one ordinary call each. + { + const { vendor, call } = sharedSeam(); + const bad: string[] = []; + for (let i = 0; i < 3; i++) + bad.push(await outcomeOf(() => call(BAD)({}))); + const healthy: string[] = []; + for (const t of HEALTHY) + healthy.push(await outcomeOf(() => call(t)({}))); + + checkSeq('(a) the bad tenant, 3 calls', bad, ['401', '401', '401']); + check('(a) healthy tenants called', HEALTHY.length, 9); + check('(a) → how many of them FAILED', blastRadius(healthy), 9); + checkSeq('(a) their outcomes', [...new Set(healthy)], ['503']); + // They did not fail at the vendor — they never got there. + check( + '(a) healthy requests that reached the vendor', + vendor.calls.filter((c) => c.tenant !== BAD).length, + 0, + ); + note( + '(a) → one customer whose refresh token was revoked', + 'took down 9 of 9 healthy customers, and their calls never left the process', + ); + } + + // ── (b) what the healthy tenant's error actually is ──────────────────────────────────────── + // A 503 with a body-free `circuit open` message: nothing in it names the tenant that caused it, + // so the on-call page for customer #7 says the vendor is down when the vendor is fine. + { + const { call } = sharedSeam(); + for (let i = 0; i < 3; i++) await outcomeOf(() => call(BAD)({})); + const r = await call('t1')({}).safe(); + const err = r.error as (Error & { status?: number }) | undefined; + check('(b) healthy call ok?', r.ok, false); + check('(b) error name', err?.name, 'StitchError'); + check('(b) status', err?.status, 503); + check('(b) message', err?.message, 'circuit open'); + check( + '(b) does the error name the tenant that opened it?', + JSON.stringify(err ?? {}).includes(BAD), + false, + ); + note( + '(b) → `CircuitOpenError` carries `status = 503` (resilience.ts:251-257)', + 'surfaced to the caller as a StitchError with no attribution to the tenant that tripped it', + ); + } + + // ── (c) the threshold is CONSECUTIVE failures, pooled across tenants ─────────────────────── + // Interleaving healthy traffic does reset the counter (`onSuccess` clears it, + // resilience.ts:381-387) — so the breaker opens only when the bad tenant's calls happen to run + // back-to-back. That is a scheduling accident, not a safety property: one tenant polling on a + // timer is enough. + { + const { call } = sharedSeam(); + const interleaved: string[] = []; + for (let i = 0; i < 3; i++) { + interleaved.push(await outcomeOf(() => call(BAD)({}))); + interleaved.push(await outcomeOf(() => call('t1')({}))); + } + checkSeq('(c) bad/healthy interleaved 3×', interleaved, [ + '401', + 'ok', + '401', + 'ok', + '401', + 'ok', + ]); + // …and now three of the bad tenant's calls in a row, which is all it takes. + const burst: string[] = []; + for (let i = 0; i < 3; i++) + burst.push(await outcomeOf(() => call(BAD)({}))); + const after = await outcomeOf(() => call('t1')({})); + check('(c) healthy tenant after 3 consecutive bad calls', after, '503'); + note( + '(c) → the breaker counts CONSECUTIVE failures over one shared counter', + 'healthy traffic resets it, so whether the outage happens depends on interleaving — a tenant polling on a timer trips it reliably', + ); + } + + // ── (d) THE FINDING THE CAPTURE MISSES: the outage does not end ──────────────────────────── + // `cooldown` elapses, the breaker goes half-open and admits ONE trial. The broken tenant is the + // one hammering the endpoint, so it wins the probe, fails, and re-arms a fresh cooldown + // (resilience.ts:389-404). Four windows, and no healthy tenant is ever admitted. + { + const { clock, call } = sharedSeam(); + for (let i = 0; i < 3; i++) await outcomeOf(() => call(BAD)({})); + const rounds: string[] = []; + for (let round = 0; round < 4; round++) { + await clock.advance(30_000); // the full cooldown elapses + await outcomeOf(() => call(BAD)({})); // the broken tenant takes the trial + rounds.push(await outcomeOf(() => call('t1')({}))); + } + checkSeq('(d) healthy tenant across 4 cooldown windows', rounds, [ + '503', + '503', + '503', + '503', + ]); + check( + '(d) virtual seconds elapsed, still failing', + clock.now() / 1000, + 120, + ); + note( + '(d) → half-open admits exactly ONE trial (resilience.ts:375-379)', + 'the tenant most likely to take it is the broken one; its failure re-opens the breaker, so the outage is self-sustaining, not `cooldown`-long', + ); + + // The counter-case, so the mechanism is unambiguous: when a HEALTHY tenant happens to win + // the probe, its success closes the breaker for everyone. + const fresh = sharedSeam(); + for (let i = 0; i < 3; i++) await outcomeOf(() => fresh.call(BAD)({})); + await fresh.clock.advance(30_000); + const won = await outcomeOf(() => fresh.call('t1')({})); + const next = await outcomeOf(() => fresh.call('t2')({})); + checkSeq( + '(d) when a HEALTHY tenant wins the probe', + [won, next], + ['ok', 'ok'], + ); + note( + '(d) → recovery is a race between the broken tenant and a healthy one', + 'and the broken tenant is retrying harder by construction', + ); + } + + // ── (e) the key the breaker is actually stored under ─────────────────────────────────────── + // The measurement that decides C2 as well: the principal is nowhere in it. + { + const { store, call } = sharedSeam(); + await outcomeOf(() => call('t1')({})); + await outcomeOf(() => call('t2')({})); + checkSeq( + '(e) circuit keys touched by 2 DIFFERENT tenants', + store.keys('circuit:'), + ['circuit:/v1/items'], + ); + check( + '(e) does the key contain a principal?', + store + .keys('circuit:') + .some((k) => k.includes('t1') || k.includes('t2')), + false, + ); + note( + '(e) → the breaker key is `circuit:` + `opts.key ?? hostKey(req, cfg)`', + 'resilience.ts:353 over engine.ts:860; `hostKey` is `cfg.name ?? cfg.path ?? "stitch"` (engine.ts:140,265-274) — `AuthContext.principal` never reaches it', + ); + } + + finish( + 'C1', + 'CONFIRMED, and worse than the capture predicts. One tenant with a revoked credential and a `circuit: { failures: 3, cooldown: "30s" }` on the shared seam failed 9 of 9 healthy tenants — a blast radius of 100% — and 0 of their requests ever reached the vendor: they fast-failed in-process with `StitchError` status 503, message "circuit open", carrying nothing that names the tenant responsible. THE UNPREDICTED PART IS THAT THE OUTAGE DOES NOT END. Half-open admits exactly ONE trial call (resilience.ts:375-379) and the broken tenant is the one retrying hardest, so across 4 full cooldown windows (120 virtual seconds) the healthy tenant measured 503,503,503,503; recovery happens only when a healthy tenant happens to win the probe, which is a race, not a policy. `seam.as(principal)` does not participate: two different principals touched the single key `circuit:/v1/items`, because the breaker keys on `opts.key ?? hostKey(req, cfg)` = `cfg.name ?? cfg.path ?? "stitch"` (resilience.ts:353, engine.ts:140,265-274,860) and the principal never reaches the resilience layer', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c2-partition-the-breaker.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c2-partition-the-breaker.ts new file mode 100644 index 00000000..2de02f5d --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c2-partition-the-breaker.ts @@ -0,0 +1,329 @@ +// C2 — can the breaker be partitioned per tenant? Is `circuit.key` (types.ts:1088) static config, +// or can it vary per call? Does a bound principal influence it at all? +// +// The answers, all measured: `circuit.key` is a static `string` and nothing principal-derived +// compiles (a); the bound principal changes nothing (b); and partitioning DOES work — but only via +// the key STRING, either an explicit `circuit.key` (c) or a per-tenant `name` (d). +// +// THE CAPTURE'S WORKAROUND IS WRONG, and this is the most important thing in the file. It says "if +// [`key`] is static config, per-tenant breakers mean one stitch per tenant". One stitch per tenant +// does NOT partition the breaker: the state lives in the SEAM'S SHARED STORE under a key derived +// from the config, so 100 per-tenant stitch objects that resolve to the same `name`/`path` share +// one breaker — measured in (e). A per-tenant SEAM over a shared store doesn't do it either (f), +// and per-tenant `stitch()`es with a `url` and no `name` all collapse onto the literal key +// `circuit:stitch` (g). Isolation is a property of the key string, never of the object graph. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c2-partition-the-breaker.ts +import { seam, stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeVendor, blastRadius, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +const HEALTHY = ['t1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9']; +const BAD = 'bad'; +const CIRCUIT = { failures: 3, cooldown: '30s' } as const; + +/** Drive one tenant past the threshold, then call every healthy tenant once. */ +async function blast( + call: (tenant: string) => Stitch, +): Promise<{ bad: string[]; healthy: string[] }> { + const bad: string[] = []; + for (let i = 0; i < 3; i++) bad.push(await outcomeOf(() => call(BAD)({}))); + const healthy: string[] = []; + for (const t of HEALTHY) healthy.push(await outcomeOf(() => call(t)({}))); + return { bad, healthy }; +} + +async function main(): Promise { + heading('C2 — partitioning the breaker per tenant'); + + // ── (a) what the compiler admits in the `circuit` envelope ───────────────────────────────── + // Not a grep: each candidate is typechecked as a statement. A line that compiles is a spelling + // that exists. + { + const results = probeSpellings([ + { + label: 'circuit.key as a static string', + code: `stitch({ url: 'https://x.test/y', circuit: { failures: 3, cooldown: '30s', key: tenantId } });`, + }, + { + label: 'circuit.key as a per-call function', + code: `stitch({ url: 'https://x.test/y', circuit: { failures: 3, cooldown: '30s', key: () => tenantId } });`, + }, + { + label: 'circuit.keyOf (the P6 derivation spelling)', + code: `stitch({ url: 'https://x.test/y', circuit: { failures: 3, cooldown: '30s', keyOf: () => tenantId } });`, + }, + { + label: "circuit.tenancy: 'principal'", + code: `stitch({ url: 'https://x.test/y', circuit: { failures: 3, cooldown: '30s', tenancy: 'principal' } });`, + }, + ]); + checkSeq('(a) spellings that COMPILE', accepted(results), [ + 'circuit.key as a static string', + ]); + check( + '(a) spellings the compiler REFUSED', + rejected(results).length, + 3, + ); + note( + '(a) → `CircuitOptions` is `{ failures?, cooldown?, key? }` with `key?: string` (types.ts:1072-1089)', + 'no derivation fn, no `tenancy`; the rejections are real compile errors from `NoUnknownNestedKeys` (types.ts:411-448), not silent no-ops', + ); + } + + // ── (b) does `seam.as(principal)` influence the circuit key? ─────────────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + }); + await s.stitch({ path: '/v1/items' })({}).safe(); // root, no principal + await s.as('t1').stitch({ path: '/v1/items' })({}).safe(); + await s.as('t2').stitch({ path: '/v1/items' })({}).safe(); + checkSeq( + '(b) circuit keys for root + 2 principals', + store.keys('circuit:'), + ['circuit:/v1/items'], + ); + note( + '(b) → the principal reaches `AuthContext` and stops there', + '`Runtime.principal` is threaded into `authCtx` (engine.ts:101-102) and read by `oauth2`/`cookieSession`/`cache`; `attemptWithCircuit` never sees it (engine.ts:846-893)', + ); + } + + // ── (c) an explicit per-tenant `circuit.key` — this works ────────────────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 401 } }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + const call = (t: string) => + s.as(t).stitch({ + path: '/v1/items', + headers: { 'x-tenant': t }, + circuit: { ...CIRCUIT, key: `items:${t}` }, + }); + const { bad, healthy } = await blast(call); + checkSeq('(c) the bad tenant', bad, ['401', '401', '401']); + check('(c) → healthy tenants that FAILED', blastRadius(healthy), 0); + check('(c) distinct breaker keys', store.keys('circuit:').length, 10); + check( + '(c) the bad tenant has its own breaker', + store.keys('circuit:').includes(`circuit:items:${BAD}`), + true, + ); + // …and it is genuinely OPEN for that tenant, which is the point of partitioning. + check( + '(c) the bad tenant now fast-fails', + await outcomeOf(() => call(BAD)({})), + '503', + ); + note( + '(c) → `circuit.key` IS the partition knob', + 'it is read once per call at `createCircuit(cfg.circuit, …)` (engine.ts:857-862), so a per-tenant stitch carrying a per-tenant key gives a per-tenant breaker', + ); + } + + // ── (d) a per-tenant `name` does the same thing, implicitly ──────────────────────────────── + // `hostKey` falls back to `cfg.name ?? cfg.path` (engine.ts:140,273), so naming the stitch per + // tenant partitions the breaker without touching `circuit` at all. Convenient, and a trap in + // both directions — see (e). + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 401 } }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + }); + const call = (t: string) => + s.as(t).stitch({ + name: `items:${t}`, + path: '/v1/items', + headers: { 'x-tenant': t }, + }); + const { healthy } = await blast(call); + check('(d) healthy tenants that FAILED', blastRadius(healthy), 0); + check('(d) distinct breaker keys', store.keys('circuit:').length, 10); + note( + '(d) → the partition is the NAME, which is also a trace/diagnostic label', + 'one string is doing two jobs; renaming a stitch for readability silently re-partitions its breaker', + ); + } + + // ── (e) THE TRAP: one stitch per tenant does NOT partition ───────────────────────────────── + // The capture's proposed workaround. 10 distinct `Stitch` objects, each bound to its own + // principal, cached for the life of the process — and one breaker, because the KEY is the same. + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 401 } }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + }); + const perTenant = new Map(); + const call = (t: string) => { + let st = perTenant.get(t); + if (!st) { + st = s.as(t).stitch({ + path: '/v1/items', + headers: { 'x-tenant': t }, + }); + perTenant.set(t, st); + } + return st; + }; + const { healthy } = await blast(call); + check('(e) distinct Stitch objects constructed', perTenant.size, 10); + check('(e) distinct breaker keys', store.keys('circuit:').length, 1); + check('(e) → healthy tenants that FAILED', blastRadius(healthy), 9); + note( + '(e) → "one stitch per tenant" is NOT the fix the capture assumes', + "breaker state lives in the seam's SHARED store at `circuit:` (resilience.ts:353); 10 objects resolving to the same `path` are 10 handles on one record", + ); + } + + // ── (f) …and neither is one SEAM per tenant, if they share a store ───────────────────────── + // The construction that looks most isolated of all — a whole seam per customer — and the + // breaker is still shared, because a seam's identity is not in the circuit key. + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 401 } }); + const seams = new Map>(); + const call = (t: string) => { + let sm = seams.get(t); + if (!sm) { + sm = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, // the shared/durable store a real deployment configures + clock, + circuit: CIRCUIT, + }); + seams.set(t, sm); + } + return sm.as(t).stitch({ + path: '/v1/items', + headers: { 'x-tenant': t }, + }); + }; + const { healthy } = await blast(call); + check('(f) distinct seams constructed', seams.size, 10); + check('(f) distinct breaker keys', store.keys('circuit:').length, 1); + check('(f) → healthy tenants that FAILED', blastRadius(healthy), 9); + note( + '(f) → a per-tenant seam isolates the RATE bucket but not the BREAKER', + 'the bucket key carries the seam id (`seam:sN`, seam.ts:59) and the breaker key does not — see C5 (f) for the asymmetry measured side by side', + ); + } + + // ── (g) the worst default: a `url`-only stitch keys on the literal string "stitch" ───────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 500 } }); + const made = new Map(); + const call = (t: string) => { + let st = made.get(t); + if (!st) { + st = stitch({ + url: 'https://api.vendor.test/v1/items', + adapter: vendor.adapter(), + headers: { 'x-tenant': t }, + store, // one shared store, as a real deployment configures + clock, + circuit: CIRCUIT, + }); + made.set(t, st); + } + return st; + }; + const { healthy } = await blast(call); + checkSeq( + '(g) breaker keys for 10 url-only stitches', + store.keys('circuit:'), + ['circuit:stitch'], + ); + check('(g) → healthy tenants that FAILED', blastRadius(healthy), 9); + // And it is not per-endpoint either: a completely different call collides too. + const other = stitch({ + url: 'https://api.vendor.test/v1/orders', + adapter: vendor.adapter(), + headers: { 'x-tenant': 'unrelated' }, + store, + clock, + circuit: CIRCUIT, + }); + check( + '(g) an UNRELATED endpoint on the same shared store', + await outcomeOf(() => other({})), + '503', + ); + note( + '(g) → `nameOf` is `cfg.name ?? cfg.path ?? "stitch"` (engine.ts:140)', + 'a `url`-only config has neither, so every such stitch sharing a store shares one process-wide breaker under the literal key `circuit:stitch`', + ); + } + + // ── (h) what the correct partition costs to build at 100 tenants ─────────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + const t0 = Date.now(); + const calls = Array.from({ length: 100 }, (_, i) => + s.as(`t${i}`).stitch({ + path: '/v1/items', + headers: { 'x-tenant': `t${i}` }, + circuit: { ...CIRCUIT, key: `items:t${i}` }, + }), + ); + const elapsed = Date.now() - t0; + await Promise.all(calls.map((c) => c({}).safe())); + check( + '(h) 100 per-tenant keyed stitches → distinct breakers', + store.keys('circuit:').length, + 100, + ); + check('(h) timers armed by constructing them', clock.pending(), 0); + check('(h) construction under 100ms', elapsed < 100, true); + note('(h) construction time (ms)', elapsed); + } + + finish( + 'C2', + 'YES — but only through the key STRING, and the capture\'s workaround does not work. `circuit.key` is a static `string`: of four candidate spellings typechecked, only `key: ` compiles; `key: () => id`, `keyOf`, and `tenancy: \'principal\'` are compile errors (types.ts:1072-1089, NoUnknownNestedKeys types.ts:411-448). A bound principal changes nothing — root, `.as("t1")` and `.as("t2")` all touched the single key `circuit:/v1/items`. What DOES partition is a per-tenant `circuit.key` (10 keys, 0 of 9 healthy tenants failed, and the bad tenant\'s own breaker still opened → 503) or, implicitly, a per-tenant `name`, since `hostKey` falls back to `cfg.name ?? cfg.path` (engine.ts:140,273). THE CAPTURE SAYS "PER-TENANT BREAKERS MEAN ONE STITCH PER TENANT" AND THAT IS FALSE: 10 distinct Stitch objects, each `.as()`-bound, resolving to the same `path` produced 1 breaker key and 9 of 9 healthy tenants down; so did 10 separate SEAMS sharing one store. Worst of all, 10 `url`-only stitches on a shared store collapse onto the literal key `circuit:stitch` — where an unrelated endpoint also measured 503. Isolation is a property of the key string, never of the object graph. The correct partition is cheap: 100 keyed stitches built in under 100ms with 0 timers armed', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c3-401-is-not-a-dependency-failure.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c3-401-is-not-a-dependency-failure.ts new file mode 100644 index 00000000..6a08d6b9 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c3-401-is-not-a-dependency-failure.ts @@ -0,0 +1,321 @@ +// C3 — is a `401` counted as a circuit failure? It shouldn't be: it says the CREDENTIAL is bad, not +// the dependency. Can `verdict`/`acceptStatus` exclude it from the breaker WITHOUT also swallowing +// the error from the caller? +// +// YES, and the mechanism is more interesting than the capture expects. The engine routes on WHAT +// failed (engine.ts:807-833): a bad STATUS throws and the throw is what `attemptWithCircuit` counts +// as a failure, while a well-formed response the SURFACE rejected comes back as a returned +// `{ ok: false }` outcome — which reaches the caller as a failed call but records a circuit +// SUCCESS. So the two halves of "error the caller, spare the breaker" are already separated; the +// question is only how to get a 401 onto the second path. +// +// Two ways, both measured. Pure config: `verdict: { accept: [401], flag: }` — `accept` moves +// the 401 off the status-failure path, `flag` fails it on body grounds. Or ~5 lines of surface: +// an `interpret` that rejects 401/403 itself, for a vendor whose error body has no usable flag. +// +// The capture's worry is real and is measured in (b): `accept` ALONE is the swallowing case — the +// call succeeds and the caller is handed the 401 error body as its DATA. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c3-401-is-not-a-dependency-failure.ts +import { seam, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { FakeVendor, blastRadius, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +const HEALTHY = ['t1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9']; +const BAD = 'bad'; +const CIRCUIT = { failures: 3, cooldown: '30s' } as const; + +/** + * USER CODE — the surface a multi-tenant integration wants: a 401/403 is a CREDENTIAL verdict, not + * a transport verdict. It composes `verdictOf` (surface.ts:174) so the stitch's own `verdict` + * config is still honoured, exactly as the http surface does. + * + * It only works paired with `verdict: { accept: [401, 403] }`: `accept` is what stops the ENGINE + * throwing on the status before the surface is consulted (engine.ts:824), and this hook is what + * turns the accepted response back into a failure the caller sees. + */ +const credentialAware: Surface = { + id: 'http', + interpret: (res, cfg) => { + if (res.status === 401 || res.status === 403) + return { + ok: false, + message: `credential rejected (HTTP ${res.status})`, + status: res.status, + }; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, +}; + +function fixture(opts: { + failBody?: unknown; + failing?: Record; +}) { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ + clock, + failing: opts.failing ?? { [BAD]: 401 }, + ...(opts.failBody !== undefined ? { failBody: opts.failBody } : {}), + }); + return { clock, store, vendor }; +} + +/** Did the breaker ever record a failure? Reading the record beats inferring it from behaviour. */ +async function circuitRecord( + store: ReturnType, + key: string, +): Promise<{ failures: number; tripped: boolean }> { + const r = (await store.get(key)) as + { failures?: number; tripped?: boolean } | undefined; + return { failures: r?.failures ?? 0, tripped: r?.tripped ?? false }; +} + +async function main(): Promise { + heading('C3 — keeping a credential failure off the dependency breaker'); + + // ── (a) baseline: a 401 IS a circuit failure ─────────────────────────────────────────────── + { + const { clock, store, vendor } = fixture({}); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + for (let i = 0; i < 3; i++) await outcomeOf(() => call(BAD)({})); + const rec = await circuitRecord(store, 'circuit:/v1/items'); + check('(a) failures recorded by three 401s', rec.failures, 3); + check('(a) breaker tripped', rec.tripped, true); + note( + '(a) → a 401 reaches `attemptWithCircuit` as a THROW', + '`classifyStatus` says 401 ≥ 400 and is not accepted, so the engine throws (engine.ts:824-831) and the catch records `circuit.onFailure()` (engine.ts:879-890)', + ); + } + + // ── (b) `verdict.accept: [401]` alone — the breaker is spared and the ERROR IS SWALLOWED ─── + // Exactly the trade the capture worries about, and it is worse than "swallowed": the caller is + // handed the vendor's error envelope as a successful RESULT, so a revoked credential looks like + // data all the way up the stack. + { + const { clock, store, vendor } = fixture({}); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + verdict: { accept: [401] }, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + const bad: string[] = []; + for (let i = 0; i < 4; i++) + bad.push(await outcomeOf(() => call(BAD)({}))); + const r = await call(BAD)({}).safe(); + checkSeq('(b) the bad tenant, 4 calls', bad, ['ok', 'ok', 'ok', 'ok']); + check('(b) the call reports ok', r.ok, true); + check( + '(b) → and its DATA is the 401 error body', + JSON.stringify(r.data), + '{"error":"invalid_token"}', + ); + const rec = await circuitRecord(store, 'circuit:/v1/items'); + check('(b) circuit failures recorded', rec.failures, 0); + note( + '(b) → `accept` can only turn a failure into a SUCCESS (surface.ts:174-190)', + 'it spares the breaker by making the call succeed, which is not the trade a multi-tenant caller wants', + ); + } + + // ── (c) `accept` + `flag`: pure config, error preserved, breaker spared ──────────────────── + // For any vendor whose error envelope carries an explicitly-falsy flag. `accept` moves the + // status off the throw path; `flag` fails the call on BODY grounds, which is an + // application-level rejection — and the engine deliberately keeps those off the breaker. + { + const { clock, store, vendor } = fixture({ + failBody: { ok: false, error: 'invalid_token' }, + }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + verdict: { accept: [401], flag: 'ok' }, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + const bad: string[] = []; + for (let i = 0; i < 5; i++) + bad.push(await outcomeOf(() => call(BAD)({}))); + const healthy: string[] = []; + for (const t of HEALTHY) + healthy.push(await outcomeOf(() => call(t)({}))); + + checkSeq( + '(c) the bad tenant, 5 calls — still an ERROR', + [...new Set(bad)], + ['401'], + ); + const r = await call(BAD)({}).safe(); + check( + '(c) error status', + (r.error as { status?: number })?.status, + 401, + ); + check( + '(c) error message', + r.error?.message, + 'verdict.flag `ok` is false', + ); + check('(c) → healthy tenants that FAILED', blastRadius(healthy), 0); + const rec = await circuitRecord(store, 'circuit:/v1/items'); + check('(c) circuit failures recorded', rec.failures, 0); + check('(c) breaker tripped', rec.tripped, false); + note( + '(c) → the split already exists in the engine (engine.ts:807-833)', + 'a bad STATUS throws and is counted; a response the SURFACE rejected returns `{ ok: false }`, fails the call, and records `circuit.onSuccess()` — "the transport is healthy, the payload is not"', + ); + } + + // ── (d) the same result with a surface, for a vendor with no usable flag ─────────────────── + { + const { clock, store, vendor } = fixture({}); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: CIRCUIT, + verdict: { accept: [401, 403] }, + }); + const call = (t: string) => + s.as(t).stitch({ + path: '/v1/items', + headers: { 'x-tenant': t }, + kind: credentialAware, + }); + const bad: string[] = []; + for (let i = 0; i < 5; i++) + bad.push(await outcomeOf(() => call(BAD)({}))); + const healthy: string[] = []; + for (const t of HEALTHY) + healthy.push(await outcomeOf(() => call(t)({}))); + const r = await call(BAD)({}).safe(); + + checkSeq('(d) the bad tenant, 5 calls', [...new Set(bad)], ['401']); + check( + '(d) error message names the credential', + r.error?.message, + 'credential rejected (HTTP 401)', + ); + check('(d) → healthy tenants that FAILED', blastRadius(healthy), 0); + check( + '(d) circuit failures recorded', + (await circuitRecord(store, 'circuit:/v1/items')).failures, + 0, + ); + note( + '(d) → 5 lines of `Surface.interpret` (surface.ts:56-62), composing `verdictOf`', + 'needed only because `verdict.flag` requires an explicitly-falsy field in the error body; a bare `{ "error": "..." }` has none', + ); + } + + // ── (e) …and it is SURGICAL: a real outage still opens the breaker ───────────────────────── + // The exclusion has to be status-shaped, not a blanket "never trip". Under exactly the config + // from (d), a 500 from the vendor trips it in 3. + { + const { clock, store, vendor } = fixture({ failing: {} }); + const down: Adapter = async () => ({ + status: 500, + headers: {}, + body: { error: 'upstream' }, + }); + void vendor; + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: down, + store, + clock, + circuit: CIRCUIT, + verdict: { accept: [401, 403] }, + }); + const call = (t: string) => + s.as(t).stitch({ + path: '/v1/items', + headers: { 'x-tenant': t }, + kind: credentialAware, + }); + const spine: string[] = []; + for (const t of ['t1', 't2', 't3', 't4']) + spine.push(await outcomeOf(() => call(t)({}))); + checkSeq('(e) a genuine 500 outage', spine, [ + '500', + '500', + '500', + '503', + ]); + check( + '(e) breaker tripped on the real outage', + (await circuitRecord(store, 'circuit:/v1/items')).tripped, + true, + ); + note( + '(e) → the breaker still does its job', + 'only the credential statuses were moved off it; the 4th call fast-failed at 503 as designed', + ); + } + + // ── (f) the interaction nobody sets out to configure: `retry` triples the 401s ───────────── + // Without the exclusion, a `retry` policy that lists 401 turns one broken tenant's single call + // into `attempts` circuit failures. Measured against the plain construction. + { + const { clock, store, vendor } = fixture({}); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + circuit: { failures: 5, cooldown: '30s' }, + retry: { + attempts: 3, + on: [401, 500], + backoff: { curve: 'fixed', base: 10 }, + }, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + const p = call(BAD)({}).safe(); + await clock.advance(1000); + await p; + check( + '(f) vendor requests for ONE logical call', + vendor.forTenant(BAD).length, + 3, + ); + check( + '(f) circuit failures recorded by that one call', + (await circuitRecord(store, 'circuit:/v1/items')).failures, + 1, + ); + note( + '(f) → the breaker counts the CALL, not the attempts', + '`attemptWithCircuit` wraps the whole retry loop (engine.ts:846-893), so a retried 401 is one failure — but it is still 3 requests the vendor sees from a credential that will never work', + ); + } + + finish( + 'C3', + 'YES, and the trade the capture fears is avoidable. Baseline first: three 401s recorded `failures: 3` and tripped the breaker, because a bad status reaches `attemptWithCircuit` as a THROW (engine.ts:824-831,879-890). `verdict: { accept: [401] }` ALONE is exactly the swallowing case — 4 of 4 calls measured "ok" and the caller was handed `{"error":"invalid_token"}` as its DATA, with 0 circuit failures. The construction that does both is `verdict: { accept: [401], flag: "ok" }`, PURE CONFIG: the bad tenant got StitchError status 401 (message "verdict.flag `ok` is false") on all 5 calls, 0 of 9 healthy tenants failed, and the breaker recorded 0 failures and never tripped. It works because the engine already routes on WHAT failed (engine.ts:807-833) — a bad STATUS throws and is counted, a response the SURFACE rejected returns `{ ok: false }`, fails the call and records a circuit SUCCESS. For a vendor whose 401 body carries no falsy flag, 5 lines of `Surface.interpret` composing `verdictOf` do the same ("credential rejected (HTTP 401)", 0 of 9 healthy failed, 0 circuit failures). The exclusion is surgical: under that same config a genuine 500 outage measured 500,500,500,503 and tripped the breaker as designed', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c4-noisy-neighbour.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c4-noisy-neighbour.ts new file mode 100644 index 00000000..96f79dd4 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c4-noisy-neighbour.ts @@ -0,0 +1,192 @@ +// C4 — noisy neighbour. With `throttle: { rate }` on a shared seam, does one tenant's burst consume +// other tenants' budget? Measure per-tenant ARRIVAL TIMES on an injected clock. +// +// It does, and the number is exact because the clock is. A seam's throttle re-keys EVERY member +// acquire onto one seam-stable key (`seam:${seamId}`, seam.ts:51-69), so a 20-call burst from one +// customer at `'10/s'` (a 100ms minimum spacing) reserves the next 2000ms of grants and the quiet +// customer's single call leaves at t=2000 instead of t=0. +// +// The concurrency half is sharper still and the capture does not mention it: `concurrency` is a +// FIFO queue over the same shared key (resilience.ts:120-127,159-177), so a quiet tenant is not +// merely slowed, it is queued BEHIND every call the noisy tenant already placed. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c4-noisy-neighbour.ts +import { seam } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { FakeVendor } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +const NOISY = 'noisy'; +const QUIET = 'quiet'; +const BURST = 20; + +/** + * One seam, one throttle, `.as()` per customer — the construction the scenario is about. Fires + * `BURST` calls for the noisy tenant and ONE for the quiet tenant, all in the same tick, then runs + * the virtual clock out and reports arrival times. + */ +async function noisyNeighbour( + throttle: NonNullable, + slow: Record = {}, +) { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, slow }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + throttle, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + const inFlight = [ + ...Array.from({ length: BURST }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await clock.advance(120_000); + await Promise.all(inFlight); + return { clock, store, vendor }; +} + +async function main(): Promise { + heading("C4 — one tenant's burst against everyone else's budget"); + + // ── (a) the rate case: the quiet tenant's arrival time ───────────────────────────────────── + { + const { vendor } = await noisyNeighbour({ rate: '10/s' }); + const quiet = vendor.arrivals(QUIET); + const noisy = vendor.arrivals(NOISY); + check('(a) requests the vendor saw', vendor.calls.length, BURST + 1); + checkSeq( + "(a) the quiet tenant's single call left at (virtual ms)", + quiet, + [2000], + ); + check( + '(a) the noisy burst occupied', + `${noisy[0]}..${noisy.at(-1)}`, + '0..1900', + ); + check( + '(a) → virtual ms the quiet tenant waited for a budget it did not spend', + quiet[0], + 2000, + ); + note( + "(a) → `'10/s'` is a 100ms MINIMUM SPACING, not a bucket (types.ts:1006-1031)", + 'the burst reserves `nextGrantAt` 20 slots ahead (store.ts:187-231), and the quiet tenant queues behind all of them', + ); + } + + // ── (b) the budget is ONE key, and `.as()` does not split it ─────────────────────────────── + { + const { store } = await noisyNeighbour({ rate: '10/s' }); + const rl = store.keys('rl:'); + check('(b) rate-counter keys for 2 different principals', rl.length, 1); + // The id itself is a process-wide counter (`s1`, `s2`, … — seam.ts:38,233), so the SHAPE is + // what is asserted. C5 (g) measures what that counter costs across processes. + check( + '(b) …and its shape', + /^rl:seam:s\d+:\d+$/.test(rl[0] ?? ''), + true, + ); + check( + '(b) does the key contain a principal?', + rl.some((k) => k.includes(NOISY) || k.includes(QUIET)), + false, + ); + note('(b) the measured key', rl[0]); + note( + '(b) → `seamBucket` re-keys EVERY acquire onto `seam:${seamId}` (seam.ts:51-69)', + "the member's own key is discarded (`acquire: (_key, opts) => inner.acquire(key, opts)`, seam.ts:64), so nothing a member declares can widen or split the seam budget", + ); + } + + // ── (c) severity scales with the burst, not with the window ──────────────────────────────── + // `'600/m'` declares the same 100ms spacing as `'10/s'` — the limiter reads only the RATIO — so + // a "generous per-minute quota" buys the quiet tenant nothing. + { + const perMinute = await noisyNeighbour({ rate: '600/m' }); + checkSeq( + '(c) quiet arrival under `600/m`', + perMinute.vendor.arrivals(QUIET), + [2000], + ); + const slower = await noisyNeighbour({ rate: '2/s' }); + checkSeq( + '(c) quiet arrival under `2/s`', + slower.vendor.arrivals(QUIET), + [10_000], + ); + note( + '(c) → the tighter the declared rate, the worse the neighbour damage', + "a 20-call burst at `2/s` pushed one unrelated customer's single call out by 10 virtual seconds", + ); + } + + // ── (d) the concurrency case: the quiet tenant is QUEUED, not just paced ─────────────────── + // No `rate` at all — just a cap on simultaneous calls, which is the other half of + // `ThrottleOptions`. The noisy tenant's calls are slow; the quiet tenant's is instant and still + // waits for all 20 of them, because the waiter queue is FIFO over one shared key. + { + const { vendor } = await noisyNeighbour( + { concurrency: 2 }, + { [NOISY]: 500 }, + ); + const quiet = vendor.arrivals(QUIET); + check('(d) requests the vendor saw', vendor.calls.length, BURST + 1); + checkSeq( + "(d) the quiet tenant's call left at (virtual ms)", + quiet, + [5000], + ); + check( + '(d) noisy calls that went out BEFORE it', + vendor.arrivals(NOISY).filter((at) => at < quiet[0]!).length, + 20, + ); + note( + '(d) → concurrency waiters are served FIFO over the shared key (resilience.ts:120-127,159-177)', + 'the quiet tenant is behind the whole queue: it is not slowed proportionally, it is last', + ); + } + + // ── (e) the isolated baseline, so the numbers above have a zero to be measured against ───── + // The identical burst with NO shared throttle: the quiet tenant leaves at t=0. + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + const inFlight = [ + ...Array.from({ length: BURST }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await clock.advance(1000); + await Promise.all(inFlight); + checkSeq( + '(e) quiet arrival with no shared budget', + vendor.arrivals(QUIET), + [0], + ); + check('(e) rate-counter keys written', store.keys('rl:').length, 0); + } + + finish( + 'C4', + "CONFIRMED, with an exact number. On a shared seam at `throttle: { rate: \"10/s\" }`, a 20-call burst from ONE customer pushed an unrelated customer's single call from t=0 to t=2000 virtual ms — the noisy tenant occupied 0..1900 and the quiet one queued behind all of it. The budget is one key: two different principals touched exactly ONE key, `rl:seam:s:`, because `seamBucket` DISCARDS the member's key and re-keys every acquire onto `seam:${seamId}` (seam.ts:51-69,64). A longer window does not help — `600/m` declares the same 100ms spacing as `10/s` and measured the same 2000ms — while a tighter one is worse: the same burst at `2/s` cost the quiet tenant 10 virtual seconds. THE CONCURRENCY HALF IS SHARPER AND THE CAPTURE DOES NOT MENTION IT: with `concurrency: 2` and no rate at all, the quiet tenant's instant call left at t=5000 with all 20 of the noisy tenant's slow calls ahead of it, because waiters are served FIFO over the same shared key (resilience.ts:120-127,159-177) — it is not slowed proportionally, it is last. The isolated baseline for all of these is t=0", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c5-partition-the-throttle.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c5-partition-the-throttle.ts new file mode 100644 index 00000000..3b091eca --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c5-partition-the-throttle.ts @@ -0,0 +1,421 @@ +// C5 — DECIDING. Can the throttle be partitioned per tenant AT ALL? `ThrottleOptions.pool` is +// `'stitch' | 'host'` (types.ts:1039) — no `'principal'`. Try `seam.as()`, a per-tenant seam, a +// per-tenant stitch, `key`. +// +// THE CAPTURE'S HYPOTHESIS IS WRONG IN THE OPTIMISTIC DIRECTION. It says "the rate bucket looks +// un-partitionable by tenant", and it is partitionable — twice over. What is missing is only the +// DECLARATION: there is no `pool: 'principal'` and no `throttle.key` (a), so the partition has to +// be smuggled in through the limiter key, which the engine derives from the stitch's NAME +// (engine.ts:265-274) or, on a seam, from the SEAM ID (seam.ts:59). Both work, both measured at +// t=0 for the quiet tenant. +// +// And the asymmetry in (f) is the thing to put in the docs: a per-tenant SEAM isolates the rate +// bucket and NOT the breaker, while a per-tenant NAME isolates the breaker and only isolates the +// rate bucket if the throttle is declared on the MEMBER. The two resources are keyed by different +// rules, so one construction cannot be reasoned about — each has to be checked. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c5-partition-the-throttle.ts +import { seam } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeVendor, blastRadius, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +const NOISY = 'noisy'; +const QUIET = 'quiet'; +const BURST = 20; +const RATE = '10/s'; + +/** + * Fire `BURST` calls for the noisy tenant and ONE for the quiet tenant in the same tick, through + * whatever per-tenant construction `build` returns, then run the clock out. The quiet tenant's + * arrival time is the answer: 0 = isolated, 2000 = sharing one budget. + */ +async function race( + build: (ctx: ReturnType) => (tenant: string) => Stitch, +) { + const ctx = context(); + const call = build(ctx); + const inFlight = [ + ...Array.from({ length: BURST }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await ctx.clock.advance(120_000); + await Promise.all(inFlight); + return ctx; +} + +function context() { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + return { clock, store, vendor }; +} + +async function main(): Promise { + heading('C5 — partitioning the rate budget per tenant'); + + // ── (a) what the compiler admits in the `throttle` envelope ──────────────────────────────── + { + const results = probeSpellings([ + { + label: "pool: 'stitch'", + code: `stitch({ url: 'https://x.test/y', throttle: { rate: '10/s', pool: 'stitch' } });`, + }, + { + label: "pool: 'host'", + code: `stitch({ url: 'https://x.test/y', throttle: { rate: '10/s', pool: 'host' } });`, + }, + { + label: "pool: 'principal'", + code: `stitch({ url: 'https://x.test/y', throttle: { rate: '10/s', pool: 'principal' } });`, + }, + { + label: 'throttle.key', + code: `stitch({ url: 'https://x.test/y', throttle: { rate: '10/s', key: tenantId } });`, + }, + { + label: "throttle.tenancy: 'principal'", + code: `stitch({ url: 'https://x.test/y', throttle: { rate: '10/s', tenancy: 'principal' } });`, + }, + { + label: "seam-level pool: 'principal'", + code: `seam({ baseUrl: 'https://x.test', throttle: { rate: '10/s', pool: 'principal' } });`, + }, + ]); + checkSeq('(a) throttle spellings that COMPILE', accepted(results), [ + "pool: 'stitch'", + "pool: 'host'", + ]); + check( + '(a) spellings the compiler REFUSED', + rejected(results).length, + 4, + ); + note( + '(a) → the declaration genuinely does not exist', + "`ThrottleOptions` is `{ rate?, concurrency?, pool?: 'stitch' | 'host', delegate?, on? }` (types.ts:1005-1049) — no `key`, no `tenancy`, no `'principal'` pool", + ); + } + + // ── (b) the baseline this claim is measured against ──────────────────────────────────────── + { + const { vendor, store } = await race((ctx) => { + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + throttle: { rate: RATE }, + }); + return (t) => + s + .as(t) + .stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + }); + checkSeq( + '(b) shared seam + `.as()` — quiet arrival', + vendor.arrivals(QUIET), + [2000], + ); + check('(b) distinct rate keys', store.keys('rl:').length, 1); + } + + // ── (c) a per-tenant SEAM isolates the budget ────────────────────────────────────────────── + // Even sharing one store: the bucket key carries the seam id (seam.ts:59), and each `seam()` + // call mints a fresh one. + { + const { vendor, store } = await race((ctx) => { + const seams = new Map>(); + return (t) => { + let sm = seams.get(t); + if (!sm) { + sm = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, // ONE shared store + clock: ctx.clock, + throttle: { rate: RATE }, + }); + seams.set(t, sm); + } + return sm + .as(t) + .stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + }; + }); + checkSeq( + '(c) per-tenant SEAM — quiet arrival', + vendor.arrivals(QUIET), + [0], + ); + check('(c) distinct rate keys', store.keys('rl:').length, 2); + note( + '(c) → the partition is the SEAM ID, not the principal', + '`seamBucket` keys on `seam:${seamId}` (seam.ts:51-69); two seams are two budgets even over one store', + ); + } + + // ── (d) a per-tenant NAME + a MEMBER throttle isolates it too ────────────────────────────── + // No seam-level throttle here: the member's own throttle is keyed by the engine's `hostKey`, + // which falls back to `cfg.name` (engine.ts:140,273,614). + { + const { vendor, store } = await race((ctx) => { + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + }); + const made = new Map(); + return (t) => { + let st = made.get(t); + if (!st) { + st = s.as(t).stitch({ + name: `items:${t}`, + path: '/v1/items', + headers: { 'x-tenant': t }, + throttle: { rate: RATE }, + }); + made.set(t, st); + } + return st; + }; + }); + checkSeq( + '(d) per-tenant NAME + member throttle — quiet arrival', + vendor.arrivals(QUIET), + [0], + ); + checkSeq('(d) rate keys', store.keys('rl:'), [ + `rl:items:${NOISY}:0`, + `rl:items:${QUIET}:0`, + ]); + } + + // ── (e) THE TRAP: a per-tenant member throttle with the SAME name shares one counter ─────── + // 2 stitch objects, 2 in-process limiters — and one budget, because the counter lives in the + // seam's shared store under the config-derived key. This is the construction that looks + // isolated and is not. + { + const { vendor, store } = await race((ctx) => { + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + }); + const made = new Map(); + return (t) => { + let st = made.get(t); + if (!st) { + st = s.as(t).stitch({ + path: '/v1/items', // no per-tenant name + headers: { 'x-tenant': t }, + throttle: { rate: RATE }, + }); + made.set(t, st); + } + return st; + }; + }); + checkSeq( + '(e) same path, per-tenant member throttle — quiet arrival', + vendor.arrivals(QUIET), + [2000], + ); + checkSeq('(e) rate keys', store.keys('rl:'), ['rl:/v1/items:0']); + note( + '(e) → each stitch got its OWN `createStoreThrottle` (seam.ts:97-103)', + 'and they all `increment` the same `rl::` counter (store.ts:203-206), so the per-process objects are decoration', + ); + } + + // ── (f) THE ASYMMETRY: the same construction isolates one resource and not the other ─────── + // One run, both resources measured. A per-tenant seam sharing a store: rate ISOLATED (the seam + // id is in the key), breaker SHARED (it is not). + { + const ctx = context(); + ctx.vendor.fail('broken', 401); + const seams = new Map>(); + const call = (t: string) => { + let sm = seams.get(t); + if (!sm) { + sm = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + throttle: { rate: RATE }, + circuit: { failures: 3, cooldown: '30s' }, + }); + seams.set(t, sm); + } + return sm + .as(t) + .stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + }; + // rate: the noisy burst against a quiet tenant. + const inFlight = [ + ...Array.from({ length: BURST }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await ctx.clock.advance(120_000); + await Promise.all(inFlight); + // breaker: the broken tenant, then three healthy ones. + for (let i = 0; i < 3; i++) { + const p = outcomeOf(() => call('broken')({})); + await ctx.clock.advance(1000); + await p; + } + const healthy: string[] = []; + for (const t of ['h1', 'h2', 'h3']) { + const p = outcomeOf(() => call(t)({})); + await ctx.clock.advance(1000); + healthy.push(await p); + } + check( + '(f) rate keys (one per seam) → ISOLATED', + ctx.store.keys('rl:').length >= 2, + true, + ); + checkSeq( + '(f) quiet tenant arrival → ISOLATED', + ctx.vendor.arrivals(QUIET), + [0], + ); + checkSeq('(f) breaker keys → SHARED', ctx.store.keys('circuit:'), [ + 'circuit:/v1/items', + ]); + check('(f) → healthy tenants that FAILED', blastRadius(healthy), 3); + note( + '(f) → one construction, two answers', + 'per-tenant seams isolate the rate budget (`seam:sN`) and NOT the breaker (`cfg.name ?? cfg.path`); nothing in the config surface says so', + ); + } + + // ── (g) `pool: 'host'` collapses every partition, INCLUDING the breaker ──────────────────── + // The one declared pooling knob widens rather than narrows — and it silently re-keys the + // circuit too, because `hostKey` reads `cfg.throttle?.pool` (engine.ts:265-274) and the circuit + // uses `hostKey` as its fallback key (engine.ts:860). + { + const { vendor, store } = await race((ctx) => { + const seams = new Map>(); + return (t) => { + let sm = seams.get(t); + if (!sm) { + sm = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + throttle: { rate: RATE, pool: 'host' }, + }); + seams.set(t, sm); + } + return sm.as(t).stitch({ + name: `items:${t}`, // a per-tenant name, which now buys nothing + path: '/v1/items', + headers: { 'x-tenant': t }, + }); + }; + }); + checkSeq( + "(g) `pool: 'host'` over per-tenant seams AND names — quiet arrival", + vendor.arrivals(QUIET), + [2000], + ); + checkSeq('(g) rate keys', store.keys('rl:'), ['rl:api.vendor.test:0']); + + // …and the breaker moves with it. + const ctx = context(); + ctx.vendor.fail('broken', 500); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + circuit: { failures: 3, cooldown: '30s' }, + throttle: { pool: 'host' }, // pooling declared for the RATE + }); + const broken = s.as('broken').stitch({ + name: 'items:broken', + path: '/v1/items', + headers: { 'x-tenant': 'broken' }, + }); + const unrelated = s.as('other').stitch({ + name: 'orders:other', // a different endpoint AND a different tenant + path: '/v1/orders', + headers: { 'x-tenant': 'other' }, + }); + for (let i = 0; i < 3; i++) await outcomeOf(() => broken({})); + checkSeq( + '(g) breaker keys under `pool: host`', + ctx.store.keys('circuit:'), + ['circuit:api.vendor.test'], + ); + check( + '(g) → an unrelated endpoint for an unrelated tenant', + await outcomeOf(() => unrelated({})), + '503', + ); + note( + '(g) → `throttle.pool` silently re-keys the CIRCUIT', + '`hostKey` reads `cfg.throttle?.pool === "host"` (engine.ts:265-274) and is the circuit\'s fallback key (engine.ts:860); a per-tenant `name` partition evaporates when someone tunes the rate pool', + ); + } + + // ── (h) the seam id is a creation-ORDER counter, which does not survive two processes ────── + // `seam:s1` in worker A and `seam:s1` in worker B are the same key in a shared Redis. The ids + // are handed out by construction order (seam.ts:38,233), which is tenant order only by + // accident, so per-tenant seams over a shared store cross-pollinate across the fleet. + { + const ctx = context(); + const order = ['zulu', 'alpha', 'mike']; + for (const t of order) { + const sm = seam({ + baseUrl: 'https://api.vendor.test', + adapter: ctx.vendor.adapter(), + store: ctx.store, + clock: ctx.clock, + throttle: { rate: RATE }, + }); + const p = sm + .as(t) + .stitch({ path: '/v1/items', headers: { 'x-tenant': t } })({}) + .safe(); + await ctx.clock.advance(1000); + await p; + } + const ids = ctx.store + .keys('rl:seam:') + .map((k) => k.split(':')[2] ?? '') + .map((s) => Number(s.slice(1))); + check('(h) seams created', ids.length, 3); + check( + '(h) their ids are consecutive (creation order, not tenant)', + ids[1] === ids[0]! + 1 && ids[2] === ids[1]! + 1, + true, + ); + check( + '(h) does any id derive from the tenant name?', + ctx.store + .keys('rl:seam:') + .some((k) => order.some((t) => k.includes(t))), + false, + ); + note( + '(h) → `seamCounter` is a module-level counter (seam.ts:38,233)', + "two processes each hand out s1, s2, s3 — so over a SHARED store, worker A's tenant-1 seam and worker B's tenant-7 seam are the same rate bucket", + ); + } + + finish( + 'C5', + 'YES — the capture is wrong in the optimistic direction. What is missing is the DECLARATION, not the capability: of six candidate spellings typechecked, only `pool: "stitch"` and `pool: "host"` compile — `pool: "principal"`, `throttle.key` and `throttle.tenancy` are compile errors (types.ts:1005-1049). But the budget IS partitionable, two ways, both measured with the quiet tenant leaving at t=0 instead of t=2000: a per-tenant SEAM (the bucket key carries the seam id, `rl:seam:sN`, seam.ts:51-69 — and it holds even over ONE shared store), or a per-tenant `name` plus a MEMBER-level throttle (`rl:items:`, engine.ts:140,273,614). TWO TRAPS. Per-tenant member throttles with the SAME name are 2 limiter objects over ONE store counter (`rl:/v1/items:0`, quiet at t=2000): the objects are decoration. And `pool: "host"` collapses every partition, per-tenant seams and per-tenant names alike, back to `rl:api.vendor.test` — while ALSO silently re-keying the CIRCUIT onto `circuit:api.vendor.test` (engine.ts:265-274,860), where an unrelated endpoint for an unrelated tenant measured 503. THE ASYMMETRY IS THE HEADLINE: in one run, per-tenant seams over a shared store isolated the rate budget (quiet at t=0) and SHARED the breaker (one key, 3 of 3 healthy tenants down). The two resources are keyed by different rules and neither construction can be reasoned about as a whole. Finally, seam ids are a creation-ORDER counter (measured consecutive, with no tenant derivation), so per-tenant seams over a shared durable store collide across processes', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c6-token-isolation.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c6-token-isolation.ts new file mode 100644 index 00000000..9c1351d3 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c6-token-isolation.ts @@ -0,0 +1,231 @@ +// C6 — token isolation. Confirm `oauth2({ tenancy: 'principal' })` keeps tenant tokens separate in +// the multi-tenant construction, and that one tenant's refresh storm doesn't disturb another +// tenant's in-flight calls. +// +// It does, on both counts, and this is the one axis where the library's answer is the correct one +// out of the box — the principal DOES reach the auth layer (`AuthContext.principal`, +// types.ts:1214-1219), which is exactly what the resilience layer lacks. +// +// The finding the capture does not draw out is the difference between a token and a CREDENTIAL. +// `tenancy: 'principal'` partitions the token CACHE; it does not give each customer their own +// client id/secret, because `Secret` is `string | (() => string)` (auth.ts:47) — a NILADIC thunk, +// with no `AuthContext` in scope. So a shared `oauth2()` mints every tenant's token from the SAME +// client credentials (e). The escape hatch is real and one line: a custom `AuthStrategy.apply(req, +// ctx)` DOES receive the context, so `ctx.principal` selects the credential (f). +// +// And the default is the wrong way round for this scenario: `tenancy` defaults to `'app'` +// (auth.ts:483-486), which serves every customer one shared token (b). +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c6-token-isolation.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { AuthStrategy } from '../../../../packages/core/src/types'; +import { FakeIdp, FakeVendor, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +const TOKEN_URL = 'https://idp.vendor.test/token'; + +function fixture(opts: { + tenancy?: 'principal' | 'app'; + failing?: Record; + auth?: AuthStrategy; +}) { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: opts.failing ?? {} }); + const idp = new FakeIdp(); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + auth: + opts.auth ?? + oauth2({ + tokenUrl: TOKEN_URL, + clientId: 'saas-app', + clientSecret: 'shh', + adapter: idp.adapter(), + ...(opts.tenancy !== undefined + ? { tenancy: opts.tenancy } + : {}), + }), + }); + const call = (t: string) => + s.as(t).stitch({ path: '/v1/items', headers: { 'x-tenant': t } }); + return { clock, store, vendor, idp, seam: s, call }; +} + +async function main(): Promise { + heading('C6 — per-tenant tokens in the multi-tenant construction'); + + // ── (a) `tenancy: 'principal'` — one token per customer, cached per customer ─────────────── + { + const { store, vendor, idp, call } = fixture({ tenancy: 'principal' }); + for (const t of ['t1', 't2', 't3', 't1']) await call(t)({}).safe(); + check('(a) calls made', vendor.calls.length, 4); + check('(a) token requests', idp.mints, 3); + checkSeq( + '(a) the Authorization header each call carried', + vendor.calls.map((c) => c.authorization), + [ + 'Bearer tok-saas-app-1', + 'Bearer tok-saas-app-2', + 'Bearer tok-saas-app-3', + 'Bearer tok-saas-app-1', // t1's SECOND call reused t1's token + ], + ); + check('(a) distinct vault keys', store.keys('vault:oauth2:').length, 3); + check( + "(a) …and each is the tokenUrl + '\\0' + principal", + store.keys('vault:oauth2:').every((k) => k.includes('\u0000')), + true, + ); + note( + '(a) → `keyFor(ctx)` folds `ctx.principal` in (auth.ts:485-499)', + 'the principal reaches the AUTH layer — this is the one place `seam.as()` is load-bearing at runtime', + ); + } + + // ── (b) the DEFAULT is `'app'`, and it hands every customer the same token ───────────────── + { + const { store, vendor, idp, call } = fixture({}); + for (const t of ['t1', 't2', 't3']) await call(t)({}).safe(); + check('(b) token requests for 3 different customers', idp.mints, 1); + checkSeq( + '(b) headers', + [...new Set(vendor.calls.map((c) => c.authorization))], + ['Bearer tok-saas-app-1'], + ); + check('(b) distinct vault keys', store.keys('vault:oauth2:').length, 1); + note( + "(b) → `OAuth2Options.tenancy` defaults to `'app'` (auth.ts:483-486)", + 'correct for client_credentials, wrong for a per-customer integration — and it is silent: nothing about the call site says whose token went out', + ); + } + + // ── (c) `tenancy: 'principal'` fails CLOSED with no bound principal ──────────────────────── + // The safety property that makes (a) trustworthy: you cannot accidentally get an unscoped token. + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const idp = new FakeIdp(); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + auth: oauth2({ + tokenUrl: TOKEN_URL, + clientId: 'saas-app', + clientSecret: 'shh', + adapter: idp.adapter(), + tenancy: 'principal', + }), + }); + const r = await s.stitch({ path: '/v1/items' })({}).safe(); // no `.as()` + check('(c) unbound call ok?', r.ok, false); + check( + '(c) error mentions the fix', + (r.error?.message ?? '').includes('seam.as('), + true, + ); + check('(c) token requests made', idp.mints, 0); + } + + // ── (d) one tenant's refresh storm does not disturb another's ────────────────────────────── + // The broken tenant 401s, which the strategy reads as "token rejected" and answers with a fresh + // fetch + retry — repeatedly. Measure that the healthy tenant's token is untouched and its call + // still succeeds. + { + const { vendor, idp, call } = fixture({ + tenancy: 'principal', + failing: { broken: 401 }, + }); + await call('t1')({}).safe(); + const before = vendor.forTenant('t1').map((c) => c.authorization); + const mintsAfterT1 = idp.mints; + + for (let i = 0; i < 5; i++) await outcomeOf(() => call('broken')({})); + const stormMints = idp.mints - mintsAfterT1; + + const r = await call('t1')({}).safe(); + const after = vendor.forTenant('t1').map((c) => c.authorization); + + check('(d) token fetches the storm caused', stormMints, 6); + check( + '(d) requests the broken tenant made', + vendor.forTenant('broken').length, + 10, + ); + checkSeq('(d) t1 tokens, before and after the storm', after, [ + before[0]!, + before[0]!, + ]); + check('(d) t1 still succeeds', r.ok, true); + check( + '(d) t1 vendor requests across the whole storm', + vendor.forTenant('t1').length, + 2, + ); + note( + '(d) → the storm cost 6 token fetches and 10 vendor requests', + "all of them charged to the broken tenant's own vault key; nothing crossed over", + ); + } + + // ── (e) the CREDENTIAL cannot be per-tenant on a shared `oauth2()` ───────────────────────── + // `tenancy` partitions the token cache. It does not choose which client id mints the token, + // because the resolver takes no context. + { + const { idp, call } = fixture({ tenancy: 'principal' }); + for (const t of ['t1', 't2', 't3']) await call(t)({}).safe(); + checkSeq( + '(e) client_ids the IdP saw for 3 different customers', + [...new Set(idp.clientIds)], + ['saas-app'], + ); + const niladic: () => string = () => 'x'; + check( + '(e) `Secret` thunk arity (a principal-aware resolver would take 1)', + niladic.length, + 0, + ); + note( + '(e) → `Secret = string | (() => string)` (auth.ts:47)', + 'no `AuthContext` parameter, so `clientId`/`clientSecret`/`scope` are fixed per strategy instance — per-customer credentials need one strategy (hence one seam or one stitch) per customer', + ); + } + + // ── (f) …and the escape hatch is one line, because `apply` DOES get the context ──────────── + { + const perTenant: AuthStrategy = { + name: 'per-tenant-bearer', + apply(req, ctx) { + req.headers['authorization'] = + `Bearer cred-for-${ctx.principal}`; + }, + }; + const { vendor, call } = fixture({ auth: perTenant }); + for (const t of ['t1', 't2']) await call(t)({}).safe(); + checkSeq( + '(f) a custom strategy reading `ctx.principal`', + vendor.calls.map((c) => c.authorization), + ['Bearer cred-for-t1', 'Bearer cred-for-t2'], + ); + note( + '(f) → `AuthStrategy.apply(req, ctx)` receives `AuthContext` (types.ts:1206-1233)', + 'this is the ONLY user-reachable hook in the library that can see the bound principal at call time — the resilience layer has no equivalent', + ); + } + + finish( + 'C6', + 'CONFIRMED, and it is the one axis the library gets right by construction. `oauth2({ tenancy: "principal" })` on a shared seam with `.as()` per customer minted 3 tokens for 3 customers, wrote 3 distinct vault keys (tokenUrl + NUL + principal, auth.ts:485-499), and reused t1\'s token on t1\'s second call — 4 calls, 3 fetches. It fails CLOSED: the same config called without `.as()` errored with a message naming `seam.as(` and made 0 token requests. The refresh storm is contained: 5 doomed calls from a revoked tenant cost 6 token fetches and 10 vendor requests, all charged to that tenant\'s own key, while the healthy tenant carried the identical token before and after and still succeeded. TWO THINGS TO CARRY FORWARD. The DEFAULT is `tenancy: "app"` (auth.ts:483-486) — 3 different customers measured 1 token fetch and one shared Authorization header, which is a credential bleed nothing at the call site hints at. And `tenancy` partitions the token CACHE, not the CREDENTIAL: all 3 customers\' tokens were minted from client_id `saas-app`, because `Secret = string | (() => string)` (auth.ts:47) is a niladic thunk with no `AuthContext` in scope. Per-customer credentials need one strategy instance per customer — or the one-line escape hatch, a custom `AuthStrategy.apply(req, ctx)` reading `ctx.principal`, which measured `Bearer cred-for-t1` / `cred-for-t2`. That hook is the only user-reachable place in the library that sees the bound principal at call time', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c7-cost-of-isolation.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c7-cost-of-isolation.ts new file mode 100644 index 00000000..73cddca3 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c7-cost-of-isolation.ts @@ -0,0 +1,306 @@ +// C7 — the cost of the correct construction. If isolation requires per-tenant seams/stitches, +// what does 100 tenants actually cost, what is shared vs duplicated, and is there a leak? +// +// THE CAPTURE'S COST MODEL IS WRONG. It calls "one client instance per tenant" correct-but- +// unscalable — "4,000 pools, timers and caches". Measured here: 100 per-tenant seams cost ~1ms and +// ~7kb each, arm ZERO timers, and hold ZERO connection pools, because a seam owns none of those +// things — the adapter (and its pool) is a config value that per-tenant seams SHARE. The expensive +// construction from the literature is not the expensive construction here. +// +// The real costs are three, and all three are leaks rather than footprint: +// +// • Breaker records NEVER EXPIRE. `circuit.onSuccess`/`onFailure` write with no TTL +// (resilience.ts:382-403), so one key per tenant lives forever — measured still present after a +// virtual YEAR of the tenant not existing (d). +// • A rate-paced limiter key is never dropped from its in-process map. `release` only deletes a +// key with no window bookkeeping (store.ts:249-254), so a `rate` throttle retains one entry per +// tenant key for the life of the process (e). +// • `seam.stitch()` — the ROOT builder — retains every stitch it ever made in the seam's registry +// (seam.ts:138-141). `seam.as(p).stitch()` does not. Measured with `WeakRef` after a forced GC: +// 200/200 root-created alive, 0/200 principal-created (f). +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c7-cost-of-isolation.ts +import { seam } from '../../../../packages/core/src/index'; +import { + THROTTLE_LOCAL, + createStoreThrottle, +} from '../../../../packages/core/src/store'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeVendor } from './fake-vendor'; +import { check, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; + +const N = 100; +const CIRCUIT = { failures: 5, cooldown: '30s' } as const; + +// A real GC, without needing a CLI flag — the retention question in (f) is not answerable by +// heap-size guessing, and `WeakRef` only tells the truth after a collection. +setFlagsFromString('--expose-gc'); +const gc = runInNewContext('gc') as () => void; +/** Collect, yielding to the macrotask queue between passes so finalizers actually run. */ +async function collect(): Promise { + for (let i = 0; i < 5; i++) { + gc(); + await new Promise((r) => setTimeout(r, 5)); + } +} +const heapKb = (): number => Math.round(process.memoryUsage().heapUsed / 1024); + +async function main(): Promise { + heading('C7 — what 100 isolated tenants cost, and what leaks'); + + // ── (a) 100 per-tenant SEAMS ─────────────────────────────────────────────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const adapter = vendor.adapter(); // ONE adapter — the connection pool is shared by config + await collect(); + const before = heapKb(); + const t0 = Date.now(); + const seams = Array.from({ length: N }, () => + seam({ + baseUrl: 'https://api.vendor.test', + adapter, + store, + clock, + throttle: { rate: '10/s' }, + circuit: CIRCUIT, + }), + ); + const elapsed = Date.now() - t0; + await collect(); + const perSeamKb = (heapKb() - before) / N; + + check('(a) seams constructed', seams.length, N); + check('(a) construction under 200ms', elapsed < 200, true); + check('(a) timers armed', clock.pending(), 0); + check('(a) under 40kb per seam', perSeamKb < 40, true); + note('(a) construction time (ms)', elapsed); + note('(a) heap per seam (kb)', perSeamKb.toFixed(1)); + note( + '(a) → a seam owns no transport', + '`adapter` is a config value (types.ts) and the 100 seams here share ONE; the "4,000 connection pools" cost the literature warns about is not a cost this construction has', + ); + } + + // ── (b) the alternative shape: 1 seam + 100 per-tenant KEYED stitches ────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + await collect(); + const before = heapKb(); + const t0 = Date.now(); + const calls = Array.from({ length: N }, (_, i) => + s.as(`t${i}`).stitch({ + name: `items:t${i}`, + path: '/v1/items', + headers: { 'x-tenant': `t${i}` }, + throttle: { rate: '10/s' }, + circuit: { ...CIRCUIT, key: `items:t${i}` }, + }), + ); + const elapsed = Date.now() - t0; + await collect(); + const perTenantKb = (heapKb() - before) / N; + check('(b) stitches constructed', calls.length, N); + check('(b) construction under 200ms', elapsed < 200, true); + check('(b) timers armed', clock.pending(), 0); + check('(b) under 40kb per tenant', perTenantKb < 40, true); + note('(b) construction time (ms)', elapsed); + note('(b) heap per tenant (kb)', perTenantKb.toFixed(1)); + note( + '(b) → the two shapes cost the same order of magnitude', + 'so the choice between them is about which resource each one isolates (C2/C5), not about scale', + ); + } + + // ── (c) what 100 isolated tenants put in the store ───────────────────────────────────────── + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + const inFlight = Array.from({ length: N }, (_, i) => + s + .as(`t${i}`) + .stitch({ + name: `items:t${i}`, + path: '/v1/items', + headers: { 'x-tenant': `t${i}` }, + throttle: { rate: '10/s' }, + circuit: { ...CIRCUIT, key: `items:t${i}` }, + })({}) + .safe(), + ); + await clock.advance(60_000); + await Promise.all(inFlight); + check( + '(c) breaker keys after ONE call each', + store.keys('circuit:').length, + N, + ); + check('(c) rate-counter keys', store.keys('rl:').length, N); + check('(c) → store keys per tenant', store.keys().length / N, 2); + note( + "(c) → at 500 customers × 8 connections (the capture's number)", + `that is ${(4000 * 2).toLocaleString('en-US')} keys in the shared store, of which half never expire — see (d)`, + ); + } + + // ── (d) THE LEAK: breaker records have no TTL ─────────────────────────────────────────────── + // A customer churns out. Their per-tenant breaker key stays in Redis forever, because + // `circuit.onSuccess`/`onFailure` call `store.set(key, record)` with no `ttl` + // (resilience.ts:382-403), and the store treats a missing ttl as "live forever" (store.ts:45). + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + await s + .as('churned') + .stitch({ + path: '/v1/items', + headers: { 'x-tenant': 'churned' }, + circuit: { ...CIRCUIT, key: 'items:churned' }, + })({}) + .safe(); + const born = store.live('circuit:').length; + await clock.advance(365 * 24 * 60 * 60 * 1000); // a virtual year of not existing + const stillThere = + (await store.get('circuit:items:churned')) !== undefined; + check('(d) breaker keys written', born, 1); + check('(d) still resident after a virtual YEAR', stillThere, true); + check('(d) live circuit keys', store.live('circuit:').length, 1); + note( + '(d) → the rate counter DOES expire (`rate.per + 100` ms, store.ts:203-206)', + 'the breaker record does not; a per-tenant breaker is one immortal key per tenant per endpoint, and nothing in the library sweeps them', + ); + } + + // ── (e) …and the in-process limiter state for a rate-paced key is never dropped ──────────── + // Measured on the primitive directly, since the engine's instances are internal. `release` + // deletes a key only when it carries no window bookkeeping (store.ts:249-254) — true for a + // concurrency-only limiter, never for a rate-paced one. + { + const clock = manualClock(); + const store = probeStore(); + const paced = createStoreThrottle({ rate: '10/s' }, store, clock); + const pacedLocal = ( + paced as unknown as Record> + )[THROTTLE_LOCAL]!; + for (let i = 0; i < N; i++) { + await paced.acquire(`items:t${i}`); + paced.release(`items:t${i}`); + } + check( + '(e) rate-paced limiter: entries retained after release', + pacedLocal.size, + N, + ); + + const capped = createStoreThrottle({ concurrency: 2 }, store, clock); + const cappedLocal = ( + capped as unknown as Record> + )[THROTTLE_LOCAL]!; + for (let i = 0; i < N; i++) { + await capped.acquire(`items:t${i}`); + capped.release(`items:t${i}`); + } + check( + '(e) concurrency-only limiter: entries retained', + cappedLocal.size, + 0, + ); + note( + '(e) → the retention is deliberate and documented (store.ts:243-254)', + 'dropping a rate-paced key mid-pace would reset its cursor and let the next acquire burst — but the consequence for a per-TENANT key is unbounded growth in a long-lived process', + ); + } + + // ── (f) THE OTHER LEAK: `seam.stitch()` retains; `seam.as(p).stitch()` does not ──────────── + // The registry exists for lifecycle/introspection and is only populated by the ROOT builder + // (seam.ts:136-141). Caching one ROOT-created stitch per tenant — the obvious optimisation — + // pins every one of them for the life of the seam. + { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock }); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + }); + const M = 200; + const rootRefs = Array.from( + { length: M }, + (_, i) => + new WeakRef( + s.stitch({ name: `root:${i}`, path: '/v1/items' }), + ), + ); + const principalRefs = Array.from( + { length: M }, + (_, i) => + new WeakRef( + s + .as(`t${i}`) + .stitch({ name: `principal:${i}`, path: '/v1/items' }), + ), + ); + await collect(); + const aliveRoot = rootRefs.filter( + (r) => r.deref() !== undefined, + ).length; + const alivePrincipal = principalRefs.filter( + (r) => r.deref() !== undefined, + ).length; + check('(f) root-created stitches still reachable', aliveRoot, M); + check( + '(f) principal-created stitches still reachable', + alivePrincipal, + 0, + ); + + await s.close(); + await collect(); + check( + '(f) root-created still reachable after `seam.close()`', + rootRefs.filter((r) => r.deref() !== undefined).length, + 0, + ); + note( + '(f) → `runtime.register` is set only when `principal === undefined` (seam.ts:136-141)', + 'so the per-request shape (`seam.as(id).stitch(...)`) is the one that does NOT leak, and the only way to free a root registry is `close()`, which also closes the store', + ); + } + + finish( + 'C7', + "CHEAP IN FOOTPRINT, LEAKY IN STATE — and the capture's cost model does not apply. 100 per-tenant seams constructed in single-digit ms at well under 40kb each, arming ZERO timers, because a seam owns no transport: `adapter` is a config value and all 100 shared one, so the \"4,000 connection pools\" the literature warns about is not a cost this construction has. 1 seam + 100 per-tenant KEYED stitches measured the same order of magnitude, so the choice between the two shapes is about which resource each isolates (C2/C5), not about scale. The real price is THREE pieces of state that are never freed. (1) Breaker records have NO TTL (resilience.ts:382-403 writes with no `ttl`; store.ts:45 treats that as live-forever): a churned tenant's key was still resident after a virtual YEAR — at the capture's 4,000 connections that is 4,000 immortal keys, and nothing in the library sweeps them, while the rate counter beside it does expire. (2) A rate-paced limiter retains one in-process map entry per key for the life of the process — 100/100 after acquire+release, versus 0/100 for a concurrency-only limiter (store.ts:243-254). (3) `seam.stitch()` pins every stitch it creates in the seam registry: measured with WeakRef after a forced GC, 200/200 root-created still reachable versus 0/200 created through `seam.as(p).stitch()`, and the only release is `seam.close()` — which also closes the store. The per-request shape is the one that does not leak", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/c8-four-resources.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/c8-four-resources.ts new file mode 100644 index 00000000..f744e26c --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/c8-four-resources.ts @@ -0,0 +1,355 @@ +// C8 — state plainly, for each of the four shared resources, whether it ends up isolated per tenant +// and by what mechanism. Then run the whole construction and measure the blast radius. +// +// The one-line answer: TWO of the four are isolated by the principal, and TWO are isolated only by +// a string you have to remember to write. +// +// token → `oauth2({ tenancy: 'principal' })` + `seam.as(id)` — the PRINCIPAL. Fail-closed. +// cache → `tenancy: 'principal'`, the DEFAULT — the PRINCIPAL. Fail-closed. +// rate → a per-tenant limiter KEY (member `name` or a per-tenant seam). No principal. +// breaker → a per-tenant `circuit.key`. No principal. +// +// The split is exactly the auth/resilience line: `AuthContext.principal` (types.ts:1214-1219) is +// threaded to the auth strategies and the cache-key builder, and to nothing else. So the two +// resources whose isolation is a SECURITY property fail closed, and the two whose isolation is an +// AVAILABILITY property fail open — silently, with no type error and no warning. +// +// (e) is the honest caveat on the assembled answer: a global vendor quota and per-tenant fairness +// cannot both be had from the built-ins, because a seam-level throttle re-introduces exactly the +// coupling the per-tenant keys removed. +// +// pnpm exec tsx docs/scenarios/proofs/multi-tenant-blast-radius/c8-four-resources.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { seam, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeIdp, FakeVendor, blastRadius, outcomeOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; + +const HEALTHY = ['t1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9']; +const BAD = 'bad'; +const NOISY = 'noisy'; +const QUIET = 'quiet'; + +/** C3's surface: a 401/403 is a CREDENTIAL verdict, so it fails the CALL without failing the HOST. */ +const credentialAware: Surface = { + id: 'http', + interpret: (res, cfg) => { + if (res.status === 401 || res.status === 403) + return { + ok: false, + message: `credential rejected (HTTP ${res.status})`, + status: res.status, + }; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, +}; + +/** + * THE ASSEMBLED CONSTRUCTION. One seam carries everything genuinely shared — store, vault, clock, + * adapter (and its connection pool), trace sink, the auth strategy, the credential-aware verdict. + * Each tenant's member carries the three per-tenant STRINGS that do the isolating. + */ +function tenantAware(opts: { seamThrottle?: boolean } = {}) { + const clock = manualClock(); + const store = probeStore(); + const vendor = new FakeVendor({ clock, failing: { [BAD]: 401 } }); + const idp = new FakeIdp(); + const s = seam({ + baseUrl: 'https://api.vendor.test', + adapter: vendor.adapter(), + store, + clock, + // token: per-tenant, fail-closed, by the PRINCIPAL. + auth: oauth2({ + tokenUrl: 'https://idp.vendor.test/token', + clientId: 'saas-app', + clientSecret: 'shh', + adapter: idp.adapter(), + tenancy: 'principal', + }), + // a credential failure is not a dependency failure (C3). + verdict: { accept: [401, 403] }, + // (e) toggles the global vendor quota on, to measure what declaring it costs. + ...(opts.seamThrottle ? { throttle: { rate: '10/s' } } : {}), + }); + const made = new Map(); + const call = (tenant: string): Stitch => { + let st = made.get(tenant); + if (!st) { + st = s.as(tenant).stitch({ + // rate: the limiter key comes from the NAME (engine.ts:140,273). + name: `items:${tenant}`, + path: '/v1/items', + headers: { 'x-tenant': tenant }, + kind: credentialAware, + throttle: { rate: '10/s' }, + // breaker: the ONE knob that partitions it. + circuit: { + failures: 3, + cooldown: '30s', + key: `items:${tenant}`, + }, + }); + made.set(tenant, st); + } + return st; + }; + return { clock, store, vendor, idp, seam: s, call }; +} + +/** + * Every member here declares its own `rate`, so a SEQUENTIAL call needs the virtual clock moved + * before it can be granted. `drive` fires the call, runs the clock past any pacing wait, and + * returns the settled outcome — the ordering the breaker claims depend on, on an injected clock. + */ +/** Which per-tenant breakers are actually OPEN — the read that distinguishes "keyed" from "tripped". */ +async function trippedBreakers( + store: ReturnType, +): Promise { + const open: string[] = []; + for (const key of store.keys('circuit:')) { + const r = (await store.get(key)) as { tripped?: boolean } | undefined; + if (r?.tripped) open.push(key); + } + return open; +} + +async function drive( + clock: ReturnType, + call: () => PromiseLike, +): Promise { + const p = outcomeOf(call); + await clock.advance(1000); + return p; +} + +async function main(): Promise { + heading('C8 — the four shared resources, and the assembled answer'); + + // ── (a) the blast radius of a revoked credential, end to end ─────────────────────────────── + { + const { clock, store, vendor, call } = tenantAware(); + const bad: string[] = []; + for (let i = 0; i < 5; i++) + bad.push(await drive(clock, () => call(BAD)({}))); + const healthy: string[] = []; + for (const t of HEALTHY) + healthy.push(await drive(clock, () => call(t)({}))); + + checkSeq( + '(a) the broken tenant still gets a real error', + [...new Set(bad)], + ['401'], + ); + check('(a) healthy tenants called', healthy.length, 9); + check( + '(a) → BLAST RADIUS (healthy tenants that failed)', + blastRadius(healthy), + 0, + ); + check( + '(a) healthy requests that reached the vendor', + vendor.calls.filter((c) => c.tenant !== BAD).length, + 9, + ); + check( + '(a) distinct breaker keys (one per tenant)', + store.keys('circuit:').length, + 10, + ); + checkSeq( + '(a) → breakers that TRIPPED', + await trippedBreakers(store), + [], + ); + note( + '(a) → C1 measured 9 of 9 healthy tenants down under the naive construction', + 'the same failure under this one is 0 of 9, and the broken tenant is still told its credential is bad', + ); + } + + // ── (b) …and a REAL outage still trips, per tenant ────────────────────────────────────────── + // The exclusion must not have disarmed the breaker. A tenant hitting a genuinely broken + // endpoint gets its own breaker opened; nobody else does. + { + const { clock, store, vendor, call } = tenantAware(); + vendor.fail('degraded', 500); + const spine: string[] = []; + for (let i = 0; i < 4; i++) + spine.push(await drive(clock, () => call('degraded')({}))); + const others: string[] = []; + for (const t of ['t1', 't2', 't3']) + others.push(await drive(clock, () => call(t)({}))); + checkSeq('(b) the degraded tenant', spine, [ + '500', + '500', + '500', + '503', + ]); + check('(b) → other tenants that failed', blastRadius(others), 0); + check('(b) distinct breaker keys', store.keys('circuit:').length, 4); + checkSeq('(b) → breakers that TRIPPED', await trippedBreakers(store), [ + 'circuit:items:degraded', + ]); + } + + // ── (c) the noisy neighbour, under the same construction ─────────────────────────────────── + { + const { clock, vendor, call } = tenantAware(); + const inFlight = [ + ...Array.from({ length: 20 }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await clock.advance(120_000); + await Promise.all(inFlight); + checkSeq( + '(c) the quiet tenant left at (virtual ms)', + vendor.arrivals(QUIET), + [0], + ); + check( + '(c) the noisy tenant was still paced at its own 10/s', + `${vendor.arrivals(NOISY)[0]}..${vendor.arrivals(NOISY).at(-1)}`, + '0..1900', + ); + note( + '(c) → C4 measured the quiet tenant at t=2000 under the naive construction', + 'per-tenant limiter keys move it to t=0 while leaving the noisy tenant paced exactly as declared', + ); + } + + // ── (d) the token, and the cache, on the principal ───────────────────────────────────────── + { + const { clock: tokenClock, store, vendor, idp, call } = tenantAware(); + for (const t of ['t1', 't2', 't1']) + await drive(tokenClock, () => call(t)({})); + check('(d) token fetches for 2 tenants over 3 calls', idp.mints, 2); + check( + '(d) distinct token vault keys', + store.keys('vault:oauth2:').length, + 2, + ); + checkSeq( + '(d) the tokens that went out', + vendor.calls.map((c) => c.authorization), + [ + 'Bearer tok-saas-app-1', + 'Bearer tok-saas-app-2', + 'Bearer tok-saas-app-1', + ], + ); + + // The cache is the fourth resource, and its default is the safe one. + const clock = manualClock(); + const cacheStore = probeStore(); + const v = new FakeVendor({ clock }); + const cs = seam({ + baseUrl: 'https://api.vendor.test', + adapter: v.adapter(), + store: cacheStore, + clock, + }); + const cached = (t: string, tenancy: 'principal' | 'app') => + cs.as(t).stitch({ + name: 'me', + path: '/v1/me', + headers: { 'x-tenant': t }, + cache: { ttl: '60s', tenancy, version: 'v1' }, + }); + const p1 = await cached('t1', 'principal')({}).safe(); + const p2 = await cached('t2', 'principal')({}).safe(); + checkSeq( + "(d) cache `tenancy: 'principal'` (the DEFAULT) — whose data each tenant got", + [ + (p1.data as { tenant?: string })?.tenant, + (p2.data as { tenant?: string })?.tenant, + ], + ['t1', 't2'], + ); + const a1 = await cached('t1', 'app')({}).safe(); + const a2 = await cached('t2', 'app')({}).safe(); + checkSeq( + "(d) cache `tenancy: 'app'` — whose data each tenant got", + [ + (a1.data as { tenant?: string })?.tenant, + (a2.data as { tenant?: string })?.tenant, + ], + ['t1', 't1'], + ); + note( + "(d) → `tenancy: 'app'` served t2 t1's response body", + "the default is `'principal'` and fail-closed (types.ts:1136-1147), so this one only bites someone who opts out", + ); + } + + // ── (e) THE CAVEAT: a global vendor quota re-couples the tenants ─────────────────────────── + // A real integration also has a quota with the VENDOR, not just fairness between customers. A + // seam-level throttle expresses that — and a member's own throttle only ever TIGHTENS on top of + // it (seam.ts:94-106), so declaring the global cap puts the noisy neighbour straight back. + { + const { clock, vendor, call } = tenantAware({ seamThrottle: true }); + const inFlight = [ + ...Array.from({ length: 20 }, () => call(NOISY)({}).safe()), + call(QUIET)({}).safe(), + ]; + await clock.advance(120_000); + await Promise.all(inFlight); + checkSeq( + '(e) quiet arrival WITH a global seam quota', + vendor.arrivals(QUIET), + [2000], + ); + note( + '(e) → the two policies cannot both be declared', + 'a member throttle stacks tighten-only on the seam bucket (seam.ts:94-106); expressing "1000/m to the vendor AND 10/s per customer" needs the outer gate to be user code (`throttle.delegate`) or a per-tenant seam plus your own global limiter', + ); + } + + // ── (f) the summary, printed as the table a docs page should carry ───────────────────────── + { + const rows: [string, string, string][] = [ + [ + 'token', + 'isolated', + "`oauth2({ tenancy: 'principal' })` + `seam.as(id)` — the PRINCIPAL, fail-closed", + ], + [ + 'cache', + 'isolated', + "`CacheOptions.tenancy` defaults to 'principal' — the PRINCIPAL, fail-closed", + ], + [ + 'rate budget', + 'isolated ONLY by a key', + 'a per-tenant member `name` (+ member `throttle`) or a per-tenant seam — NOT the principal', + ], + [ + 'circuit breaker', + 'isolated ONLY by a key', + 'a per-tenant `circuit.key` — NOT the principal, and not per-tenant objects', + ], + ]; + for (const [resource, verdict, how] of rows) + note(`(f) ${resource.padEnd(15)} ${verdict}`, how); + check( + '(f) resources isolated by the bound principal', + rows.filter(([, v]) => v === 'isolated').length, + 2, + ); + check( + '(f) resources isolated only by a hand-written key', + rows.filter(([, v]) => v.includes('ONLY')).length, + 2, + ); + } + + finish( + 'C8', + 'TWO OF THE FOUR ARE ISOLATED BY THE PRINCIPAL; TWO ARE ISOLATED ONLY BY A STRING YOU HAVE TO REMEMBER TO WRITE. Token: `oauth2({ tenancy: "principal" })` + `seam.as(id)` — 2 tenants over 3 calls measured 2 token fetches and 2 vault keys, fail-closed. Cache: `tenancy` defaults to "principal" — t1 and t2 got their own bodies, while opting into "app" served t2 t1\'s response. Rate budget: isolated only by a per-tenant limiter KEY (a member `name` plus a member `throttle`, or a per-tenant seam). Breaker: isolated only by a per-tenant `circuit.key`. The split is exactly the auth/resilience line — `AuthContext.principal` reaches the auth strategies and the cache-key builder and nothing else — so the two resources whose isolation is a SECURITY property fail closed, and the two whose isolation is an AVAILABILITY property fail open, silently. ASSEMBLED, IT WORKS: the same revoked credential that took down 9 of 9 healthy tenants in C1 took down 0 of 9 here, with the broken tenant still receiving a real 401 and all 9 healthy calls reaching the vendor; a genuine 500 still opened that tenant\'s OWN breaker (500,500,500,503) with 0 of 3 others affected; and the 20-call burst that pushed a quiet tenant to t=2000 in C4 measured t=0. ONE CAVEAT THE BUILT-INS CANNOT CLOSE: adding a seam-level `throttle` to express the GLOBAL vendor quota puts the noisy neighbour straight back (quiet at t=2000), because a member throttle stacks tighten-only on the seam bucket (seam.ts:94-106) — "1000/m to the vendor AND 10/s per customer" is not expressible in one construction', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/fake-vendor.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/fake-vendor.ts new file mode 100644 index 00000000..f7129ef9 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/fake-vendor.ts @@ -0,0 +1,156 @@ +// The vendor API every tenant calls, and the IdP that mints their tokens. Both are plain +// `Adapter`s over an injected {@link Clock} — no network, no timers of their own. +// +// The scenario's whole measurement is PER-TENANT, so the fake is built around that: a request is +// attributed to a tenant by an `x-tenant` header the caller sets, one named tenant can be made to +// fail persistently (`fail('bad', 401)`), and every request is recorded with its tenant, status +// and ARRIVAL TIME on the injected clock. "How many of the N healthy tenants also failed" and +// "when did the quiet tenant's call actually leave" are then both reads off `calls`. +// +// The tenant header is how a real multi-tenant fan-out identifies the customer to itself in a +// test double; in production the discriminator is the credential, which is what C6 measures. +import type { + Adapter, + AdapterRequest, + Clock, +} from '../../../../packages/core/src/types'; + +/** One recorded request: who made it, what came back, and WHEN on the injected clock. */ +export interface VendorCall { + tenant: string; + status: number; + /** Virtual ms at which the request reached the vendor — the arrival time C4/C5 measure. */ + at: number; + path: string; + authorization: string; +} + +export interface FakeVendorOptions { + clock: Clock; + /** Tenants that fail persistently, and with what status. */ + failing?: Record; + /** Body returned with a failing status. Default `{ error: 'invalid_token' }`. */ + failBody?: unknown; + /** + * Tenants whose requests take this many virtual ms to answer — a batch job's slow page. Used + * by the `concurrency` half of the noisy-neighbour measurement, where the question is whether + * one tenant's in-flight call holds a slot another tenant needs. + */ + slow?: Record; +} + +/** + * The vendor. `adapter()` is what a stitch/seam is handed; `calls` is the per-tenant ledger every + * claim reads its numbers off. + */ +export class FakeVendor { + readonly calls: VendorCall[] = []; + private readonly clock: Clock; + private readonly failing: Map; + private readonly failBody: unknown; + private readonly slow: Map; + + constructor(opts: FakeVendorOptions) { + this.clock = opts.clock; + this.failing = new Map(Object.entries(opts.failing ?? {})); + this.failBody = opts.failBody ?? { error: 'invalid_token' }; + this.slow = new Map(Object.entries(opts.slow ?? {})); + } + + /** Make one named tenant fail persistently from now on — a revoked token, a lost permission. */ + fail(tenant: string, status = 401): void { + this.failing.set(tenant, status); + } + + /** Heal a tenant (its credential was re-connected). */ + heal(tenant: string): void { + this.failing.delete(tenant); + } + + /** Requests attributed to one tenant. */ + forTenant(tenant: string): VendorCall[] { + return this.calls.filter((c) => c.tenant === tenant); + } + + /** Arrival times (virtual ms) of one tenant's requests — the noisy-neighbour measurement. */ + arrivals(tenant: string): number[] { + return this.forTenant(tenant).map((c) => c.at); + } + + adapter(): Adapter { + return async (req: AdapterRequest) => { + const tenant = req.headers['x-tenant'] ?? ''; + const status = this.failing.get(tenant) ?? 200; + // Recorded on ARRIVAL, before any hold: `at` is when the request left the process, + // which is the number the rate/concurrency claims measure. + this.calls.push({ + tenant, + status, + at: this.clock.now(), + path: new URL(req.url).pathname, + authorization: req.headers['authorization'] ?? '', + }); + const held = this.slow.get(tenant); + if (held !== undefined) await this.clock.sleep(held); + if (status >= 400) + return { status, headers: {}, body: this.failBody }; + return { + status: 200, + headers: {}, + body: { ok: true, tenant, at: this.clock.now() }, + }; + }; + } +} + +/** + * The token endpoint. Mints a distinct, traceable token per request so C6 can tell whose token a + * call carried — `tok--`; `mints` counts how many token requests were made, which is + * what "one tenant's refresh storm" is measured in. + */ +export class FakeIdp { + mints = 0; + /** client_ids seen on token requests — the evidence for whether a per-tenant CREDENTIAL got through. */ + readonly clientIds: string[] = []; + + adapter(): Adapter { + return async (req: AdapterRequest) => { + this.mints += 1; + const body = (req.body ?? {}) as Record; + const clientId = body['client_id'] ?? ''; + this.clientIds.push(clientId); + return { + status: 200, + headers: {}, + body: { + access_token: `tok-${clientId}-${this.mints}`, + expires_in: 3600, + }, + }; + }; + } +} + +/** + * Run one call and reduce it to a short outcome token — `'ok'`, or `''` for a failure. + * + * `PromiseLike`, not `Promise`: a stitch call returns a lazy `StitchResult` thenable that starts on + * `.then` (stitch.ts:729,781), and it is deliberately not a full `Promise`. + * Every blast-radius number in this scenario is a count over these tokens, so they are deliberately + * tiny and printable as a sequence. + */ +export async function outcomeOf( + call: () => PromiseLike, +): Promise { + try { + await call(); + return 'ok'; + } catch (e) { + const err = e as Error & { status?: number }; + return String(err.status ?? err.name); + } +} + +/** How many of a measured outcome spine were not `'ok'` — the blast radius, as one number. */ +export const blastRadius = (outcomes: readonly string[]): number => + outcomes.filter((o) => o !== 'ok').length; diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/harness.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/harness.ts new file mode 100644 index 00000000..21299b79 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/harness.ts @@ -0,0 +1,68 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is COUNTS and KEY STRINGS: how many of N healthy tenants failed +// because one tenant's credential was revoked, and what string the engine actually keyed the +// breaker / rate bucket on. So both assertions print the measured value whether they pass or +// fail — `9/9 healthy tenants failed` and `circuit:/v1/items` ARE the findings, and they have to +// be readable out of context. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-tenant outcome spine + * (`["ok","ok","503","ok"]`) and the arrival-time spine (`[0,100,200]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a FAILURE of the library — the verdict statement carries + * the direction, because "PASS C1" on a claim whose content is "one tenant takes down all of them" + * is otherwise unreadable. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/probe-store.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/probe-store.ts new file mode 100644 index 00000000..be46b595 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/probe-store.ts @@ -0,0 +1,54 @@ +// A `StitchStore` that records every key the ENGINE touches, over a real `memoryStore`. +// +// This is the scenario's most load-bearing instrument. Whether a resource is shared or isolated +// per tenant is not a matter of which objects were constructed — C2 and C5 both measure +// constructions that look isolated and are not — it is a matter of WHAT STRING the engine keyed +// the state on. The breaker lives at `circuit:` (resilience.ts:353) and the rate counter at +// `rl::` (store.ts:204), so reading the key set off the store answers "is this one +// budget or N" directly, rather than by inference from behaviour. +import { memoryStore } from '../../../../packages/core/src/index'; +import type { StitchStore } from '../../../../packages/core/src/types'; + +export interface ProbeStore extends StitchStore { + /** Every key touched, in order, with duplicates — `keys('circuit:')` is the usual read. */ + readonly touched: string[]; + /** The distinct keys touched under a prefix, in first-touch order. */ + keys(prefix?: string): string[]; + /** Keys currently HOLDING a value (a `set(key, undefined)` deletes) — the residency measurement. */ + live(prefix?: string): string[]; +} + +export function probeStore(inner: StitchStore = memoryStore()): ProbeStore { + const touched: string[] = []; + const resident = new Set(); + const record = (key: string): void => { + touched.push(key); + }; + return { + touched, + keys(prefix = '') { + return [...new Set(touched.filter((k) => k.startsWith(prefix)))]; + }, + live(prefix = '') { + return [...resident].filter((k) => k.startsWith(prefix)); + }, + get(key) { + record(key); + return inner.get(key); + }, + async set(key, value, ttl) { + record(key); + // `set(key, undefined)` is the store's delete (store.ts:39-42) — the throttle uses it + // to drop a rolled-over window, so residency has to honour it. + if (value === undefined) resident.delete(key); + else resident.add(key); + return inner.set(key, value, ttl); + }, + async increment(key, ttl) { + record(key); + resident.add(key); + return inner.increment(key, ttl); + }, + close: () => inner.close?.() ?? Promise.resolve(), + }; +} diff --git a/docs/scenarios/proofs/multi-tenant-blast-radius/type-probe.ts b/docs/scenarios/proofs/multi-tenant-blast-radius/type-probe.ts new file mode 100644 index 00000000..bf898f75 --- /dev/null +++ b/docs/scenarios/proofs/multi-tenant-blast-radius/type-probe.ts @@ -0,0 +1,119 @@ +// Ask the COMPILER which per-tenant spellings exist, instead of grepping for them. +// +// "The built-in can't" and "I couldn't find the spelling" are different findings, and only one of +// them is the library's problem. `NoUnknownNestedKeys` (types.ts:411-448) makes an unknown key +// inside a house envelope a compile error naming the slot, so the honest way to establish that +// there is no `throttle.key` / `pool: 'principal'` / `circuit.tenancy` is to hand the compiler +// each candidate spelling and read back its diagnostics. A line that compiles is a spelling that +// EXISTS; a line that doesn't is one the vocabulary refuses. +// +// The fixture is written to a temp dir (not into the repo) and deleted afterwards, so this leaves +// nothing behind and never lands in `prettier --check`. It imports core by ABSOLUTE path, which is +// why it can live outside the tree. +// +// `typescript` is loaded through a `require` ANCHORED AT `packages/core`, which is the workspace +// package that declares it. A bare `import ts from 'typescript'` resolves under `tsx` and NOT under +// plain Node from this directory (pnpm gives `docs/` no `node_modules`), so the bare form would be +// a script that runs one way and typechecks another. The compiler surface used is tiny, so it is +// declared structurally here rather than imported as a type. +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +/** Absolute path to core's barrel, so the fixture can be compiled from anywhere. */ +export const CORE = join(HERE, '../../../../packages/core/src/index'); + +/** The slice of the TypeScript compiler API this probe uses. */ +interface TsCompiler { + readonly ScriptTarget: Record; + readonly ModuleKind: Record; + readonly ModuleResolutionKind: Record; + createProgram( + rootNames: readonly string[], + options: Record, + ): unknown; + getPreEmitDiagnostics(program: unknown): readonly { + code: number; + start?: number | undefined; + file?: + | { + fileName: string; + getLineAndCharacterOfPosition(pos: number): { line: number }; + } + | undefined; + }[]; +} + +const ts = createRequire(join(HERE, '../../../../packages/core/package.json'))( + 'typescript', +) as TsCompiler; + +export interface Candidate { + /** What a reader would call this spelling — printed in the report. */ + label: string; + /** One statement. Compiles ⇒ the spelling exists. */ + code: string; +} + +export interface ProbeResult extends Candidate { + compiles: boolean; + /** First diagnostic code, e.g. 2769 (no overload matches) or 2322 (not assignable). */ + diagnostic?: number; +} + +/** + * Typecheck each candidate as its own statement in one program and report which compile. + * Diagnostics are attributed by LINE, so each candidate must be a single line. + */ +export function probeSpellings( + candidates: readonly Candidate[], +): ProbeResult[] { + const dir = mkdtempSync(join(tmpdir(), 'stitch-tenancy-probe-')); + const file = join(dir, 'probe.ts'); + const header = [ + `import { seam, stitch } from ${JSON.stringify(CORE)};`, + `const tenantId: string = 'tenant-42';`, + `void [seam, stitch, tenantId];`, + ]; + try { + writeFileSync( + file, + [...header, ...candidates.map((c) => c.code)].join('\n'), + ); + const program = ts.createProgram([file], { + target: ts.ScriptTarget['ES2022'], + module: ts.ModuleKind['ESNext'], + moduleResolution: ts.ModuleResolutionKind['Bundler'], + strict: true, + noEmit: true, + skipLibCheck: true, + exactOptionalPropertyTypes: true, + noUncheckedIndexedAccess: true, + }); + const byLine = new Map(); + for (const d of ts.getPreEmitDiagnostics(program)) { + if (d.file?.fileName !== file || d.start === undefined) continue; + const { line } = d.file.getLineAndCharacterOfPosition(d.start); + if (!byLine.has(line)) byLine.set(line, d.code); + } + return candidates.map((c, i) => { + const diagnostic = byLine.get(header.length + i); + return diagnostic === undefined + ? { ...c, compiles: true } + : { ...c, compiles: false, diagnostic }; + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The spellings that compiled — the vocabulary that actually exists. */ +export const accepted = (results: readonly ProbeResult[]): string[] => + results.filter((r) => r.compiles).map((r) => r.label); + +/** The spellings the compiler refused. */ +export const rejected = (results: readonly ProbeResult[]): string[] => + results.filter((r) => !r.compiles).map((r) => r.label); diff --git a/docs/scenarios/proofs/multipart-upload/README.md b/docs/scenarios/proofs/multipart-upload/README.md new file mode 100644 index 00000000..7e72feb6 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/README.md @@ -0,0 +1,181 @@ +# Proofs — the upload you must clean up after: S3-style multipart + +Runnable evidence for the claims in +[`../../multipart-upload.md`](../../multipart-upload.md). + +Every script is standalone, offline and deterministic: it injects a fake S3-shaped multipart provider +through StitchAPI's `adapter` / `Surface.execute` seam, drives the timeout cases off an injected +`manualClock()`, and controls part-completion ORDER with microtask turns rather than timers — so a +bounded eight-part fan is exact and never flaky. No network, no `node:test`, no sleeping. + +**The measurement that decides this scenario is `orphanParts`.** A part stored under an `UploadId` +that was never completed and never aborted is exactly what AWS bills for and hides from `aws s3 ls`. +The fake counts it, alongside `orphanBytes`, `danglingUploads` and `aborted` (how many `DELETE`s +actually arrived). Three other numbers carry claims of their own: `peakInFlight` (incremented by the +server on entry, decremented on exit — nothing about concurrency is inferred from config), +`completionOrder` (the order the server actually stored parts in) and `partPutOrder` (every part +number that arrived, so "only part 3 was re-sent" is `[1,2,3,4,3]` rather than an argument). + +The provider is strict on purpose: `complete` rejects an out-of-order list with `400 +InvalidPartOrder` and an incomplete or ETag-mismatched one with `400 InvalidPart`. Ordering is +therefore checked by the server, not assumed by the proof. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/multipart-upload/c4-no-compensation.ts + +# all of them +for f in docs/scenarios/proofs/multipart-upload/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/multipart-upload/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ---------------------------- | ---------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| `c1-upload-progress.ts` | does `xhrAdapter` really report bytes SENT? | **Yes — 4 upload ticks vs fetch's 0**, and the gap is readable before any call. The diagnostic is gated | +| `c2-etag-header.ts` | can the `ETag` header be captured, in order? | **Yes, via `interpret`.** No result accessor carries headers; `Promise.all` already does the ordering | +| `c3-bounded-concurrency.ts` | what actually bounds the part fan? | **`throttle.concurrency` on ONE stitch → peak 3.** `all()` bounds nothing; `all()` + per-member limit → 8 | +| `c4-no-compensation.ts` | is there ANY seam that guarantees the abort? | **No — and `onError` fired 0 times on an HTTP 500.** Orphan: 3 parts / 15 MiB / 0 DELETEs | +| `c5-retry-granularity.ts` | per-part retry? is whole-upload retry stopped? | **Per part: `[1,2,3,4,3]`.** Whole-upload retry is not prevented: 3 UploadIds, 9 orphans, 45 MiB | +| `c6-cancelled-siblings.ts` | what happens to parts that already landed? | **2 stored, 0 nameable.** `all()` discards the successes; a `hooks.onResponse` side channel recovers them | +| `c7-progress-aggregation.ts` | can per-part bytes become one number? | **Yes — per-part HIGH-WATER.** The naive `Σ loaded` measured 400 against a 160-byte file | +| `c8-assembled-solution.ts` | best answer, run on both paths, worth it? | **141 vs 163 lines, 0 orphans on every exit** — and the `try/finally` is identical on both sides | + +## Files + +- `fake-s3.ts` — the provider. `POST ?uploads` → `{ UploadId }`; `PUT ?partNumber=N&uploadId=…` → + **200 with an `ETag` RESPONSE HEADER and no body**; `POST ?uploadId=…` with an ordered + `[{PartNumber, ETag}]` list → the assembled object, or `400 InvalidPartOrder` / `400 InvalidPart`; + `DELETE ?uploadId=…` → abort. Knobs: `partTicks` (delay a part's response by N microtask turns, so + completion order can be made to differ from part order), `hangParts` (a stalled socket that answers + only when its signal aborts — which is what makes a `timeout` a reachable failure mode) and + `failPart(n, times, status)`. Exposes both an `Adapter` and a `fetch`-shaped entry point so C8's + two implementations share one transport contract. +- `fake-xhr.ts` — a structural `XhrLike`, injected into `xhrAdapter(FakeXhr)`. Not a workaround: + `xhrAdapter` takes a constructor for exactly this (xhr-adapter.ts:49-68). It fires + `upload.onprogress` on a deterministic schedule, so C1 and C7 assert an exact tick sequence. +- `multipart.ts` — **user code**, the assembled answer and the subject of C8's line count. Four + stitches, one `try/finally`, a mandatory-by-default `onCleanupFailure`. +- `hand-rolled.ts` — the same behaviour with no StitchAPI in it, feature-matched down to the FIFO + pool and the retryable-status set, so the line comparison is honest. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C4 is the finding, and it is a refusal.** There is no compensation seam: `StitchConfig` has 26 + keys and none of them is one, `Hooks` is exactly `{onRequest,onResponse,onError,onRetry}` + (types.ts:1285-1290), and `linked()` is `Promise.resolve(body(run))` (pipe.ts:357-369) with no + finally. The capture predicted that. What it did not predict is that **`onError` is not a failure + hook at all** — it lives in the `catch` around `withTimeout(transport)` (engine.ts:668-680), so a + failed call carrying HTTP 500 measured the hook sequence `["onRequest","onResponse"]` and **zero** + `onError` calls, while a stalled socket fired it **three** times (once per attempt). Cleanup wired + to `onError` runs on the network blips and skips the application failures. That is worse than an + absent hook, because it looks wired. +- **The orphan, measured three ways.** A failing part with no user cleanup: **3 parts / 15 MiB / 1 + dangling UploadId / 0 DELETEs**. A caller `AbortSignal` mid-flight: **2 orphaned parts, 0 DELETEs** + — the user pressed Cancel and the bucket kept the bytes. A `timeout.total` expiry: **3 orphaned + parts, 0 DELETEs**. Cancellation is built in; cleanup is not, and the two are different concerns. +- **The failure mode that counts double is a cleanup that reports success.** `abort.safe()` cannot + throw. Pointed at a wrong UploadId inside a correct-looking `try/finally`, it measured **0 accepted + DELETEs, 3 orphaned parts, and nothing thrown anywhere**. Every `.safe()` on a compensating call + needs its `ok` inspected; `multipart.ts` makes `onCleanupFailure` mandatory-by-default (omitted, it + throws) for exactly this reason. +- **`all()` is the wrong tool for a part fan, twice over, and the capture nominates it.** It bounds + nothing — `runAllArray` is `members.map(...)` straight into `Promise.all` (pipe.ts:122-136), which + measured a peak of **8** on eight parts. And it hands EVERY member the same `StitchInput` + (pipe.ts:75-76): one stitch × eight members produced eight PUTs all carrying `partNumber=1` and + stored **one** part. So a fan needs eight distinct stitches — and eight stitches each configured + `concurrency: 3` measured a peak of **8**, because the default `pool: 'stitch'` gives every stitch + its own state map (resilience.ts:103-109). The config reads `concurrency: 3` eight times and bounds + nothing. +- **Three spellings do bound it; only one is obvious after the fact.** ONE stitch called eight times + with `throttle: { concurrency: 3 }` → **3**. `{ concurrency: 3, pool: 'host' }` across eight + stitches → **3** (the host-pooled state map is module-level). A `seam({ throttle: { concurrency: 3 +} })` bucket → **3** (seam.ts:46-68). +- **The ETag is reachable, and the ordering was never the hard part.** `Surface.interpret` returns + the header as the stitch's data; `hooks.onResponse` and `Surface.execute` see it too but can only + write it to a closure. What is NOT reachable is anything on the awaited path: a bare stitch on a + part PUT measured `ok:true`/`data:undefined`, and `.inspect()` measured `status:200`/`raw:null` + with **zero** header-bearing fields (`Inspection`, types.ts:1736-1766). Meanwhile `Promise.all` + resolves in INPUT order regardless of settle order, so with the server storing parts `[3,1,4,2]` + the list was already `[1,2,3,4]` — the sort everyone writes by hand is redundant. The bug is + collecting inside the `await` callback: pushing on settle gave `[3,1,4,2]` and `400 +InvalidPartOrder` with 4 parts orphaned. +- **`interpret` REPLACES the default verdict, and forgetting `verdictOf` is silent.** A part surface + written the natural way (`interpret: (res) => ({ ok: true, data: res.headers['etag'] })`) turned an + HTTP 500 into `ok:true`/`data:undefined` — the part "succeeded" carrying no ETag, and the failure + surfaced two calls later as `InvalidPart`. `verdictOf(res, cfg) ?? …` (surface.ts:174-191) is not + optional boilerplate. +- **`retry.on` defaults to `[429,502,503,504]`, and S3's own transient error is `500 InternalError`.** + The same `retry: { attempts: 3 }` that produced a clean `[1,2,3,4,3]` against a 503 measured **4 + PUTs, 1 failed part and 3 orphans** against a 500 (engine.ts:612). Widening to + `on: [429,500,502,503,504]` restored it. A THROWN transport error is retried regardless of + `retry.on` — measured ok with `on: [418]` — which is a different code path (engine.ts:681). +- **Whole-upload retry is not prevented and multiplies the orphan.** The whole flow as one + `Surface.execute` stitch with `retry: { attempts: 3 }` measured **3 `POST ?uploads`, 12 part PUTs, + 3 dangling UploadIds, 9 orphaned parts, 45 MiB**. Nothing warns. Moving the abort INSIDE `execute` + measured 3 initiates / 3 DELETEs / **0 orphans** — still the wrong granularity, no longer a billing + incident. +- **`all()`'s auto-cancel works, and cleanup is a separate question.** With parts 1-2 landed, part 3 + failing and part 4 in flight, the server saw statuses **[200,200,500,499]** — part 4 genuinely cut + off. Two parts were stored and the client could name **zero** of their ETags, because `all()` is + `Promise.all` and discards resolved values on rejection. A `hooks.onResponse` side channel + recovered exactly `[1,2]`. There is no `allSettled` (the subpath exports exactly + `["all","any","linked","race"]`, and pipe.ts:20 says the omission is deliberate); composing + `.safe()` members keeps every value but disables the fail-fast — measured `[200,200,500,200]`, + i.e. part 4 uploaded in full into an upload that was already doomed. +- **The upload progress story is real, and narrower than it looks.** `xhrAdapter` with an injected + constructor reported 4 `direction:'upload'` ticks (`loaded [27,54,81,108]` of `total 108`) before + the response existed; the identical call through the real `fetchAdapter` reported **0**. The + capture says progress "is not available over `fetch` at all" — half right: fetch reports DOWNLOAD + progress fine, so `onProgress` on a POST is half-served rather than ignored. The gap is readable + with no call made (`supports` measured `["uploadProgress","downloadProgress"]` vs + `["stream","downloadProgress"]` — strict complements, so choosing a progress bar costs you + streaming), and asking fetch for upload progress emits one `info` event, + `adapter.upload-progress-unsupported`, naming `xhrAdapter()` (engine.ts:1107-1123). **The gate:** + it fires only when the adapter DECLARED capabilities. A custom/BYO transport declares nothing and + measured 0 info events and 0 ticks — silence, which is the case most real code is in. And the note + is an EVENT: `.safe()` and `await` never see it. +- **Progress aggregation has one correct shape and it is not the obvious one.** Every upload tick + carries exactly `direction,loaded,total` (types.ts:858-867) — no part number, no request, no run + id — so a SHARED `onProgress` across a concurrent fan is unattributable. Summing raw `loaded` + measured **400** against a 160-byte file, because each tick is cumulative WITHIN its part. Binding + the part number at the call site and keeping a per-part high-water mark produced 16 monotonic ticks + ending at exactly 100%. A retry replays the part's ticks from zero (measured `[20,40,20,40]`), + which makes the naive sum 300 while the high-water aggregate stays 160. There is no byte EVENT: + `ProgressPhase` (types.ts:1293-1305) has no such phase and `onProgress` is absent from `__config`. +- **C8's line count is honest in both directions.** 141 executable lines against a 163-line + feature-matched hand-rolled twin — 22 lines shorter, and the difference attributes exactly to the + retry loop with backoff, the FIFO concurrency pool, the retryable-status set and the URL assembly, + all of which became config. What did NOT shrink is the part this scenario is about: the + `try/finally`, the loud-cleanup rule, the per-part high-water progress map and the input-order + assembly are the same on both sides, line for line. +- **Do not make the upload one `Surface.execute` stitch to "let the engine own the lifecycle".** + `withTimeout` (resilience.ts:230-244) rejects the caller's promise the instant the timer fires and + lets `fn` keep running, so a `try/finally` inside `execute` cleans up AFTER the caller has already + returned. Measured at the instant the caller's promise settled: **3 orphans, 0 DELETEs**; several + turns later, 0 and 1. In a lambda, or any process that exits on the error, the later half never + happens. +- **Two small corrections worth carrying.** `backoff` has no `delay` field — it is + `{ curve, base, max }` (types.ts:967-974); `delay` is a compile error and a runtime no-op. And the + `stitchapi/pipe` subpath exports `all` / `any` / `race` / `linked` and **no `pipe()` combinator at + all**, despite the subpath's name and a reference to `pipe()` in + `apps/docs/content/docs/concepts/run-identity.mdx:26`. diff --git a/docs/scenarios/proofs/multipart-upload/c1-upload-progress.ts b/docs/scenarios/proofs/multipart-upload/c1-upload-progress.ts new file mode 100644 index 00000000..2be04cc9 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c1-upload-progress.ts @@ -0,0 +1,277 @@ +// C1 — does `xhrAdapter` actually report UPLOAD progress, does `fetchAdapter` report none, and is +// the difference visible BEFORE the call? +// +// The capture says "`xhrAdapter` exists precisely for this" and wonders whether there is "real +// capability negotiation, and possibly a diagnostic when you ask `fetch` for progress". Both halves +// are testable. `xhrAdapter` is browser-only by default but takes an injected constructor +// (xhr-adapter.ts:49-68), so a structural fake runs it on Node with no polyfill. +// +// The part worth reading twice is (e): the diagnostic exists, but it is gated on the adapter having +// DECLARED capabilities (engine.ts:1113-1114). A custom transport — which is what every fake, every +// test double, and every BYO client in this repo's own proofs is — declares nothing, and gets +// silence. The teaching note is a property of the built-ins, not of the engine. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c1-upload-progress.ts +import { + fetchAdapter, + stitch, + xhrAdapter, +} from '../../../../packages/core/src/index'; +import type { + Adapter, + AdapterProgress, + StitchEvent, +} from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { fakeXhrCtor } from './fake-xhr'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** Collect the engine's event stream for one call. */ +async function eventsOf( + run: (sink: (e: StitchEvent) => void) => Promise, +): Promise { + const evts: StitchEvent[] = []; + await run((e) => evts.push(e)); + return evts; +} + +/** Open a real upload on the server so the part PUTs below run on the success path. */ +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +async function main(): Promise { + heading( + 'C1 — upload progress, and whether the transport gap is visible in advance', + ); + + // ── (a) the capability descriptor, read BEFORE any call is made ─────────────────────────── + // This is the "detectable in advance" half. Both built-ins hang an `AdapterCapabilities` + // off the function itself (types.ts:936-941), so a host can branch on it at wiring time. + { + const xhr = xhrAdapter(fakeXhrCtor(new FakeS3().adapter())); + const fetchA = fetchAdapter({ fetch: new FakeS3().fetchImpl() }); + checkSeq( + '(a) xhrAdapter().capabilities.supports', + xhr.capabilities?.supports ?? [], + ['uploadProgress', 'downloadProgress'], + ); + checkSeq( + '(a) fetchAdapter().capabilities.supports', + fetchA.capabilities?.supports ?? [], + ['stream', 'downloadProgress'], + ); + check( + '(a) can a host ask "does this transport do upload progress?" with no call', + String(xhr.capabilities?.supports.includes('uploadProgress')) + + '/' + + String( + fetchA.capabilities?.supports.includes('uploadProgress'), + ), + 'true/false', + ); + // And the inverse gap, which matters for the OTHER half of this scenario: xhr cannot stream. + check( + '(a) xhr supports stream', + xhr.capabilities?.supports.includes('stream'), + false, + ); + note( + '(a) → the two built-ins are strict complements', + 'xhr: upload+download progress, no stream. fetch: stream+download, no upload progress', + ); + } + + // ── (b) xhrAdapter through a real stitch: does `direction: 'upload'` actually arrive? ────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const ticks: AdapterProgress[] = []; + const put = stitch({ + url: FakeS3.url('video.mp4'), + method: 'PUT', + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 4 })), + }); + const r = await put.safe({ + query: { partNumber: 1, uploadId }, + body: { chunk: 'x'.repeat(96) }, + onProgress: (p) => ticks.push(p), + }); + + check('(b) xhr → the part PUT succeeded', r.ok, true); + const uploads = ticks.filter((t) => t.direction === 'upload'); + const downloads = ticks.filter((t) => t.direction === 'download'); + check('(b) xhr → upload ticks', uploads.length, 4); + checkSeq( + '(b) xhr → upload `loaded` sequence', + uploads.map((t) => t.loaded), + [27, 54, 81, 108], + ); + checkSeq( + '(b) xhr → upload `total` (same on every tick)', + [...new Set(uploads.map((t) => t.total))], + [108], + ); + check('(b) xhr → download ticks', downloads.length, 1); + note( + '(b) → the upload phase is genuinely instrumented', + 'ticks arrive BEFORE the response exists — the thing a progress bar needs', + ); + } + + // ── (c) the same call over the REAL fetchAdapter: how many upload ticks? ────────────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const ticks: AdapterProgress[] = []; + const put = stitch({ + url: FakeS3.url('video.mp4'), + method: 'PUT', + adapter: fetchAdapter({ fetch: api.fetchImpl() }), + }); + await put.safe({ + query: { partNumber: 1, uploadId }, + body: { chunk: 'x'.repeat(96) }, + onProgress: (p) => ticks.push(p), + }); + + check( + '(c) fetch → upload ticks', + ticks.filter((t) => t.direction === 'upload').length, + 0, + ); + check( + '(c) fetch → download ticks', + ticks.filter((t) => t.direction === 'download').length > 0, + true, + ); + note( + '(c) → `onProgress` is not ignored on fetch, it is HALF-served', + 'the download phase reports; the upload phase is silent. A bar wired to `loaded` moves only after the bytes are already gone', + ); + } + + // ── (d) …and the engine says so. The `info` event, on the built-in fetch adapter ────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const put = stitch({ + url: FakeS3.url('video.mp4'), + method: 'PUT', + adapter: fetchAdapter({ fetch: api.fetchImpl() }), + }); + const evts = await eventsOf(async (sink) => { + for await (const e of put.stream({ + query: { partNumber: 1, uploadId }, + body: { chunk: 'x' }, + onProgress: () => undefined, + })) + sink(e); + }); + const info = evts.find((e) => e.type === 'info'); + check( + '(d) info event topic', + info && 'topic' in info ? info.topic : '(none)', + 'adapter.upload-progress-unsupported', + ); + check( + '(d) the detail names the fix', + info && 'detail' in info + ? String(info.detail).includes('xhrAdapter()') + : false, + true, + ); + + // It is gated on a BODY (engine.ts:1111): a bodyless GET asking for progress is a + // legitimate download-progress request, so no note fires. + const evtsNoBody = await eventsOf(async (sink) => { + for await (const e of put.stream({ + query: { partNumber: 2, uploadId }, + onProgress: () => undefined, + })) + sink(e); + }); + check( + '(d) same call with NO body → info events', + evtsNoBody.filter((e) => e.type === 'info').length, + 0, + ); + note( + '(d) → asking fetch for upload progress is NOT silence', + 'one `info` event per call, naming xhrAdapter() — but only on the event stream', + ); + } + + // ── (e) the gate: the note only exists if the adapter DECLARED capabilities ─────────────── + // Every custom transport — and every fake in these proofs — declares nothing, so it is treated + // as unknown and the open contract stands (engine.ts:1114). Silence, not a note. + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const bare: Adapter = (req) => api.adapter()(req); + const put = stitch({ + url: FakeS3.url('video.mp4'), + method: 'PUT', + adapter: bare, + }); + const ticks: AdapterProgress[] = []; + const evts = await eventsOf(async (sink) => { + for await (const e of put.stream({ + query: { partNumber: 1, uploadId }, + body: { chunk: 'x' }, + onProgress: (p) => ticks.push(p), + })) + sink(e); + }); + check( + '(e) undeclared custom adapter → info events', + evts.filter((e) => e.type === 'info').length, + 0, + ); + check( + '(e) undeclared custom adapter → progress ticks', + ticks.length, + 0, + ); + note( + '(e) → this is the real-world default and it IS silent', + 'a BYO transport (or any test double) gets zero ticks and zero diagnostics', + ); + } + + // ── (f) the awaited path never sees the note ────────────────────────────────────────────── + // `.safe()` / `await` resolve normally; `info` is an EVENT. A team that never wires a trace + // sink or `.stream()` gets the same silence the note exists to prevent. + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const put = stitch({ + url: FakeS3.url('video.mp4'), + method: 'PUT', + adapter: fetchAdapter({ fetch: api.fetchImpl() }), + }); + const r = await put.safe({ + query: { partNumber: 1, uploadId }, + body: { chunk: 'x' }, + onProgress: () => undefined, + }); + check('(f) .safe() → ok', r.ok, true); + check('(f) .safe() → error', r.error, null); + note( + '(f) → the diagnostic is opt-in by observation', + '`await put(...)` and `.safe()` surface nothing; you must consume `.stream()` or attach a trace sink', + ); + } + + finish( + 'C1', + 'YES on both halves, with one gate. `xhrAdapter` with an injected constructor reported 4 `direction: "upload"` ticks (loaded [27,54,81,108] of total 108) BEFORE the response existed, plus 1 download tick; the identical call through the real `fetchAdapter` reported 0 upload ticks and download ticks only. The difference is readable with NO call made: `xhrAdapter().capabilities.supports` measured ["uploadProgress","downloadProgress"] and `fetchAdapter().capabilities.supports` measured ["stream","downloadProgress"] — strict complements, so choosing progress costs you streaming. Asking fetch for upload progress is not silence: one `info` event, topic `adapter.upload-progress-unsupported`, whose detail names `xhrAdapter()` (engine.ts:1107-1123) — gated on a request body being present, so a bodyless GET gets none. THE GATE: it fires only when the adapter declared `capabilities`. A custom/BYO transport declares nothing and measured 0 info events and 0 progress ticks — silence, which is the case most real code is in', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c2-etag-header.ts b/docs/scenarios/proofs/multipart-upload/c2-etag-header.ts new file mode 100644 index 00000000..964418c7 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c2-etag-header.ts @@ -0,0 +1,312 @@ +// C2 — the part's result is a RESPONSE HEADER. Which seam can capture it, and can N of them be +// assembled in PART order when they arrive in a different order? +// +// The fake makes part 3 land first (`partTicks`), and the server REJECTS a mis-ordered list with +// `400 InvalidPartOrder`, so "in part order, not completion order" is checked by the server rather +// than assumed by the proof. `completionOrder` reports what actually happened on the wire. +// +// Four seams are tried. The finding is that `interpret` is the right one and the ordinary result +// accessors are the wrong ones — `.safe()`, `await`, and `.inspect()` carry NO response headers at +// all (`Inspection` is `{data, raw, findings, status, error, source}` — types.ts:1736-1766), so a +// stitch whose surface does not lift the header has permanently lost it. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c2-etag-header.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + Adapter, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** The part surface: the stitch's DATA is the `ETag` response header. */ +const partSurface: Surface = { + id: 'multipart-part', + // `verdictOf` first (surface.ts:174-191) — an `interpret` hook REPLACES the default verdict, + // so a surface that forgets this turns a 500 into a success. Case (d) measures exactly that. + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +/** The same surface written the way it is easy to write it — no verdict. The footgun in (d). */ +const naivePartSurface: Surface = { + id: 'multipart-part-naive', + interpret: (res) => ({ ok: true, data: res.headers['etag'] }), +}; + +/** Land part 3 first, then 1, then 4, then 2 — completion order ≠ part order, by construction. */ +const OUT_OF_ORDER = { 3: 0, 1: 12, 4: 24, 2: 36 }; + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +async function main(): Promise { + heading( + 'C2 — the ETag is a response header, and the list must be in part order', + ); + + // ── (a) which seams can see the header at all? ──────────────────────────────────────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const seen: string[] = []; + let fromExecute = '(none)'; + let fromHook = '(none)'; + + const spyExecute: Adapter = async (req) => { + const res: AdapterResponse = await api.adapter()(req); + fromExecute = res.headers['etag'] ?? '(none)'; + return res; + }; + + const withInterpret = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + hooks: { + onResponse: (ctx) => { + fromHook = ctx.res?.headers['etag'] ?? '(none)'; + }, + }, + }); + const r = await withInterpret.safe({ + params: { key: 'video.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'a' }, + }); + seen.push(String(r.data)); + + const withExecute = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: { id: 'exec-part', execute: spyExecute }, + adapter: api.adapter(), + }); + await withExecute.safe({ + params: { key: 'video.mp4' }, + query: { partNumber: 2, uploadId }, + body: { chunk: 'b' }, + }); + + check( + '(a) `interpret` → the ETag IS the data', + seen[0], + `"${uploadId}-p1"`, + ); + check( + '(a) `hooks.onResponse` → ctx.res.headers.etag', + fromHook, + `"${uploadId}-p1"`, + ); + check( + '(a) `Surface.execute` → its own response', + fromExecute, + `"${uploadId}-p2"`, + ); + note( + '(a) → all three seams see it; they differ in where the value can GO', + '`interpret` returns it as the value; the other two must write to a closure', + ); + } + + // ── (b) and which seams CANNOT: the ordinary result accessors carry no headers ──────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'video.mp4'); + const plain = stitch({ + url: FakeS3.template, + method: 'PUT', + adapter: api.adapter(), + }); + const r = await plain.safe({ + params: { key: 'video.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'a' }, + }); + const probe = await plain.inspect({ + params: { key: 'video.mp4' }, + query: { partNumber: 2, uploadId }, + body: { chunk: 'b' }, + }); + check('(b) a bare stitch → ok', r.ok, true); + check('(b) a bare stitch → data (the part body)', r.data, undefined); + check('(b) .inspect() → status', probe.status, 200); + check('(b) .inspect() → raw (the part body)', probe.raw, null); + checkSeq( + '(b) header-bearing fields on Inspection', + Object.keys(probe).filter((k) => + k.toLowerCase().includes('header'), + ), + [], + ); + note( + '(b) → a part upload SUCCEEDS and yields nothing usable', + 'no accessor on the awaited path exposes response headers — the surface/hook seam is the only way in', + ); + } + + // ── (c) N parts, landing out of order, assembled in PART order ──────────────────────────── + { + const api = new FakeS3({ partTicks: OUT_OF_ORDER }); + const uploadId = await openUpload(api, 'video.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const complete = stitch({ + url: FakeS3.template, + method: 'POST', + adapter: api.adapter(), + }); + + const nums = [1, 2, 3, 4]; + const etags = await Promise.all( + nums.map(async (n) => ({ + PartNumber: n, + ETag: (await part({ + params: { key: 'video.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `part-${n}` }, + })) as string, + })), + ); + + checkSeq( + '(c) the order the SERVER stored parts in', + api.completionOrder, + [3, 1, 4, 2], + ); + checkSeq( + '(c) the order the ETags are held in (Promise.all preserves INPUT order)', + etags.map((e) => e.PartNumber), + [1, 2, 3, 4], + ); + + const done = await complete.safe({ + params: { key: 'video.mp4' }, + query: { uploadId }, + body: { Parts: etags }, + }); + check('(c) complete → ok', done.ok, true); + check( + '(c) assembled parts', + (done.data as { Parts?: number }).Parts, + 4, + ); + check('(c) orphaned parts after a clean run', api.orphanParts, 0); + note( + '(c) → `Promise.all` already does the ordering', + 'it resolves in INPUT order regardless of settle order — the sort most write by hand is redundant', + ); + } + + // ── (d) the order really is checked: send COMPLETION order and watch it fail ────────────── + { + const api = new FakeS3({ partTicks: OUT_OF_ORDER }); + const uploadId = await openUpload(api, 'video.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const complete = stitch({ + url: FakeS3.template, + method: 'POST', + adapter: api.adapter(), + }); + const collected: { PartNumber: number; ETag: string }[] = []; + await Promise.all( + [1, 2, 3, 4].map(async (n) => { + const etag = (await part({ + params: { key: 'video.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `part-${n}` }, + })) as string; + collected.push({ PartNumber: n, ETag: etag }); // push order = COMPLETION order + }), + ); + checkSeq( + '(d) collected in completion order', + collected.map((c) => c.PartNumber), + [3, 1, 4, 2], + ); + const bad = await complete.safe({ + params: { key: 'video.mp4' }, + query: { uploadId }, + body: { Parts: collected }, + }); + check('(d) complete with that list → ok', bad.ok, false); + check('(d) status', bad.error?.status, 400); + // `.inspect().raw` is `null` on a failure (measured), so the server's error CODE is + // reachable only through `StitchError.body` — worth knowing when the code IS the diagnosis. + check( + '(d) the server’s complaint (StitchError.body.Code)', + (bad.error?.body as { Code?: string } | undefined)?.Code, + 'InvalidPartOrder', + ); + check('(d) …and the parts are still sitting there', api.orphanParts, 4); + note( + '(d) → collecting in an `await` callback is the bug', + 'push-on-settle gives completion order; the fix is to return the value and let `Promise.all` order it', + ); + } + + // ── (e) the interpret footgun: a surface that forgets `verdictOf` ───────────────────────── + // `interpret` REPLACES the default verdict (surface.ts:57-64). A naive one turns a 500 into + // `{ ok: true, data: undefined }` — the part upload "succeeds" carrying no ETag, and the + // failure only surfaces two calls later as `InvalidPart`. + { + const api = new FakeS3(); + api.failPart(2); + const uploadId = await openUpload(api, 'video.mp4'); + const naive = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: naivePartSurface, + adapter: api.adapter(), + }); + const r = await naive.safe({ + params: { key: 'video.mp4' }, + query: { partNumber: 2, uploadId }, + body: { chunk: 'b' }, + }); + check('(e) naive interpret + HTTP 500 → ok', r.ok, true); + check('(e) naive interpret + HTTP 500 → data', r.data, undefined); + + const guarded = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const g = await guarded.safe({ + params: { key: 'video.mp4' }, + query: { partNumber: 2, uploadId }, + body: { chunk: 'b' }, + }); + check('(e) `verdictOf` composed + HTTP 500 → ok', g.ok, false); + check('(e) `verdictOf` composed → status', g.error?.status, 500); + note( + '(e) → `verdictOf(res, cfg) ?? …` is not optional boilerplate', + 'without it a failed part is indistinguishable from a successful one until `complete` rejects the list', + ); + } + + finish( + 'C2', + 'PASS via `Surface.interpret`, and the ordinary accessors are a dead end. Measured: `interpret` returns the `etag` RESPONSE HEADER as the stitch\'s data (`"upl-1-p1"`); `hooks.onResponse` (`ctx.res.headers`) and `Surface.execute` both see it too, but can only write it to a closure. A BARE stitch on the same PUT measured `ok:true`/`data:undefined`, and `.inspect()` measured `status:200`/`raw:null` with ZERO header-bearing fields — `Inspection` (types.ts:1736-1766) carries no headers, so without a surface the ETag is unrecoverable. Ordering is measured against a server that checks it: with part 3 landing first (server-recorded `completionOrder` [3,1,4,2]) `Promise.all` still resolved in INPUT order [1,2,3,4] and `complete` returned the 4-part object with 0 orphans, while the SAME ETags pushed in settle order [3,1,4,2] were rejected `400 InvalidPartOrder` and left 4 parts orphaned. The footgun: `interpret` REPLACES the default verdict, so a surface that omits `verdictOf` turned an HTTP 500 part into `ok:true`/`data:undefined` (measured) — the failure then surfaces only at `complete`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c3-bounded-concurrency.ts b/docs/scenarios/proofs/multipart-upload/c3-bounded-concurrency.ts new file mode 100644 index 00000000..e76f70dc --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c3-bounded-concurrency.ts @@ -0,0 +1,280 @@ +// C3 — can the part fan be bounded, and by what? The number reported is `peakInFlight`, which the +// FAKE SERVER increments on entry and decrements on exit. Nothing here is inferred from the config. +// +// Eight parts, each stalled long enough that an unbounded fan is unambiguously eight-wide. +// +// Two findings sit in here that the capture does not anticipate: +// +// 1. `all()` bounds NOTHING. `runAllArray` is `members.map(...)` into `Promise.all` +// (pipe.ts:122-136) — every member starts in the same turn. It is a fail-fast + auto-cancel + +// child-run combinator, not a pool. +// 2. `all()` hands EVERY member the SAME `StitchInput` (`{...input, signal}`, pipe.ts:75-76). A +// fan over N parts needs N part numbers, so it needs N pre-built stitches — and N stitches +// means N SEPARATE throttle buckets, because the default `pool: 'stitch'` gives each stitch +// its own state map (resilience.ts:103-109). So the two features do not compose: the obvious +// `all([...]) + throttle.concurrency` measured a peak of 8 against a limit of 3. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c3-bounded-concurrency.ts +import { seam, stitch, verdictOf } from '../../../../packages/core/src/index'; +import { all } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const partSurface: Surface = { + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +const PARTS = [1, 2, 3, 4, 5, 6, 7, 8]; +/** Every part stalls 20 microtask turns, so an unbounded fan is unmistakably 8 wide. */ +const SLOW = Object.fromEntries(PARTS.map((n) => [n, 20])); + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +/** + * The throttle shape these cases use. Deliberately NOT `ThrottleOptions`: that type is optional in + * every field, and `StitchConfig.throttle` is `AtLeastOne` (P20 — the opaque `{}` + * is a compile error), so a bare `ThrottleOptions` is not assignable to the config slot. + */ +type Bound = { concurrency: number; pool?: 'stitch' | 'host' }; + +/** One reusable part stitch — the fan is N CALLS to it. */ +function onePartStitch(api: FakeS3, throttle?: Bound): Stitch { + return stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + ...(throttle ? { throttle } : {}), + }) as Stitch; +} + +/** N part stitches, one per part number — what `all()` forces, since every member gets one input. */ +function perPartStitches( + api: FakeS3, + uploadId: string, + throttle?: Bound, +): Stitch[] { + return PARTS.map( + (n) => + stitch({ + url: `${FakeS3.template}?partNumber=${n}&uploadId=${uploadId}`, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + ...(throttle ? { throttle } : {}), + }) as Stitch, + ); +} + +async function main(): Promise { + heading('C3 — bounded concurrency: the measured peak in-flight count'); + + // ── (a) baseline: no bound at all ───────────────────────────────────────────────────────── + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = onePartStitch(api); + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check('(a) 8 parts, no throttle → peak in-flight', api.peakInFlight, 8); + } + + // ── (b) does `all()` bound anything? ────────────────────────────────────────────────────── + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + await all(perPartStitches(api, uploadId))({ + params: { key: 'v.mp4' }, + body: { chunk: 'c' }, + }); + check('(b) all() over 8 members → peak in-flight', api.peakInFlight, 8); + check('(b) parts actually stored', api.storedParts(uploadId), 8); + note( + '(b) → `all()` is fail-fast + auto-cancel + child runs, NOT a pool', + 'pipe.ts:122-136 is `members.map(...)` straight into `Promise.all` — nothing rations starts', + ); + } + + // ── (c) …and every member gets the SAME input ───────────────────────────────────────────── + // The reason (b) had to build 8 stitches. Give `all()` the same stitch 8 times and all 8 PUTs + // carry the same part number — 8 requests, ONE part stored. + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = onePartStitch(api); + await all([part, part, part, part, part, part, part, part])({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'c' }, + }); + check('(c) requests the server saw', api.partPuts, 8); + checkSeq( + '(c) part numbers on those requests', + [...new Set(api.partPutOrder)], + [1], + ); + check('(c) distinct parts stored', api.storedParts(uploadId), 1); + note( + '(c) → `all()` cannot vary input across members', + '`runMember` spreads ONE `StitchInput` over all of them (pipe.ts:75-76) — a part fan needs N stitches or N closures', + ); + } + + // ── (d) one stitch, N calls, `throttle.concurrency` — the bound that WORKS ──────────────── + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = onePartStitch(api, { concurrency: 3 }); + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check( + '(d) one stitch × 8 calls, concurrency: 3 → peak', + api.peakInFlight, + 3, + ); + check('(d) all 8 parts landed', api.storedParts(uploadId), 8); + note( + '(d) → this is the real answer', + 'the bound lives on the STITCH, so the fan has to be N calls to ONE stitch', + ); + } + + // ── (e) …and the combination that looks right and is not ────────────────────────────────── + // `all()` + `throttle.concurrency: 3` on each member. Default `pool: 'stitch'` gives each + // stitch its OWN state map (resilience.ts:103-109), so 8 stitches = 8 buckets of 3 = no bound. + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + await all(perPartStitches(api, uploadId, { concurrency: 3 }))({ + params: { key: 'v.mp4' }, + body: { chunk: 'c' }, + }); + check( + '(e) all() + concurrency: 3 on EVERY member → peak', + api.peakInFlight, + 8, + ); + note( + '(e) → the limit is per stitch, and `all()` needs one stitch per part', + 'the config reads "concurrency: 3" eight times and bounds nothing', + ); + } + + // ── (f) the two spellings that DO pool across stitches ──────────────────────────────────── + // `pool: 'host'` keys the bucket on the URL host and keeps its state in a module-level map + // (resilience.ts:107), so separate stitch instances share it. + { + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + await all( + perPartStitches(api, uploadId, { concurrency: 3, pool: 'host' }), + )({ params: { key: 'v.mp4' }, body: { chunk: 'c' } }); + check( + '(f) all() + concurrency: 3, pool: "host" → peak', + api.peakInFlight, + 3, + ); + } + { + // A seam re-keys every member's acquire onto one seam-stable key (seam.ts:46-68). + const api = new FakeS3({ partTicks: SLOW }); + const uploadId = await openUpload(api, 'v.mp4'); + const bucket = seam({ + throttle: { concurrency: 3 }, + adapter: api.adapter(), + }); + const members = PARTS.map( + (n) => + bucket.stitch({ + url: `${FakeS3.template}?partNumber=${n}&uploadId=${uploadId}`, + method: 'PUT', + kind: partSurface, + }) as Stitch, + ); + await all(members)({ params: { key: 'v.mp4' }, body: { chunk: 'c' } }); + check( + '(f) seam({ throttle: { concurrency: 3 } }) → peak', + api.peakInFlight, + 3, + ); + note( + '(f) → two working spellings, neither of them the obvious one', + '`pool: "host"` or a seam bucket; plain per-stitch `concurrency` is a per-stitch bound', + ); + } + + // ── (g) the bound holds under the ordering C2 needs ─────────────────────────────────────── + // A pool changes WHEN parts land; it must not change what order the ETags end up in. + { + const api = new FakeS3({ + partTicks: { 8: 0, 7: 1, 6: 2, 5: 3, 4: 40, 3: 41, 2: 42, 1: 43 }, + }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = onePartStitch(api, { concurrency: 3 }); + const etags = await Promise.all( + PARTS.map(async (n) => ({ + PartNumber: n, + ETag: (await part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + })) as string, + })), + ); + const complete = stitch({ + url: FakeS3.template, + method: 'POST', + adapter: api.adapter(), + }); + const done = await complete.safe({ + params: { key: 'v.mp4' }, + query: { uploadId }, + body: { Parts: etags }, + }); + check('(g) peak under the pool', api.peakInFlight, 3); + check( + '(g) server storage order differed from part order', + api.completionOrder.join(',') !== PARTS.join(','), + true, + ); + checkSeq( + '(g) ETag list order', + etags.map((e) => e.PartNumber), + PARTS, + ); + check('(g) complete → ok', done.ok, true); + check('(g) orphans', api.orphanParts, 0); + } + + finish( + 'C3', + 'PASS, but NOT via `all()`. Measured peaks over 8 stalled parts: no throttle → **8**; `all()` over 8 members → **8** (it bounds nothing — pipe.ts:122-136 maps every member straight into `Promise.all`); ONE stitch called 8 times with `throttle: { concurrency: 3 }` → **3**. The trap is the combination that reads correct: `all()` hands every member the SAME `StitchInput` (measured: one stitch × 8 members produced 8 PUTs all carrying `partNumber=1` and stored ONE part), so a part fan needs 8 distinct stitches — and 8 stitches with `concurrency: 3` each measured a peak of **8**, because the default `pool: "stitch"` gives every stitch its own state map (resilience.ts:103-109). Two spellings do pool across stitches: `{ concurrency: 3, pool: "host" }` → **3**, and a `seam({ throttle: { concurrency: 3 } })` bucket → **3** (seam.ts:46-68). Under the pool the ordering C2 needs still holds: peak 3, server storage order ≠ part order, ETag list [1..8], complete ok, 0 orphans', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c4-no-compensation.ts b/docs/scenarios/proofs/multipart-upload/c4-no-compensation.ts new file mode 100644 index 00000000..2a9ac0f0 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c4-no-compensation.ts @@ -0,0 +1,415 @@ +// C4 — THE DECIDING CLAIM. Is there ANY seam that guarantees the abort runs on failure? And when +// nothing runs it, how many parts are left billing? +// +// The capture's hypothesis is "there is no compensation hook anywhere in core". That is confirmed, +// and the confirmation is sharper than the hypothesis: the hook that LOOKS like the answer — +// `hooks.onError` — is not a failure hook at all. It fires only when the TRANSPORT THROWS +// (engine.ts:668-680: it is the `catch` around `withTimeout(transport)`), so an HTTP 500, an HTTP +// 400, and a surface `{ ok: false }` verdict all reach the caller as failures having fired ZERO +// `onError` callbacks. Wiring cleanup to `onError` produces a cleanup that runs on the network +// blips and skips the application failures. +// +// Everything here is counted by the server: `orphanParts` (stored under an UploadId that was never +// completed or aborted), `orphanBytes`, `danglingUploads`, and `aborted` (how many DELETEs actually +// arrived). +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c4-no-compensation.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchConfig, + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const partSurface: Surface = { + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +const MiB = 1024 * 1024; + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +async function main(): Promise { + heading( + 'C4 — is there a compensation seam, and what does the orphan cost?', + ); + + // ── (a) the four hooks, and what each one actually answers to ───────────────────────────── + // `Hooks` is exactly `{ onRequest, onResponse, onError, onRetry }` (types.ts:1285-1290). The + // question is not whether a fifth one exists — it is whether the fourth does the job. + { + const api = new FakeS3(); + api.failPart(1, 1, 500); // an ordinary application failure + const uploadId = await openUpload(api, 'v.mp4'); + const fired: string[] = []; + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + hooks: { + onRequest: () => void fired.push('onRequest'), + onResponse: () => void fired.push('onResponse'), + onError: () => void fired.push('onError'), + onRetry: () => void fired.push('onRetry'), + }, + }); + const r = await part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'a' }, + }); + check('(a) the call failed', r.ok, false); + check('(a) …with status', r.error?.status, 500); + checkSeq('(a) hooks that fired', fired, ['onRequest', 'onResponse']); + check( + '(a) onError calls on a FAILED call', + fired.filter((f) => f === 'onError').length, + 0, + ); + note( + '(a) → `onError` is a TRANSPORT-EXCEPTION hook, not a failure hook', + 'engine.ts:676 sits in the `catch` around `withTimeout(transport)`; a 500 is a RESPONSE, so it never gets there', + ); + } + + // ── (b) …and when it DOES fire, it fires per ATTEMPT ────────────────────────────────────── + { + const clock = manualClock(); + const api = new FakeS3({ hangParts: [1] }); + const uploadId = await openUpload(api, 'v.mp4'); + const fired: number[] = []; + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + clock, + timeout: { perAttempt: 1_000 }, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + hooks: { onError: (ctx) => void fired.push(ctx.attempt) }, + }); + const pending = part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'a' }, + }); + await clock.advance(60_000); + const r = await pending; + check('(b) stalled part → ok', r.ok, false); + checkSeq('(b) onError fired on attempts', fired, [1, 2, 3]); + note( + '(b) → an abort written into `onError` would fire 3 times for one part', + 'and would fire on transient blips that the retry is about to fix', + ); + } + + // ── (c) the trace sink sees the terminal `done` — but per CALL ──────────────────────────── + { + const api = new FakeS3(); + api.failPart(3, 1, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const dones: { name: string; ok: boolean }[] = []; + const sink: TraceSink = { + handle: (e: StitchEvent, ctx: TraceContext) => { + if (e.type === 'done') dones.push({ name: ctx.name, ok: e.ok }); + }, + }; + const part = stitch({ + name: 'part', + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + trace: sink, + }); + await Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check('(c) `done` events seen by the sink', dones.length, 4); + check('(c) …of which failed', dones.filter((d) => !d.ok).length, 1); + checkSeq( + '(c) names on those events', + [...new Set(dones.map((d) => d.name))], + ['part'], + ); + note( + '(c) → a sink CAN observe the failure, and still cannot compensate', + 'it fires per stitch call, names only the stitch, and carries no UploadId — and it is a LOG seam, not a control seam', + ); + } + + // ── (d) THE ORPHAN. Part 3 fails, nothing cleans up ─────────────────────────────────────── + { + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const results = await Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check( + '(d) parts that succeeded', + results.filter((r) => r.ok).length, + 3, + ); + check('(d) ORPHANED PARTS', api.orphanParts, 3); + check('(d) ORPHANED BYTES', api.orphanBytes, 15 * MiB); + check('(d) dangling UploadIds', api.danglingUploads, 1); + check('(d) DELETEs the library issued', api.aborted, 0); + check( + '(d) upload status on the server', + api.statusOf(uploadId), + 'open', + ); + note( + '(d) → 15 MiB of a 20 MiB upload is now invisible and billed', + 'nothing in the failure path knows an UploadId exists, so nothing can free it', + ); + } + + // ── (e) a CANCELLED run skips cleanup the same way — and more quietly ───────────────────── + // Parts 1-2 land, then the caller's AbortSignal fires. The library cancels; it does not clean. + { + const api = new FakeS3({ partTicks: { 1: 0, 2: 0, 3: 60, 4: 60 } }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const ac = new AbortController(); + const pending = Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + signal: ac.signal, + }), + ), + ); + for (let i = 0; i < 30; i++) await Promise.resolve(); + ac.abort(); + const results = await pending; + check( + '(e) parts that landed before the abort', + results.filter((r) => r.ok).length, + 2, + ); + check('(e) ORPHANED PARTS after cancellation', api.orphanParts, 2); + check('(e) DELETEs issued', api.aborted, 0); + note( + '(e) → cancellation and cleanup are different concerns, and only one is built in', + 'the user pressed Cancel; the bucket kept the bytes', + ); + } + + // ── (f) a TIMEOUT skips it too, and this one fires `onError` ────────────────────────────── + { + const clock = manualClock(); + const api = new FakeS3({ hangParts: [3] }); + const uploadId = await openUpload(api, 'v.mp4'); + let onErrorCalls = 0; + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + clock, + timeout: { total: 1_000 }, + hooks: { onError: () => void (onErrorCalls += 1) }, + }); + const pending = Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + await clock.advance(10_000); + const results = await pending; + check( + '(f) parts that succeeded', + results.filter((r) => r.ok).length, + 3, + ); + check('(f) onError calls', onErrorCalls, 1); + check('(f) ORPHANED PARTS after a timeout', api.orphanParts, 3); + check('(f) DELETEs issued', api.aborted, 0); + note( + '(f) → the one path where `onError` fires is also the one where it is least useful', + 'it fires on the PART stitch, per attempt, with no UploadId in `HookContext` (types.ts:1279-1284)', + ); + } + + // ── (g) an unknown config key is accepted at RUNTIME and silently ignored ───────────────── + // TypeScript's `NoUnknownConfigKeys` rejects `onFinally` at compile time, which is the real + // guard. But the runtime does not: the key survives onto `__config` and is never called. A JS + // consumer — or anything that builds config dynamically — gets a hook that reads as wired. + { + const api = new FakeS3(); + api.failPart(1, 1, 500); + const uploadId = await openUpload(api, 'v.mp4'); + let ran = false; + // Written as a literal, `NoUnknownConfigKeys` makes this a COMPILE error — that guard is + // the only thing standing between a team and a hook they think is wired. Cast through + // `Partial` to reach the runtime and see what it does with the key. + const invented = { + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + onFinally: () => { + ran = true; + }, + } as unknown as Partial; + const part = stitch(invented); + await part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'a' }, + }); + check('(g) the invented `onFinally` ran', ran, false); + check( + '(g) …but it IS on the resolved config', + Object.keys( + (part as unknown as { __config: Record }) + .__config, + ).includes('onFinally'), + true, + ); + note( + '(g) → there is no compensation key to find, and inventing one fails silently at runtime', + 'the TS guard is the only thing that catches it', + ); + } + + // ── (h) the only thing that works, and the way it silently does not ─────────────────────── + { + // h1: a real `try/finally` in user code. Zero orphans. + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const abort = stitch({ + url: FakeS3.template, + method: 'DELETE', + adapter: api.adapter(), + }); + let done = false; + try { + await Promise.all( + [1, 2, 3, 4].map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + done = true; + } catch { + /* the abort is in `finally` */ + } finally { + if (!done) + await abort.safe({ + params: { key: 'v.mp4' }, + query: { uploadId }, + }); + } + check('(h1) user try/finally → orphaned parts', api.orphanParts, 0); + check('(h1) …DELETEs issued', api.aborted, 1); + } + { + // h2: the SAME code with one thing wrong — the abort targets the wrong UploadId. `.safe()` + // swallows the 404, nothing throws, and the cleanup reports success it did not achieve. + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const abort = stitch({ + url: FakeS3.template, + method: 'DELETE', + adapter: api.adapter(), + }); + let cleanupThrew = false; + try { + await Promise.all( + [1, 2, 3, 4].map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + } catch { + /* fall through */ + } finally { + const r = await abort.safe({ + params: { key: 'v.mp4' }, + query: { uploadId: `${uploadId}-typo` }, + }); + cleanupThrew = !r.ok && false; // `.safe()` never throws — that is the point + } + check('(h2) the cleanup call threw', cleanupThrew, false); + check('(h2) DELETEs the SERVER accepted', api.aborted, 0); + check('(h2) ORPHANED PARTS', api.orphanParts, 3); + note( + '(h2) → this is the failure mode that counts double', + 'the finally ran, the code looks correct, and 15 MiB is still billing. `.safe()` on a cleanup call hides the one error you must not ignore', + ); + } + + finish( + 'C4', + 'NO — there is no compensation seam anywhere in core, and the hook that looks like one is worse than absent. `Hooks` is exactly `{onRequest,onResponse,onError,onRetry}` (types.ts:1285-1290) and `onError` fires only from the `catch` around the transport (engine.ts:668-680): on an HTTP 500 the measured hook sequence was ["onRequest","onResponse"] with **0** `onError` calls, while a stalled socket fired it **3** times (once per attempt). `HookContext` (types.ts:1279-1284) carries `{name,attempt,req,res,error}` — no UploadId, no run-scoped slot to keep one. A trace sink sees the terminal `done` (measured 4 events, 1 with `ok:false`) but is a LOG seam, per stitch call, with no UploadId either. `linked()` is `Promise.resolve(body(run))` (pipe.ts:357-369) — no finally. Inventing `onFinally` is accepted at RUNTIME, lands on `__config`, and never runs (measured). THE ORPHAN, with a part failing and no user cleanup: **3 parts / 15 MiB / 1 dangling UploadId / 0 DELETEs**. A CANCELLED run (AbortSignal) measured **2 orphaned parts, 0 DELETEs**; a `timeout.total` expiry measured **3 orphaned parts, 0 DELETEs** — and that is the one path where `onError` fires, on the part stitch, per attempt. A user-written `try/finally` measured **0 orphans / 1 DELETE**. The same `try/finally` with `.safe()` on the cleanup and a wrong UploadId measured **0 accepted DELETEs / 3 orphans and nothing thrown**', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c5-retry-granularity.ts b/docs/scenarios/proofs/multipart-upload/c5-retry-granularity.ts new file mode 100644 index 00000000..48d800bb --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c5-retry-granularity.ts @@ -0,0 +1,335 @@ +// C5 — retry granularity. Can ONE part retry without re-sending the others, and is a WHOLE-UPLOAD +// retry prevented? +// +// The measurement is `partPutOrder` (every part number the server saw, in arrival order) and +// `initiated` (how many `POST ?uploads` arrived). A per-part retry shows one number twice; a +// whole-upload retry shows `initiated: 2` and leaves the first UploadId's parts orphaned. +// +// The finding the capture does not anticipate is (b): the default `retry.on` is +// `[429, 502, 503, 504]` (engine.ts:612). S3's own transient failure is `500 InternalError`, and +// **it is not in that set** — so the retry most people believe they configured does not fire for the +// status the vendor actually sends. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c5-retry-granularity.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const partSurface: Surface = { + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +const NO_WAIT = { attempts: 3, backoff: { curve: 'fixed' as const, base: 0 } }; +const MiB = 1024 * 1024; + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +/** + * The whole upload as ONE stitch, via `Surface.execute` (ADR 0008) — the only construction that + * puts an orchestration inside the engine's retry/timeout/trace chain. `cleanup` decides whether + * the abort is written into it. + */ +function uploadSurface( + api: FakeS3, + parts: number[], + opts: { cleanup: boolean }, +): Surface { + const transport = api.adapter(); + const execute: Adapter = async (req) => { + const url = new URL(req.url); + const key = url.pathname.replace('/bucket/', ''); + const init = await transport({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + const uploadId = (init.body as { UploadId: string }).UploadId; + try { + const etags = await Promise.all( + parts.map(async (n) => { + const res = await transport({ + url: `${FakeS3.url(key)}?partNumber=${n}&uploadId=${uploadId}`, + method: 'PUT', + headers: {}, + body: { chunk: `c${n}` }, + }); + if (res.status >= 400) + throw new Error(`part ${n}: HTTP ${res.status}`); + return { PartNumber: n, ETag: res.headers['etag'] }; + }), + ); + const done = await transport({ + url: `${FakeS3.url(key)}?uploadId=${uploadId}`, + method: 'POST', + headers: {}, + body: { Parts: etags }, + }); + if (done.status >= 400) + throw new Error(`complete: HTTP ${done.status}`); + return done; + } catch (e) { + if (opts.cleanup) + await transport({ + url: `${FakeS3.url(key)}?uploadId=${uploadId}`, + method: 'DELETE', + headers: {}, + }); + throw e; + } + }; + return { id: 'multipart-upload', execute }; +} + +async function main(): Promise { + heading('C5 — per-part retry vs whole-upload retry'); + + // ── (a) one part fails once (503) — is only that part re-sent? ──────────────────────────── + { + const api = new FakeS3(); + api.failPart(3, 1, 503); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + retry: NO_WAIT, + }); + const etags = await Promise.all( + [1, 2, 3, 4].map(async (n) => ({ + PartNumber: n, + ETag: (await part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + })) as string, + })), + ); + const complete = stitch({ + url: FakeS3.template, + method: 'POST', + adapter: api.adapter(), + }); + const done = await complete.safe({ + params: { key: 'v.mp4' }, + query: { uploadId }, + body: { Parts: etags }, + }); + check('(a) part PUTs the server saw', api.partPuts, 5); + checkSeq( + '(a) …which parts, in arrival order', + api.partPutOrder, + [1, 2, 3, 4, 3], + ); + check('(a) `POST ?uploads` calls', api.initiated, 1); + check('(a) complete → ok', done.ok, true); + check('(a) orphans', api.orphanParts, 0); + note( + '(a) → per-part retry is exact', + 'part 3 alone was re-sent; parts 1/2/4 were not touched. On a 5 GB upload that is 5 MB re-sent instead of 5 GB', + ); + } + + // ── (b) …but only for the statuses the DEFAULT set covers ───────────────────────────────── + // `retry.on` defaults to `[429, 502, 503, 504]` (engine.ts:612). S3's transient error is + // `500 InternalError`. The same config that saved (a) does nothing here. + { + const api = new FakeS3(); + api.failPart(3, 1, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + retry: NO_WAIT, + }); + const results = await Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check( + '(b) 500 with retry: { attempts: 3 } → part PUTs', + api.partPuts, + 4, + ); + check('(b) …parts that failed', results.filter((r) => !r.ok).length, 1); + check('(b) orphans left by the un-retried 500', api.orphanParts, 3); + + // The fix, measured. + const api2 = new FakeS3(); + api2.failPart(3, 1, 500); + const uploadId2 = await openUpload(api2, 'v.mp4'); + const part2 = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api2.adapter(), + retry: { ...NO_WAIT, on: [429, 500, 502, 503, 504] }, + }); + const results2 = await Promise.all( + [1, 2, 3, 4].map((n) => + part2.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId: uploadId2 }, + body: { chunk: `c${n}` }, + }), + ), + ); + checkSeq( + '(b) with `on: [...500...]` → arrival order', + api2.partPutOrder, + [1, 2, 3, 4, 3], + ); + check( + '(b) …parts that failed', + results2.filter((r) => !r.ok).length, + 0, + ); + note( + '(b) → `retry.on` must be widened for S3', + "`500 InternalError` is S3's documented transient error and is NOT in the default set", + ); + } + + // ── (c) a THROWN transport error is retried regardless of `retry.on` ────────────────────── + // Different code path (engine.ts:681 — `if (attempt < max)` with no status match), which is why + // the `execute` construction in (d) retries at all. + { + const api = new FakeS3(); + let thrown = 0; + const flaky: Adapter = async (req) => { + if (req.method === 'PUT' && (thrown += 1) === 1) + throw new Error('ECONNRESET'); + return api.adapter()(req); + }; + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: flaky, + retry: { ...NO_WAIT, on: [418] }, // deliberately matches nothing + }); + const r = await part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: 'c1' }, + }); + check('(c) thrown error + `on: [418]` → ok', r.ok, true); + check('(c) attempts made', api.partPuts, 1); + note( + '(c) → `retry.on` gates STATUS retries only', + 'a throw retries while attempts remain, which is what makes the `execute` construction below retryable', + ); + } + + // ── (d) WHOLE-UPLOAD retry: nothing prevents it, and it orphans the first UploadId ──────── + { + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const upload = stitch({ + url: FakeS3.template, + method: 'POST', + kind: uploadSurface(api, [1, 2, 3, 4], { cleanup: false }), + retry: NO_WAIT, + }); + const r = await upload.safe({ params: { key: 'v.mp4' } }); + check('(d) upload → ok', r.ok, false); + check('(d) `POST ?uploads` calls (attempts × 1)', api.initiated, 3); + check('(d) part PUTs across all attempts', api.partPuts, 12); + check('(d) DANGLING UploadIds', api.danglingUploads, 3); + check('(d) ORPHANED PARTS', api.orphanParts, 9); + check('(d) ORPHANED BYTES', api.orphanBytes, 45 * MiB); + note( + '(d) → whole-upload retry is not prevented; it MULTIPLIES the orphan', + '3 attempts re-sent 12 parts and left 3 dangling UploadIds holding 45 MiB', + ); + } + + // ── (e) the same construction with cleanup inside `execute` ─────────────────────────────── + // This is the shape C8 builds on: the retry is still whole-upload (still wasteful), but every + // attempt cleans up after itself, so the orphan is zero at every exit. + { + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const upload = stitch({ + url: FakeS3.template, + method: 'POST', + kind: uploadSurface(api, [1, 2, 3, 4], { cleanup: true }), + retry: NO_WAIT, + }); + const r = await upload.safe({ params: { key: 'v.mp4' } }); + check('(e) upload → ok', r.ok, false); + check('(e) `POST ?uploads` calls', api.initiated, 3); + check('(e) DELETEs issued', api.aborted, 3); + check('(e) ORPHANED PARTS', api.orphanParts, 0); + check('(e) dangling UploadIds', api.danglingUploads, 0); + note( + '(e) → the retry is still the wrong granularity, but it is no longer a billing incident', + 'cleanup belongs INSIDE the unit the retry re-runs', + ); + } + + // ── (f) the granularity that is actually wanted: retry per part, once, inside `execute` ─── + { + const api = new FakeS3(); + api.failPart(3, 1, 500); // one transient blip + const transport = api.adapter(); + const partStitch = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: transport, + retry: { ...NO_WAIT, on: [429, 500, 502, 503, 504] }, + }); + const uploadId = await openUpload(api, 'v.mp4'); + const etags = await Promise.all( + [1, 2, 3, 4].map(async (n) => ({ + PartNumber: n, + ETag: (await partStitch({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + })) as string, + })), + ); + const complete = stitch({ + url: FakeS3.template, + method: 'POST', + adapter: transport, + }); + const done = await complete.safe({ + params: { key: 'v.mp4' }, + query: { uploadId }, + body: { Parts: etags }, + }); + check('(f) `POST ?uploads` calls', api.initiated, 1); + check('(f) part PUTs', api.partPuts, 5); + check('(f) complete → ok', done.ok, true); + check('(f) orphans', api.orphanParts, 0); + } + + finish( + 'C5', + "Per-part retry: PASS. Whole-upload retry: NOT PREVENTED, and it multiplies the orphan. Measured: one part failing 503 once, with `retry: { attempts: 3 }` on the part stitch, produced arrival order **[1,2,3,4,3]** — 5 PUTs, one `POST ?uploads`, complete ok, 0 orphans. Only part 3 was re-sent. THE TRAP: the default `retry.on` is `[429,502,503,504]` (engine.ts:612) and S3's own transient failure is `500 InternalError` — the identical config against a 500 measured **4 PUTs, 1 failed part, 3 orphans**; widening to `on: [429,500,502,503,504]` restored [1,2,3,4,3] and 0 failures. A THROWN transport error is retried regardless of `retry.on` (measured ok with `on: [418]`), which is what makes the outer construction retryable at all. Putting `retry: { attempts: 3 }` on a whole-upload stitch (`Surface.execute` running initiate→parts→complete) measured **3 `POST ?uploads`, 12 part PUTs, 3 dangling UploadIds, 9 orphaned parts, 45 MiB** — nothing in the library flags or prevents it. The same construction with the abort written INSIDE `execute` measured **3 initiates, 3 DELETEs, 0 orphans**: still the wrong retry granularity, but no longer a billing incident", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c6-cancelled-siblings.ts b/docs/scenarios/proofs/multipart-upload/c6-cancelled-siblings.ts new file mode 100644 index 00000000..de513667 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c6-cancelled-siblings.ts @@ -0,0 +1,248 @@ +// C6 — when `all()` auto-cancels the siblings after one part fails, what happens to the parts that +// ALREADY LANDED? Are their ETags still reachable for the abort, or lost? +// +// Two different quantities, both counted by the server / the client: +// +// - **stored server-side** — parts sitting in the bucket under the UploadId. These are billed. +// - **nameable client-side** — parts whose ETag the client can still produce. +// +// `all()` rejects with the FIRST error and discards the resolved values of the members that +// succeeded (it is `Promise.all`, pipe.ts:133), so the second number is ZERO by default while the +// first is not. That gap is why "the parts that already landed still need the abort" is a real +// concern rather than a theoretical one — and it is why a resumable upload (re-using the landed +// parts instead of discarding them) is impossible without a side channel. +// +// pipe.ts:20 is explicit that there is no `allSettled` variant: "for that, compose `.safe()` members +// by hand". +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c6-cancelled-siblings.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import * as pipe from '../../../../packages/core/src/pipe'; +import { all } from '../../../../packages/core/src/pipe'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const partSurface: Surface = { + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +/** Parts 1 and 2 land immediately; part 3 fails at tick 10; part 4 is still in flight. */ +const TIMING = { 1: 0, 2: 0, 3: 10, 4: 40 }; + +/** + * Spin the microtask queue so a cancelled member's handler finishes before anything is counted. + * `all()` rejects the moment part 3 fails — part 4's request is still inside the server when + * control returns, so measuring immediately reports 3 hits and misses the cancellation itself. + */ +const drain = async (): Promise => { + for (let i = 0; i < 200; i++) await Promise.resolve(); +}; + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +/** N part stitches with the part number baked into the URL — what `all()` requires (C3(c)). */ +function members( + api: FakeS3, + uploadId: string, + onEtag?: (n: number, etag: string) => void, +): Stitch[] { + return [1, 2, 3, 4].map( + (n) => + stitch({ + url: `${FakeS3.template}?partNumber=${n}&uploadId=${uploadId}`, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + ...(onEtag + ? { + hooks: { + onResponse: (ctx) => { + const etag = ctx.res?.headers['etag']; + if (etag && ctx.res && ctx.res.status < 400) + onEtag(n, etag); + }, + }, + } + : {}), + }) as Stitch, + ); +} + +async function main(): Promise { + heading('C6 — the parts that landed before the fan was cancelled'); + + // ── (a) the default: stored ≠ nameable ──────────────────────────────────────────────────── + { + const api = new FakeS3({ partTicks: TIMING }); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + let nameable = 0; + let caught = ''; + try { + const values = (await all(members(api, uploadId))({ + params: { key: 'v.mp4' }, + body: { chunk: 'c' }, + })) as unknown[]; + nameable = values.length; + } catch (e) { + caught = (e as Error).message; + } + await drain(); + check('(a) all() rejected', caught !== '', true); + check('(a) parts STORED server-side', api.storedParts(uploadId), 2); + check('(a) parts the client can NAME', nameable, 0); + check('(a) part PUTs that reached the server', api.partPuts, 4); + checkSeq( + '(a) …their statuses', + api.hits.filter((h) => h.op === 'part').map((h) => h.status), + [200, 200, 500, 499], + ); + check('(a) orphaned parts', api.orphanParts, 2); + note( + '(a) → the auto-cancel WORKS (part 4 was cut off, status 499) and is not cleanup', + '2 parts are in the bucket and the client holds zero of their ETags', + ); + } + + // ── (b) a side channel recovers them ────────────────────────────────────────────────────── + // `hooks.onResponse` writes each ETag into a Map as it arrives, so the values survive the + // rejection. This is the shape any resumable/partial-retry design needs. + { + const api = new FakeS3({ partTicks: TIMING }); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const landed = new Map(); + try { + await all(members(api, uploadId, (n, e) => landed.set(n, e)))({ + params: { key: 'v.mp4' }, + body: { chunk: 'c' }, + }); + } catch { + /* expected */ + } + await drain(); + checkSeq( + '(b) ETags recovered from the side channel', + [...landed.keys()].sort((x, y) => x - y), + [1, 2], + ); + check( + '(b) …and they match what the server stored', + landed.get(1), + `"${uploadId}-p1"`, + ); + check('(b) stored server-side', api.storedParts(uploadId), 2); + note( + '(b) → the fix is a closure, not a config field', + 'nothing in `all()` hands back partial results; `hooks.onResponse` is the only place to catch them', + ); + } + + // ── (c) there is no `allSettled`, and that is on purpose ────────────────────────────────── + { + const combinators = Object.keys(pipe).filter( + (k) => typeof (pipe as Record)[k] === 'function', + ); + // Note the subpath is named `pipe` and there is no `pipe()` combinator in it. + checkSeq('(c) exported combinators', combinators.sort(), [ + 'all', + 'any', + 'linked', + 'race', + ]); + check( + '(c) an `allSettled` combinator exists', + combinators.includes('allSettled'), + false, + ); + note( + '(c) → pipe.ts:20 says so outright', + '"There is deliberately NO `allSettled` (best-effort) variant; for that, compose `.safe()` members by hand"', + ); + } + + // ── (d) `.safe()` members: every value survives, and so does the auto-cancel loss ───────── + // Composing `.safe()` by hand keeps the successes — but a `.safe()` member never REJECTS, so + // `all()` never cancels anything. You trade the loss of partial results for the loss of + // fail-fast. Measured: 4 parts land instead of 2. + { + const api = new FakeS3({ partTicks: TIMING }); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: api.adapter(), + }); + const settled = await Promise.all( + [1, 2, 3, 4].map((n) => + part.safe({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: `c${n}` }, + }), + ), + ); + check( + '(d) .safe() members → results the client holds', + settled.filter((r) => r.ok).length, + 3, + ); + check('(d) parts STORED server-side', api.storedParts(uploadId), 3); + checkSeq( + '(d) statuses the server saw', + api.hits.filter((h) => h.op === 'part').map((h) => h.status), + [200, 200, 500, 200], + ); + note( + '(d) → part 4 was NOT cancelled: it uploaded in full after part 3 had already failed', + 'on a real 5 GB upload that is megabytes sent for an upload that is already doomed', + ); + } + + // ── (e) the practical consequence: with the ETags in hand, the abort still needs the ID ─── + // Cancellation loses the values; it never loses the UploadId, because the UploadId is a plain + // variable in the orchestration. That asymmetry is the whole reason a `try/finally` works at + // all — and the reason nothing smaller than the orchestration can do the cleanup. + { + const api = new FakeS3({ partTicks: TIMING }); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const uploadId = await openUpload(api, 'v.mp4'); + const abort = stitch({ + url: FakeS3.template, + method: 'DELETE', + adapter: api.adapter(), + }); + try { + await all(members(api, uploadId))({ + params: { key: 'v.mp4' }, + body: { chunk: 'c' }, + }); + } catch { + await abort({ params: { key: 'v.mp4' }, query: { uploadId } }); + } + await drain(); + check('(e) DELETEs accepted', api.aborted, 1); + check('(e) orphaned parts', api.orphanParts, 0); + check('(e) upload status', api.statusOf(uploadId), 'aborted'); + } + + finish( + 'C6', + 'The auto-cancel works and it is NOT cleanup — the two counts diverge. Measured with parts 1-2 landing, part 3 failing 500, part 4 still in flight: the server saw statuses **[200,200,500,499]** (part 4 genuinely cut off by the group signal), **2 parts stored**, **2 orphaned** — and the client could name **0** of their ETags, because `all()` is `Promise.all` (pipe.ts:133) and discards resolved values on rejection. A `hooks.onResponse` side channel recovered exactly **[1,2]** with byte-exact ETags, which is the only way partial results survive. There is no `allSettled`: the pipe subpath exports exactly ["all","any","linked","race"] (there is no `pipe()` combinator in it either) and pipe.ts:20 says the omission is deliberate. Composing `.safe()` members instead keeps every value (**3 of 4 ok**) but disables the fail-fast — measured statuses **[200,200,500,200]**, i.e. part 4 uploaded in full into an upload that was already doomed. The UploadId, unlike the ETags, is never lost: it is a plain variable in the orchestration, which is why a `try/finally` at that level cleans up (measured 1 DELETE, 0 orphans, status `aborted`) and nothing narrower can', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c7-progress-aggregation.ts b/docs/scenarios/proofs/multipart-upload/c7-progress-aggregation.ts new file mode 100644 index 00000000..ad540e25 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c7-progress-aggregation.ts @@ -0,0 +1,271 @@ +// C7 — can per-part byte counts be combined into ONE number for a UI? +// +// Yes, and the sum is trivial. The three things that are not trivial are all measured here: +// +// 1. `AdapterProgress` is `{ direction, loaded, total }` (types.ts:858-867). **No part identity.** +// One shared callback across a concurrent fan cannot tell which part a tick belongs to, so the +// obvious "sum every `loaded`" is wrong — each tick is CUMULATIVE for its own part, not a +// delta. Measured: the naive sum overshoots the file size by 2.5×. +// 2. A RETRY re-sends the part and replays its ticks from zero, so a per-part high-water mark is +// the only thing that does not make the bar go backwards. +// 3. Byte progress has no event: `ProgressPhase` is `auth|request|throttled|retry|reconnect| +// paginate|circuit|cache` (types.ts:1293-1305). `onProgress` is runtime-only — it is not on +// `__config` and never reaches a trace sink. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c7-progress-aggregation.ts +import { + stitch, + verdictOf, + xhrAdapter, +} from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { AdapterProgress } from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { fakeXhrCtor } from './fake-xhr'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const partSurface: Surface = { + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +/** Four equal chunks; the encoded JSON body is 40 bytes each, so the "file" is 160 bytes. */ +const CHUNK = 'z'.repeat(28); // `{"chunk":"z…z"}` → 40 chars +const PART_BODY_BYTES = JSON.stringify({ chunk: CHUNK }).length; +const PARTS = [1, 2, 3, 4]; + +async function openUpload(api: FakeS3, key: string): Promise { + const res = await api.adapter()({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + return (res.body as { UploadId: string }).UploadId; +} + +async function main(): Promise { + heading('C7 — one number for the UI, out of N per-part byte counts'); + + check('(setup) bytes per part body', PART_BODY_BYTES, 40); + + // ── (a) the naive aggregate: sum every `loaded` ─────────────────────────────────────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 4 })), + }); + let naive = 0; + const ticks: number[] = []; + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: CHUNK }, + onProgress: (p: AdapterProgress) => { + if (p.direction !== 'upload') return; + naive += p.loaded; + ticks.push(p.loaded); + }, + }), + ), + ); + check('(a) upload ticks in total', ticks.length, 16); + checkSeq( + '(a) the distinct `loaded` values a tick can carry', + [...new Set(ticks)].sort((x, y) => x - y), + [10, 20, 30, 40], + ); + check('(a) naive Σ loaded', naive, 400); + check('(a) the actual file size', PART_BODY_BYTES * PARTS.length, 160); + note( + '(a) → summing `loaded` overshoots by 2.5×', + 'each tick is CUMULATIVE within its own part; the sum of cumulative counters is not a total', + ); + } + + // ── (b) …and the tick carries nothing to fix it with ────────────────────────────────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 2 })), + }); + const shapes: string[] = []; + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: CHUNK }, + onProgress: (p: AdapterProgress) => { + if (p.direction === 'upload') + shapes.push(Object.keys(p).sort().join(',')); + }, + }), + ), + ); + checkSeq( + '(b) fields on every upload tick', + [...new Set(shapes)], + ['direction,loaded,total'], + ); + note( + '(b) → no part number, no request, no url, no run id', + 'a SHARED `onProgress` across a fan is unattributable; the identity has to come from the call site', + ); + } + + // ── (c) the aggregate that works: one closure per part, high-water per part ─────────────── + { + const api = new FakeS3({ partTicks: { 1: 0, 2: 3, 3: 6, 4: 9 } }); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 4 })), + throttle: { concurrency: 2 }, + }); + const sent = new Map(); + const bar: number[] = []; + const total = PART_BODY_BYTES * PARTS.length; + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: CHUNK }, + // The part number is bound HERE — the only place it exists. + onProgress: (p: AdapterProgress) => { + if (p.direction !== 'upload') return; + sent.set(n, Math.max(sent.get(n) ?? 0, p.loaded)); + let done = 0; + for (const v of sent.values()) done += v; + bar.push(Math.round((done / total) * 100)); + }, + }), + ), + ); + check( + '(c) final aggregate bytes', + [...sent.values()].reduce((a, b) => a + b, 0), + 160, + ); + check('(c) final percentage', bar[bar.length - 1], 100); + check( + '(c) the bar never went backwards', + bar.every((v, i) => i === 0 || v >= (bar[i - 1] ?? 0)), + true, + ); + checkSeq( + '(c) the percentage sequence', + bar, + [6, 13, 19, 25, 31, 38, 44, 50, 56, 63, 69, 75, 81, 88, 94, 100], + ); + note( + '(c) → 16 monotonic ticks ending exactly at 100%', + 'and the peak in-flight stayed at the configured bound while it happened', + ); + check('(c) peak in-flight', api.peakInFlight, 2); + } + + // ── (d) a RETRY replays the part's ticks from zero ───────────────────────────────────────── + // The naive running sum goes backwards (or double-counts). The per-part high-water mark absorbs + // it, because `Math.max` ignores the replay. + { + const api = new FakeS3(); + api.failPart(2, 1, 503); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 2 })), + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + }); + const perPart: number[] = []; + const sent = new Map(); + let naive = 0; + await Promise.all( + PARTS.map((n) => + part({ + params: { key: 'v.mp4' }, + query: { partNumber: n, uploadId }, + body: { chunk: CHUNK }, + onProgress: (p: AdapterProgress) => { + if (p.direction !== 'upload') return; + if (n === 2) perPart.push(p.loaded); + naive += p.loaded; + sent.set(n, Math.max(sent.get(n) ?? 0, p.loaded)); + }, + }), + ), + ); + checkSeq( + '(d) part 2’s ticks across the retry', + perPart, + [20, 40, 20, 40], + ); + check('(d) naive Σ loaded (4 parts, 1 retried)', naive, 300); + check( + '(d) high-water aggregate', + [...sent.values()].reduce((a, b) => a + b, 0), + 160, + ); + note( + '(d) → the retry replays the whole part', + '`Math.max` per part is what keeps the bar honest; a `+=` counts the retried bytes twice', + ); + } + + // ── (e) there is no EVENT for bytes ──────────────────────────────────────────────────────── + { + const api = new FakeS3(); + const uploadId = await openUpload(api, 'v.mp4'); + const part = stitch({ + url: FakeS3.template, + method: 'PUT', + kind: partSurface, + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 4 })), + }); + const phases: string[] = []; + for await (const e of part.stream({ + params: { key: 'v.mp4' }, + query: { partNumber: 1, uploadId }, + body: { chunk: CHUNK }, + onProgress: () => undefined, + })) + if (e.type === 'progress') phases.push(e.phase); + checkSeq('(e) `progress` phases on the event stream', phases, [ + 'request', + ]); + check( + '(e) `onProgress` survives on the public config', + Object.keys( + (part as unknown as { __config: Record }) + .__config, + ).includes('onProgress'), + false, + ); + note( + '(e) → byte progress is a runtime callback and nothing else', + 'no event, no trace record, nothing on `__config` — a UI wires the closure or gets nothing', + ); + } + + finish( + 'C7', + 'PASS, with a correct aggregate that is not the obvious one. Over 4 parts of 40 bytes each (160-byte "file") with 4 upload ticks apiece, the naive `Σ loaded` measured **400** against a real size of **160** — each tick is cumulative WITHIN its part, so summing cumulative counters overshoots by 2.5×. The tick cannot fix this itself: every upload tick measured exactly the fields `direction,loaded,total` (types.ts:858-867) — no part number, no request, no run id — so a SHARED `onProgress` across a concurrent fan is unattributable and the identity must be bound at the call site. Binding it there and keeping a per-part HIGH-WATER mark produced a clean bar: 16 monotonic ticks, percentages [6,13,19,…,94,100], final aggregate exactly 160, peak in-flight 2 under `throttle: { concurrency: 2 }`. A retry replays the part\'s ticks from zero (measured part 2: [20,40,20,40]), which makes the naive sum **300** while the high-water aggregate stayed **160**. And there is no EVENT for bytes: the only `progress` phase on the stream was `request`, `ProgressPhase` (types.ts:1293-1305) has no byte phase, and `onProgress` is absent from `__config` — a UI wires the closure or gets nothing', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/c8-assembled-solution.ts b/docs/scenarios/proofs/multipart-upload/c8-assembled-solution.ts new file mode 100644 index 00000000..e400a336 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/c8-assembled-solution.ts @@ -0,0 +1,382 @@ +// C8 — the best answer assembled from the public API, run end to end on BOTH paths, and compared +// honestly against a feature-matched hand-rolled twin. +// +// `multipart.ts` is the StitchAPI answer; `hand-rolled.ts` is the same behaviour with no StitchAPI +// in it. Both drive the same `FakeS3` transport, so the comparison is on ergonomics and line count, +// not on who got the easier wire. +// +// The four shapes both must survive: a clean upload, a permanently failing part, a caller +// cancellation mid-flight, and a cleanup call that itself fails. Every one of them is asserted to +// leave **zero** orphaned parts (or, in the last case, to be LOUD about the ones it left). +// +// Case (e) is the reason the answer is a plain orchestration function and not a single +// `Surface.execute` stitch: `withTimeout` (resilience.ts:230-244) rejects the caller's promise the +// instant the timer fires and lets `fn` keep running, so a `try/finally` INSIDE `execute` cleans up +// AFTER the caller has already returned. Measured: 3 orphans at the moment the caller sees the +// failure, 0 several turns later. +// +// pnpm exec tsx docs/scenarios/proofs/multipart-upload/c8-assembled-solution.ts +import { stitch, xhrAdapter } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { FakeS3 } from './fake-s3'; +import { fakeXhrCtor } from './fake-xhr'; +import { handRolledUploader } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { multipartUploader } from './multipart'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const MiB = 1024 * 1024; +const CHUNKS = [1, 2, 3, 4].map((n) => ({ chunk: `part-${String(n)}` })); + +/** + * Executable lines — imports (however they wrap), blanks and comment-only lines removed on BOTH + * sides, so the number is the code someone actually writes and maintains. + */ +function executableLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .replace(/^import[\s\S]*?;$/gm, '') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +const drain = async (): Promise => { + for (let i = 0; i < 200; i++) await Promise.resolve(); +}; + +async function main(): Promise { + heading('C8 — the assembled answer, both paths, and the honest comparison'); + + // ── (a) the SUCCESS path, both implementations ──────────────────────────────────────────── + for (const [label, build] of [ + ['stitchapi', multipartUploader], + ['hand-rolled', handRolledUploader], + ] as const) { + // Under a 2-wide pool: 1+2 start, 2 lands first, then 3 starts, 1 lands, 4 starts, 4 lands + // before 3 — so the server stores them out of part order while the bound still holds. + const api = new FakeS3({ partTicks: { 1: 30, 2: 0, 3: 30, 4: 0 } }); + const { upload, lastStats } = build({ + adapter: api.adapter(), + urlTemplate: FakeS3.template, + concurrency: 2, + }); + const object = (await upload('v.mp4', CHUNKS)) as { Parts: number }; + check(`(a) [${label}] object parts`, object.Parts, 4); + check(`(a) [${label}] objects in the bucket`, api.objectCount, 1); + check(`(a) [${label}] orphaned parts`, api.orphanParts, 0); + check(`(a) [${label}] DELETEs issued`, api.aborted, 0); + check(`(a) [${label}] peak in-flight`, api.peakInFlight, 2); + check( + `(a) [${label}] server storage order ${api.completionOrder.join(',')} ≠ part order`, + api.completionOrder.join(',') !== '1,2,3,4', + true, + ); + check(`(a) [${label}] parts reported`, lastStats()?.parts, 4); + } + + // ── (a2) …with a real progress bar, over the transport that can draw one ───────────────── + // Same uploader, `xhrAdapter` instead of a plain adapter (C1). The aggregate is the uploader's + // own per-part high-water map (C7(c)), so this is the whole UI story end to end. + { + const api = new FakeS3({ partTicks: { 1: 0, 2: 4, 3: 8, 4: 12 } }); + const bar: number[] = []; + const { upload } = multipartUploader({ + adapter: xhrAdapter(fakeXhrCtor(api.adapter(), { uploadTicks: 4 })), + urlTemplate: FakeS3.template, + concurrency: 2, + }); + await upload('v.mp4', CHUNKS, { + onProgress: (f) => bar.push(Math.round(f * 100)), + }); + check('(a2) ticks the bar received', bar.length, 16); + check('(a2) final percentage', bar[bar.length - 1], 100); + check( + '(a2) monotonic', + bar.every((v, i) => i === 0 || v >= (bar[i - 1] ?? 0)), + true, + ); + checkSeq( + '(a2) the bar', + bar, + [7, 14, 19, 25, 32, 39, 44, 50, 57, 63, 69, 75, 82, 88, 94, 100], + ); + check('(a2) orphans', api.orphanParts, 0); + } + + // ── (b) the FAILING-PART path — must leave ZERO orphans ─────────────────────────────────── + for (const [label, build] of [ + ['stitchapi', multipartUploader], + ['hand-rolled', handRolledUploader], + ] as const) { + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + const { upload, lastStats } = build({ + adapter: api.adapter(), + urlTemplate: FakeS3.template, + concurrency: 2, + attempts: 2, + }); + let threw = ''; + try { + await upload('v.mp4', CHUNKS); + } catch (e) { + threw = (e as Error).message; + } + await drain(); + check(`(b) [${label}] the upload threw`, threw !== '', true); + check(`(b) [${label}] ORPHANED PARTS`, api.orphanParts, 0); + check(`(b) [${label}] ORPHANED BYTES`, api.orphanBytes, 0); + check(`(b) [${label}] dangling UploadIds`, api.danglingUploads, 0); + check(`(b) [${label}] DELETEs accepted`, api.aborted, 1); + check( + `(b) [${label}] cleanedUp reported`, + lastStats()?.cleanedUp, + true, + ); + check( + `(b) [${label}] initiates (no whole-upload retry)`, + api.initiated, + 1, + ); + check( + `(b) [${label}] part PUTs (part 3 retried once)`, + api.partPuts, + 5, + ); + } + + // ── (c) a CANCELLED run — the case C4(e) showed leaks by default ────────────────────────── + for (const [label, build] of [ + ['stitchapi', multipartUploader], + ['hand-rolled', handRolledUploader], + ] as const) { + const api = new FakeS3({ partTicks: { 1: 0, 2: 0, 3: 60, 4: 60 } }); + const { upload } = build({ + adapter: api.adapter(), + urlTemplate: FakeS3.template, + concurrency: 4, + }); + const ac = new AbortController(); + const pending = upload('v.mp4', CHUNKS, { signal: ac.signal }); + // Spin until exactly two parts have landed — a fixed microtask count is not portable + // across the two implementations (the stitch path has more awaits before dispatch). + for (let i = 0; i < 400 && api.completionOrder.length < 2; i++) + await Promise.resolve(); + ac.abort(); + let threw = ''; + try { + await pending; + } catch (e) { + threw = (e as Error).message; + } + await drain(); + check(`(c) [${label}] the upload threw`, threw !== '', true); + check( + `(c) [${label}] parts that had landed`, + api.completionOrder.length, + 2, + ); + check(`(c) [${label}] ORPHANED PARTS`, api.orphanParts, 0); + check(`(c) [${label}] DELETEs accepted`, api.aborted, 1); + } + + // ── (d) the cleanup itself fails — it must be LOUD ──────────────────────────────────────── + // Same run as (b), but the abort endpoint is broken. C4(h2) measured the silent version of this + // (`.safe()` swallowing the error, 3 parts still billing, nothing thrown). Here it reports. + { + const api = new FakeS3(); + api.failPart(3, Number.POSITIVE_INFINITY, 500); + // A transport that answers every DELETE with a 500 — the abort endpoint is down. + const broken: Adapter = async (req) => + req.method.toUpperCase() === 'DELETE' + ? { status: 500, headers: {}, body: { Code: 'InternalError' } } + : api.adapter()(req); + const reported: string[] = []; + const { upload, lastStats } = multipartUploader({ + adapter: broken, + urlTemplate: FakeS3.template, + attempts: 1, + onCleanupFailure: (id, reason) => reported.push(`${id}:${reason}`), + }); + try { + await upload('v.mp4', CHUNKS); + } catch { + /* the part failure */ + } + await drain(); + check('(d) cleanup failures reported', reported.length, 1); + check( + '(d) …naming the UploadId', + reported[0]?.startsWith('upl-'), + true, + ); + check('(d) cleanedUp reported', lastStats()?.cleanedUp, false); + check('(d) ORPHANED PARTS (correctly non-zero)', api.orphanParts, 3); + check('(d) ORPHANED BYTES', api.orphanBytes, 15 * MiB); + + // …and with no handler at all, it THROWS rather than resolving quietly. + const api2 = new FakeS3(); + api2.failPart(3, Number.POSITIVE_INFINITY, 500); + const broken2: Adapter = async (req) => + req.method.toUpperCase() === 'DELETE' + ? { status: 500, headers: {}, body: {} } + : api2.adapter()(req); + const u2 = multipartUploader({ + adapter: broken2, + urlTemplate: FakeS3.template, + attempts: 1, + }); + let msg = ''; + try { + await u2.upload('v.mp4', CHUNKS); + } catch (e) { + msg = (e as Error).message; + } + check( + '(d) no handler → the cleanup failure is the error the caller sees', + msg.includes('cleanup FAILED'), + true, + ); + note( + '(d) → this is the one place `.safe()` must NOT be the end of the story', + 'a swallowed cleanup error is an upload that bills forever while the code reads as correct', + ); + } + + // ── (e) why NOT one `Surface.execute` stitch: the cleanup runs after the caller returns ─── + { + const clock = manualClock(); + const api = new FakeS3({ hangParts: [3] }); + const server = api.adapter(); + // A DELETE is a network round trip, so give it a few turns. Without this the race is too + // tight to READ; with it, the ordering the engine actually produces is unambiguous. + const transport: Adapter = async (req) => { + if (req.method.toUpperCase() === 'DELETE') + for (let i = 0; i < 20; i++) await Promise.resolve(); + return server(req); + }; + // The whole upload as one stitch, with the try/finally INSIDE `execute` — the construction + // that looks like it puts cleanup under the engine's control. + const uploadSurface: Surface = { + id: 'multipart-upload', + execute: async (req) => { + const key = new URL(req.url).pathname.replace('/bucket/', ''); + const init = await transport({ + url: `${FakeS3.url(key)}?uploads`, + method: 'POST', + headers: {}, + }); + const uploadId = (init.body as { UploadId: string }).UploadId; + try { + const parts = await Promise.all( + [1, 2, 3, 4].map(async (n) => { + const r = await transport({ + url: `${FakeS3.url(key)}?partNumber=${n}&uploadId=${uploadId}`, + method: 'PUT', + headers: {}, + body: { chunk: `c${n}` }, + ...(req.signal ? { signal: req.signal } : {}), + }); + if (r.status >= 400) throw new Error(`part ${n}`); + return { PartNumber: n, ETag: r.headers['etag'] }; + }), + ); + return await transport({ + url: `${FakeS3.url(key)}?uploadId=${uploadId}`, + method: 'POST', + headers: {}, + body: { Parts: parts }, + }); + } catch (e) { + await transport({ + url: `${FakeS3.url(key)}?uploadId=${uploadId}`, + method: 'DELETE', + headers: {}, + }); + throw e; + } + }, + }; + const one = stitch({ + url: FakeS3.template, + method: 'POST', + kind: uploadSurface, + adapter: transport, + clock, + timeout: { total: 1_000 }, + }); + // Snapshot the bucket at the INSTANT the caller's promise settles — not after, because + // "after" is exactly the window this case is about. + let orphansWhenCallerReturned = -1; + let abortsWhenCallerReturned = -1; + const pending = one.safe({ params: { key: 'v.mp4' } }); + void pending.then(() => { + orphansWhenCallerReturned = api.orphanParts; + abortsWhenCallerReturned = api.aborted; + }); + await clock.advance(5_000); + const r = await pending; + check('(e) the caller saw a failure', r.ok, false); + check( + '(e) …the failure was the timeout', + r.error?.message.includes('timed out'), + true, + ); + check( + '(e) ORPHANS at the moment the caller returned', + orphansWhenCallerReturned, + 3, + ); + check('(e) DELETEs at that moment', abortsWhenCallerReturned, 0); + await drain(); + check('(e) ORPHANS several turns later', api.orphanParts, 0); + check('(e) DELETEs several turns later', api.aborted, 1); + note( + '(e) → `withTimeout` rejects the caller and lets `execute` keep running', + 'the cleanup is real but LATE — in a lambda or a process that exits on the error, it never lands', + ); + } + + // ── (f) what the config bought, and what it cost ────────────────────────────────────────── + heading(' the line count'); + { + const mine = executableLines('multipart.ts'); + const theirs = executableLines('hand-rolled.ts'); + note( + ' user code (`multipart.ts`)', + `${String(mine)} executable lines`, + ); + note( + ' hand-rolled (`hand-rolled.ts`)', + `${String(theirs)} executable lines`, + ); + check(' StitchAPI is shorter', mine < theirs, true); + note( + ' what the difference is', + 'the retry loop + backoff, the FIFO concurrency pool, the status classification and the URL assembly — all config on one side, ~45 lines on the other', + ); + note( + ' what is IDENTICAL on both sides', + 'the try/finally, the loud-cleanup rule, the per-part high-water progress map, and the Promise.all ordering — none of it is library-supplied', + ); + } + + finish( + 'C8', + 'ACHIEVABLE WITH USER CODE, and the user code is exactly the compensation. Both implementations pass all four shapes against the same fake bucket. SUCCESS: 4-part object, 1 object stored, 0 orphans, 0 DELETEs, peak in-flight 2, server storage order [2,1,4,3] ≠ part order. FAILING PART: threw, **0 orphaned parts / 0 bytes / 0 dangling UploadIds / 1 DELETE accepted**, 1 initiate (no whole-upload retry) and 5 part PUTs (only part 3 re-sent). CANCELLED MID-FLIGHT: 2 parts had landed, **0 orphans, 1 DELETE**. PROGRESS (`xhrAdapter`): 16 monotonic ticks, [7,14,19,…,94,100], ending at exactly 100% with 0 orphans. BROKEN ABORT ENDPOINT: 1 cleanup failure reported naming the UploadId, `cleanedUp:false`, orphans correctly reported as 3/15 MiB — and with no handler wired the cleanup failure becomes the error the caller sees. LINE COUNT: **141 vs 163 executable lines** — StitchAPI is 22 lines shorter, and the difference attributes exactly to the retry loop with backoff, the FIFO concurrency pool, the retryable-status set and the URL assembly, all of which became config. What did NOT shrink is the part that matters: the `try/finally`, the loud-cleanup rule, the per-part high-water progress map and the input-order assembly are byte-for-byte the same on both sides. And the tempting alternative — the whole upload as ONE `Surface.execute` stitch so the engine owns the lifecycle — measured **3 orphans and 0 DELETEs at the instant the caller saw the timeout**, cleaning up only several turns later (resilience.ts:230-244 rejects the caller and lets `fn` run on)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/multipart-upload/fake-s3.ts b/docs/scenarios/proofs/multipart-upload/fake-s3.ts new file mode 100644 index 00000000..56e99f92 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/fake-s3.ts @@ -0,0 +1,541 @@ +// A fake, in-memory S3-shaped MULTIPART UPLOAD provider. Four endpoints, no network: +// +// POST /bucket/{key}?uploads → 200 `{ UploadId }` +// PUT /bucket/{key}?partNumber=N&uploadId=… → 200, **`ETag` RESPONSE HEADER**, no body +// POST /bucket/{key}?uploadId=… `{ Parts: [...] }` → 200 the assembled object +// DELETE /bucket/{key}?uploadId=… → 204 abort, parts discarded +// +// Three things about it are deliberate, because they are what turns this scenario's questions into +// measurements rather than arguments: +// +// 1. **The part result is a RESPONSE HEADER.** `PUT` answers with `etag: '"p3-…"'` and a body of +// `undefined` — exactly like S3, which puts nothing useful in a part's body. A client that can +// only see `res.body` cannot complete the upload at all. +// 2. **`complete` REJECTS a wrong list.** Out-of-order parts → `400 InvalidPartOrder`; a missing +// part → `400 InvalidPart`; a wrong ETag → `400 InvalidPart`. So "the ETags were assembled in +// part order" is something the server checks, not something the proof assumes. +// 3. **Orphans are counted.** A part stored under an UploadId that was never completed and never +// aborted is exactly what AWS bills for and hides from `aws s3 ls`. `orphanParts` / +// `orphanBytes` / `danglingUploads` are the numbers every C4/C5/C6 verdict cites. +// +// Completion order is controllable and RECORDED. `partTicks` delays a part's response by N +// microtask turns, so a test can make part 3 land before part 1 — and `completionOrder` reports the +// order the server actually stored them in, so C2's "part order, not completion order" is measured +// end to end rather than assumed. Ticks, not timers: the concurrency limiter arms no timer +// (resilience.ts:118-127), so a bounded fan needs no clock advancing and stays deterministic. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +const HOST = 'https://s3.example'; +const BUCKET = 'bucket'; + +/** One part as the server stored it. */ +export interface StoredPart { + partNumber: number; + etag: string; + bytes: number; +} + +/** One multipart upload's server-side state. `open` is the billing hazard. */ +export interface UploadState { + uploadId: string; + key: string; + /** Stored parts, keyed by part number. Discarded by `complete` and by `abort`. */ + parts: Map; + /** `open` = neither completed nor aborted = **orphaned**, and billed forever. */ + status: 'open' | 'completed' | 'aborted'; +} + +/** One recorded request, as the server saw it. */ +export interface RecordedHit { + method: string; + /** `initiate` | `part` | `complete` | `abort` | `unknown`. */ + op: string; + key: string; + uploadId: string; + /** Part number on a `part` hit, else 0. */ + partNumber: number; + status: number; +} + +/** The assembled object `complete` returns. */ +export interface CompletedObject { + Location: string; + Bucket: string; + Key: string; + ETag: string; + Parts: number; + Bytes: number; +} + +export interface FakeS3Options { + /** + * Microtask turns to stall each part's response by, keyed by part number. Default 0. This is + * how a test makes COMPLETION order differ from PART order: `{ 3: 0, 1: 30, 2: 40, 4: 50 }` + * lands part 3 first. Whatever it produces, `completionOrder` records what actually happened. + */ + partTicks?: Record; + /** Bytes per part reported in progress + the assembled size. Default 5 MiB (S3's floor). */ + partBytes?: number; + /** + * Part numbers whose PUT never answers — a stalled socket. The request resolves only when its + * `signal` aborts (which is what an engine `timeout` does), so a timeout is a reachable failure + * mode and not just a hypothetical. + */ + hangParts?: number[]; +} + +/** How a part should fail, and how many times before it starts working. */ +interface PartFailure { + status: number; + /** Remaining attempts to fail. `Infinity` = always. */ + times: number; +} + +const PART_BYTES = 5 * 1024 * 1024; + +/** Wait N microtask turns — deterministic ordering with no timers and no clock. */ +const ticks = async (n: number): Promise => { + for (let i = 0; i < n; i++) await Promise.resolve(); +}; + +/** + * The multipart provider. One instance is one bucket. Every counter on it is a measurement some + * claim cites; the one that decides the scenario is {@link FakeS3.orphanParts}. + */ +export class FakeS3 { + /** Every hit, in order. */ + readonly hits: RecordedHit[] = []; + /** Part numbers in the order the server STORED them — completion order, not part order. */ + readonly completionOrder: number[] = []; + /** Peak simultaneous in-flight part PUTs. The number C3 is about. */ + peakInFlight = 0; + /** In-flight part PUTs right now. */ + private inFlight = 0; + + private readonly uploads = new Map(); + private readonly objects = new Map(); + private readonly failures = new Map(); + private readonly partTicks: Record; + private readonly partBytes: number; + private readonly hangParts: Set; + private nextUploadId = 0; + + constructor(opts: FakeS3Options = {}) { + this.partTicks = opts.partTicks ?? {}; + this.partBytes = opts.partBytes ?? PART_BYTES; + this.hangParts = new Set(opts.hangParts ?? []); + } + + // ---- the URLs a stitch is pointed at ------------------------------------------------------ + + /** `https://s3.example/bucket/{key}` — the one path all four operations share. */ + static url(key: string): string { + return `${HOST}/${BUCKET}/${key}`; + } + /** RFC 6570 template form, for a stitch that takes the key as a `params` value. */ + static readonly template = `${HOST}/${BUCKET}/{key}`; + + // ---- the measurements --------------------------------------------------------------------- + + /** + * **Parts left in the bucket under an UploadId that was never completed or aborted.** The AWS + * bill line nobody sees. Zero is the only acceptable value after a failed upload. + */ + get orphanParts(): number { + let n = 0; + for (const u of this.uploads.values()) + if (u.status === 'open') n += u.parts.size; + return n; + } + /** The same thing in bytes — what the invisible storage actually costs. */ + get orphanBytes(): number { + let n = 0; + for (const u of this.uploads.values()) + if (u.status === 'open') + for (const p of u.parts.values()) n += p.bytes; + return n; + } + /** UploadIds still `open` — each one an upload the client walked away from. */ + get danglingUploads(): number { + return [...this.uploads.values()].filter((u) => u.status === 'open') + .length; + } + /** `POST ?uploads` count. More than one per logical upload means a re-initiate. */ + get initiated(): number { + return this.hits.filter((h) => h.op === 'initiate').length; + } + /** `DELETE ?uploadId` count — how many times cleanup actually ran. */ + get aborted(): number { + return this.hits.filter((h) => h.op === 'abort' && h.status < 400) + .length; + } + /** Successful `POST ?uploadId` count. */ + get completed(): number { + return this.hits.filter((h) => h.op === 'complete' && h.status < 400) + .length; + } + /** Every part PUT that reached the server, successful or not. C5's "did it re-send?" number. */ + get partPuts(): number { + return this.hits.filter((h) => h.op === 'part').length; + } + /** Part numbers of every part PUT, in arrival order. */ + get partPutOrder(): number[] { + return this.hits + .filter((h) => h.op === 'part') + .map((h) => h.partNumber); + } + /** The ops that reached the server, in order — the request spine. */ + get ops(): string[] { + return this.hits.map((h) => h.op); + } + /** Objects that actually exist in the bucket. */ + get objectCount(): number { + return this.objects.size; + } + /** Look up an assembled object. */ + object(key: string): CompletedObject | undefined { + return this.objects.get(key); + } + /** Parts currently stored under one UploadId (regardless of its status). */ + storedParts(uploadId: string): number { + return this.uploads.get(uploadId)?.parts.size ?? 0; + } + /** One upload's status, or `'(unknown)'`. */ + statusOf(uploadId: string): string { + return this.uploads.get(uploadId)?.status ?? '(unknown)'; + } + /** Every UploadId the server ever minted, in order. */ + get uploadIds(): string[] { + return [...this.uploads.keys()]; + } + + // ---- failure injection -------------------------------------------------------------------- + + /** + * Make part `n` fail. `times` bounds it (`1` = fail once then succeed, the retry case); + * omitted = fail forever, the case that forces cleanup. + */ + failPart(n: number, times = Number.POSITIVE_INFINITY, status = 500): void { + this.failures.set(n, { status, times }); + } + + // ---- the transport ------------------------------------------------------------------------ + + /** A StitchAPI {@link Adapter} bound to this bucket. */ + adapter(): Adapter { + return (req: AdapterRequest): Promise => + this.handle(req); + } + + /** + * A `fetch`-shaped entry point onto the same server, so C8's hand-rolled twin and the StitchAPI + * implementation run over ONE transport contract rather than either getting a shortcut. + */ + fetchImpl(): typeof fetch { + return async (input, init) => { + const url = typeof input === 'string' ? input : String(input); + const headers: Record = {}; + new Headers(init?.headers).forEach((v, k) => { + headers[k] = v; + }); + const res = await this.handle({ + url, + method: init?.method ?? 'GET', + headers, + body: + typeof init?.body === 'string' + ? (JSON.parse(init.body) as unknown) + : undefined, + }); + return new Response( + res.body === undefined ? null : JSON.stringify(res.body), + { + status: res.status, + headers: { + ...res.headers, + 'content-type': 'application/json', + }, + }, + ); + }; + } + + /** The one request handler both entry points share. */ + private async handle(req: AdapterRequest): Promise { + const url = new URL(req.url); + const key = url.pathname.replace(`/${BUCKET}/`, ''); + const method = req.method.toUpperCase(); + const uploadId = url.searchParams.get('uploadId') ?? ''; + const partNumber = Number(url.searchParams.get('partNumber') ?? 0); + + if (method === 'POST' && url.searchParams.has('uploads')) + return this.initiate(key); + if (method === 'PUT' && partNumber > 0) + return this.putPart(key, uploadId, partNumber, req.signal); + if (method === 'POST' && uploadId) + return this.complete(key, uploadId, req.body); + if (method === 'DELETE' && uploadId) return this.abort(key, uploadId); + + this.hits.push({ + method, + op: 'unknown', + key, + uploadId, + partNumber, + status: 400, + }); + return { + status: 400, + headers: {}, + body: { Code: 'MalformedRequest' }, + }; + } + + private initiate(key: string): AdapterResponse { + const uploadId = `upl-${(this.nextUploadId += 1)}`; + this.uploads.set(uploadId, { + uploadId, + key, + parts: new Map(), + status: 'open', + }); + this.hits.push({ + method: 'POST', + op: 'initiate', + key, + uploadId, + partNumber: 0, + status: 200, + }); + return { + status: 200, + headers: {}, + body: { UploadId: uploadId, Key: key }, + }; + } + + private async putPart( + key: string, + uploadId: string, + partNumber: number, + signal: AbortSignal | undefined, + ): Promise { + this.inFlight += 1; + this.peakInFlight = Math.max(this.peakInFlight, this.inFlight); + try { + await ticks(this.partTicks[partNumber] ?? 0); + + // A stalled socket: answer only when someone gives up on us. `withTimeout` + // (engine.ts:668-673) aborts the attempt signal, which is what lands here. + if (this.hangParts.has(partNumber)) { + await new Promise((resolve) => { + if (signal) + signal.addEventListener('abort', () => resolve(), { + once: true, + }); + }); + this.hits.push({ + method: 'PUT', + op: 'part', + key, + uploadId, + partNumber, + status: 499, + }); + throw new Error('stalled'); + } + + // A cancelled sibling never reaches the store — that is what makes C6's + // "already-landed parts vs cancelled ones" a real distinction. + if (signal?.aborted) { + this.hits.push({ + method: 'PUT', + op: 'part', + key, + uploadId, + partNumber, + status: 499, + }); + throw new Error('aborted'); + } + + const fail = this.failures.get(partNumber); + if (fail && fail.times > 0) { + fail.times -= 1; + this.hits.push({ + method: 'PUT', + op: 'part', + key, + uploadId, + partNumber, + status: fail.status, + }); + return { + status: fail.status, + headers: {}, + body: { Code: 'InternalError' }, + }; + } + + const upload = this.uploads.get(uploadId); + if (!upload || upload.status !== 'open') { + this.hits.push({ + method: 'PUT', + op: 'part', + key, + uploadId, + partNumber, + status: 404, + }); + return { + status: 404, + headers: {}, + body: { Code: 'NoSuchUpload' }, + }; + } + + const etag = `"${uploadId}-p${partNumber}"`; + upload.parts.set(partNumber, { + partNumber, + etag, + bytes: this.partBytes, + }); + this.completionOrder.push(partNumber); + this.hits.push({ + method: 'PUT', + op: 'part', + key, + uploadId, + partNumber, + status: 200, + }); + // The whole point: the result is a HEADER. The body carries nothing. + return { status: 200, headers: { etag }, body: undefined }; + } finally { + this.inFlight -= 1; + } + } + + private complete( + key: string, + uploadId: string, + body: unknown, + ): AdapterResponse { + const upload = this.uploads.get(uploadId); + const record = (status: number): void => { + this.hits.push({ + method: 'POST', + op: 'complete', + key, + uploadId, + partNumber: 0, + status, + }); + }; + if (!upload || upload.status !== 'open') { + record(404); + return { + status: 404, + headers: {}, + body: { Code: 'NoSuchUpload' }, + }; + } + + const listed = (body as { Parts?: unknown })?.Parts; + const parts = Array.isArray(listed) + ? (listed as { PartNumber?: number; ETag?: string }[]) + : []; + + // STRICT, like S3: ascending order, no gaps, every stored part present, ETags byte-exact. + const ascending = parts.every( + (p, i) => + i === 0 || + (parts[i - 1]?.PartNumber ?? 0) < (p.PartNumber ?? 0), + ); + if (!ascending) { + record(400); + return { + status: 400, + headers: {}, + body: { + Code: 'InvalidPartOrder', + Message: + 'parts must be listed in ascending PartNumber order', + }, + }; + } + if (parts.length !== upload.parts.size) { + record(400); + return { + status: 400, + headers: {}, + body: { + Code: 'InvalidPart', + Message: `listed ${parts.length} parts, ${upload.parts.size} were uploaded`, + }, + }; + } + for (const p of parts) { + const stored = upload.parts.get(p.PartNumber ?? -1); + if (!stored || stored.etag !== p.ETag) { + record(400); + return { + status: 400, + headers: {}, + body: { + Code: 'InvalidPart', + Message: `part ${String(p.PartNumber)} has no matching stored ETag`, + }, + }; + } + } + + const object: CompletedObject = { + Location: FakeS3.url(key), + Bucket: BUCKET, + Key: key, + ETag: `"${uploadId}-${parts.length}"`, + Parts: parts.length, + Bytes: [...upload.parts.values()].reduce((n, p) => n + p.bytes, 0), + }; + this.objects.set(key, object); + upload.status = 'completed'; + upload.parts.clear(); // assembled — no longer billed as parts + record(200); + return { status: 200, headers: {}, body: object }; + } + + private abort(key: string, uploadId: string): AdapterResponse { + const upload = this.uploads.get(uploadId); + if (!upload || upload.status !== 'open') { + this.hits.push({ + method: 'DELETE', + op: 'abort', + key, + uploadId, + partNumber: 0, + status: 404, + }); + return { + status: 404, + headers: {}, + body: { Code: 'NoSuchUpload' }, + }; + } + upload.status = 'aborted'; + upload.parts.clear(); // the bill stops here + this.hits.push({ + method: 'DELETE', + op: 'abort', + key, + uploadId, + partNumber: 0, + status: 204, + }); + return { status: 204, headers: {}, body: undefined }; + } +} diff --git a/docs/scenarios/proofs/multipart-upload/fake-xhr.ts b/docs/scenarios/proofs/multipart-upload/fake-xhr.ts new file mode 100644 index 00000000..22493261 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/fake-xhr.ts @@ -0,0 +1,128 @@ +// A fake `XMLHttpRequest` that satisfies core's `XhrLike` structurally, so `xhrAdapter(FakeXhr)` +// runs off-browser. This is not a workaround: `xhrAdapter` takes an optional constructor for +// exactly this reason (xhr-adapter.ts:49-68), the same dependency-injection seam `axiosAdapter` +// uses for its client. +// +// It does the one thing `fetch` cannot: fire `upload.onprogress` with bytes SENT, before the +// response exists. The tick schedule is deterministic (`uploadTicks` evenly spaced fractions of the +// encoded body length), so C1 and C7 assert an exact sequence rather than "some ticks happened". +// +// The response itself is produced by the real {@link FakeS3} handler, so the xhr path and the fetch +// path hit one server — a difference between them is a TRANSPORT difference, which is the whole +// question C1 asks. +import type { Adapter } from '../../../../packages/core/src/types'; +import type { + XhrLike, + XhrProgress, +} from '../../../../packages/core/src/xhr-adapter'; + +export interface FakeXhrOptions { + /** How many `upload.onprogress` ticks to fire per request. Default 4. */ + uploadTicks?: number; + /** Fire one `onprogress` (download) tick as the response lands. Default true. */ + downloadTick?: boolean; + /** Report `lengthComputable: false` — the chunked-body case, where `total` is unknown. */ + unknownLength?: boolean; +} + +/** + * Build an `XhrLikeCtor` bound to a transport. `xhrAdapter` constructs it with `new` and no + * arguments, so the server has to be captured in a closure — hence the factory. + */ +export function fakeXhrCtor( + transport: Adapter, + opts: FakeXhrOptions = {}, +): new () => XhrLike { + const uploadTicks = opts.uploadTicks ?? 4; + const downloadTick = opts.downloadTick ?? true; + const unknownLength = opts.unknownLength ?? false; + + return class FakeXhr implements XhrLike { + responseType = ''; + status = 0; + response: unknown = null; + readonly upload: { onprogress: ((e: XhrProgress) => void) | null } = { + onprogress: null, + }; + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + onabort: (() => void) | null = null; + onprogress: ((e: XhrProgress) => void) | null = null; + + private method = 'GET'; + private url = ''; + private readonly reqHeaders: Record = {}; + private resHeaders: Record = {}; + private aborted = false; + + open(method: string, url: string): void { + this.method = method; + this.url = url; + } + setRequestHeader(name: string, value: string): void { + this.reqHeaders[name] = value; + } + getAllResponseHeaders(): string { + return Object.entries(this.resHeaders) + .map(([k, v]) => `${k}: ${v}`) + .join('\r\n'); + } + abort(): void { + this.aborted = true; + this.onabort?.(); + } + + send(body: string | FormData | null): void { + const total = typeof body === 'string' ? body.length : 0; + void (async () => { + // Bytes SENT, before any response exists. `fetch` has no equivalent. + const emit = this.upload.onprogress; + if (emit && total > 0) { + for (let i = 1; i <= uploadTicks; i++) { + if (this.aborted) return; + await Promise.resolve(); + const loaded = Math.round((total * i) / uploadTicks); + emit( + unknownLength + ? { lengthComputable: false, loaded, total: 0 } + : { lengthComputable: true, loaded, total }, + ); + } + } + + let res; + try { + res = await transport({ + url: this.url, + method: this.method, + headers: this.reqHeaders, + body: + typeof body === 'string' + ? (JSON.parse(body) as unknown) + : undefined, + }); + } catch { + this.onerror?.(); + return; + } + if (this.aborted) return; + + this.status = res.status; + this.resHeaders = { + ...res.headers, + 'content-type': 'application/json', + }; + const text = + res.body === undefined ? '' : JSON.stringify(res.body); + this.response = new TextEncoder().encode(text).buffer; + if (downloadTick && this.onprogress) + this.onprogress({ + lengthComputable: true, + loaded: text.length, + total: text.length, + }); + this.onload?.(); + })(); + } + }; +} diff --git a/docs/scenarios/proofs/multipart-upload/hand-rolled.ts b/docs/scenarios/proofs/multipart-upload/hand-rolled.ts new file mode 100644 index 00000000..cdfa22f8 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/hand-rolled.ts @@ -0,0 +1,182 @@ +// The same uploader with NO StitchAPI in it — the honest comparison for C8's line count. +// +// Feature-matched to `multipart.ts` line for line of BEHAVIOUR, so the difference is attributable: +// four endpoints, a bounded pool, per-part retry on `[429,500,502,503,504]` with exponential +// backoff, ETag lifted from the response header, `Promise.all` input ordering, per-part high-water +// progress aggregation, a `try/finally` abort, and the same loud-cleanup-failure rule. +// +// It drives the SAME transport the StitchAPI side does (an `Adapter`-shaped function), so neither +// implementation gets a shortcut on the wire. +import type { Adapter } from '../../../../packages/core/src/types'; +import type { UploadOptions, UploadStats, UploaderOptions } from './multipart'; + +const RETRYABLE = new Set([429, 500, 502, 503, 504]); + +/** A minimal FIFO concurrency pool — what `throttle: { concurrency }` is on the other side. */ +function pool(limit: number): (fn: () => Promise) => Promise { + let inFlight = 0; + const waiters: (() => void)[] = []; + const release = (): void => { + inFlight -= 1; + waiters.shift()?.(); + }; + return async (fn: () => Promise): Promise => { + if (inFlight >= limit) + await new Promise((r) => { + waiters.push(() => { + inFlight += 1; + r(); + }); + }); + else inFlight += 1; + try { + return await fn(); + } finally { + release(); + } + }; +} + +const sleep = (ms: number): Promise => + new Promise((r) => setTimeout(r, ms)); + +export function handRolledUploader(opts: UploaderOptions): { + upload: ( + key: string, + chunks: readonly unknown[], + options?: UploadOptions, + ) => Promise; + lastStats: () => UploadStats | undefined; +} { + const send: Adapter = opts.adapter; + const url = (key: string, qs: string): string => + `${opts.urlTemplate.replace('{key}', key)}?${qs}`; + const take = pool(opts.concurrency ?? 4); + const attempts = opts.attempts ?? 3; + let stats: UploadStats | undefined; + + async function putPart( + key: string, + uploadId: string, + n: number, + chunk: unknown, + onTick: (loaded: number, total: number | undefined) => void, + signal: AbortSignal | undefined, + ): Promise { + for (let attempt = 1; ; attempt++) { + const res = await send({ + url: url(key, `partNumber=${n}&uploadId=${uploadId}`), + method: 'PUT', + headers: {}, + body: chunk, + onProgress: (p) => { + if (p.direction === 'upload') onTick(p.loaded, p.total); + }, + ...(signal ? { signal } : {}), + }); + if (res.status < 400) { + const etag = res.headers['etag']; + if (!etag) throw new Error(`part ${n}: no ETag header`); + return etag; + } + if (!RETRYABLE.has(res.status) || attempt >= attempts) + throw new Error(`part ${n}: HTTP ${res.status}`); + await sleep(2 ** (attempt - 1) * 100); + } + } + + async function upload( + key: string, + chunks: readonly unknown[], + options: UploadOptions = {}, + ): Promise { + const { onProgress, signal } = options; + const init = await send({ + url: url(key, 'uploads'), + method: 'POST', + headers: {}, + ...(signal ? { signal } : {}), + }); + if (init.status >= 400) + throw new Error(`initiate: HTTP ${init.status}`); + const uploadId = (init.body as { UploadId: string }).UploadId; + const sent = new Map(); + let total = 0; + let settled = false; + stats = { uploadId, parts: 0, cleanedUp: false, sent: 0 }; + try { + const parts = await Promise.all( + chunks.map(async (chunk, i) => { + const PartNumber = i + 1; + const ETag = await take(() => + putPart( + key, + uploadId, + PartNumber, + chunk, + (loaded, t) => { + if (t !== undefined) + total = Math.max(total, t * chunks.length); + sent.set( + PartNumber, + Math.max(sent.get(PartNumber) ?? 0, loaded), + ); + let done = 0; + for (const v of sent.values()) done += v; + onProgress?.( + total ? done / total : 0, + done, + total, + ); + }, + signal, + ), + ); + return { PartNumber, ETag }; + }), + ); + stats.parts = parts.length; + const res = await send({ + url: url(key, `uploadId=${uploadId}`), + method: 'POST', + headers: {}, + body: { Parts: parts }, + ...(signal ? { signal } : {}), + }); + if (res.status >= 400) + throw new Error(`complete: HTTP ${res.status}`); + settled = true; + return res.body; + } finally { + let done = 0; + for (const v of sent.values()) done += v; + stats.sent = done; + if (!settled) { + let ok = false; + let reason = 'unknown'; + try { + const res = await send({ + url: url(key, `uploadId=${uploadId}`), + method: 'DELETE', + headers: {}, + }); + ok = res.status < 400; + if (!ok) reason = `HTTP ${res.status}`; + } catch (e) { + reason = (e as Error).message; + } + stats.cleanedUp = ok; + if (!ok) { + if (opts.onCleanupFailure) + opts.onCleanupFailure(uploadId, reason); + else + throw new Error( + `multipart cleanup FAILED for ${uploadId} (${reason}) — parts are still billing`, + ); + } + } + } + } + + return { upload, lastStats: () => stats }; +} diff --git a/docs/scenarios/proofs/multipart-upload/harness.ts b/docs/scenarios/proofs/multipart-upload/harness.ts new file mode 100644 index 00000000..edf0d6f4 --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/harness.ts @@ -0,0 +1,64 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is COUNTS and ORDERS: how many parts are still sitting in the bucket +// under a dangling UploadId, what the peak in-flight request count actually reached, and what +// order the ETags came back in versus what order they were sent in. So both assertions print the +// measured value whether they pass or fail — `orphaned parts: 3` is the finding, and it has to be +// readable out of context. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the validator spine + * (`["(none)", "\"v1\"", "\"v1\""]`) and the status spine (`[200, 304, 304]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/multipart-upload/multipart.ts b/docs/scenarios/proofs/multipart-upload/multipart.ts new file mode 100644 index 00000000..5afc22af --- /dev/null +++ b/docs/scenarios/proofs/multipart-upload/multipart.ts @@ -0,0 +1,198 @@ +// The assembled answer — USER CODE, and the subject of C8's line count. +// +// Four stitches and one `try/finally`. What is CONFIG here (and therefore not written out below): +// the per-part retry policy and its widened `on` set, the concurrency bound, the per-attempt +// timeout, URL templating, the trace spine, and the ETag lift (`Surface.interpret`). What is CODE +// here, because nothing in the library does it: +// +// - **the `try/finally` that aborts.** C4 established there is no compensation seam of any kind, +// so this is the whole of the cleanup story. It lives at the ORCHESTRATION level because that is +// the only scope that holds the `uploadId` (C6(e)). +// - **the loud failure when the cleanup itself fails.** `abort.safe()` cannot throw, which is +// exactly what makes it dangerous here: a swallowed cleanup error is an upload that bills +// forever while the code reads as correct (C4(h2)). `onCleanupFailure` is mandatory-by-default: +// omitted, it throws. +// - **the per-part progress bookkeeping.** A tick carries no part identity (C7(b)), so the part +// number is bound at the call site and the aggregate is a per-part HIGH-WATER mark — a running +// `+=` double-counts a retried part (C7(d)). +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + Adapter, + Clock, + Stitch, + TraceSink, +} from '../../../../packages/core/src/types'; + +/** The part surface: a part's value is its `ETag` RESPONSE HEADER (C2). */ +const partSurface: Surface = { + // `verdictOf` FIRST — `interpret` replaces the default verdict, so without it an HTTP 500 part + // becomes `ok: true` carrying `undefined` and the upload fails two calls later (C2(e)). + id: 'multipart-part', + interpret: (res, cfg) => + verdictOf(res, cfg) ?? { ok: true, data: res.headers['etag'] }, +}; + +export interface UploaderOptions { + /** Transport. `xhrAdapter(...)` if you want upload progress; `fetchAdapter(...)` otherwise (C1). */ + adapter: Adapter; + /** RFC 6570 template for the object, e.g. `https://s3.example/bucket/{key}`. */ + urlTemplate: string; + /** Max simultaneous part PUTs. The bound lives on the part stitch (C3(d)). */ + concurrency?: number; + /** Attempts per PART (never per upload — a whole-upload retry re-initiates, C5(d)). */ + attempts?: number; + /** + * Called when the compensating DELETE did not succeed. There is no safe default: a swallowed + * cleanup failure is an invisible, permanent storage bill. Omitted ⇒ throws. + */ + onCleanupFailure?: (uploadId: string, reason: string) => void; + clock?: Clock; + trace?: TraceSink; +} + +/** Reported after every run, successful or not — the numbers an operator needs. */ +export interface UploadStats { + uploadId: string; + /** Parts whose ETag the client holds. */ + parts: number; + /** Did the compensating DELETE run AND succeed? */ + cleanedUp: boolean; + /** Bytes sent, aggregated across parts (per-part high-water — C7(c)). */ + sent: number; +} + +export interface UploadOptions { + /** One aggregated 0-1 fraction for a UI. Fires only on a transport that reports upload bytes. */ + onProgress?: (fraction: number, sent: number, total: number) => void; + signal?: AbortSignal; +} + +/** + * Build an uploader bound to one bucket + transport. The returned function performs a complete + * S3-style multipart upload and leaves **zero** orphaned parts on every exit path. + */ +export function multipartUploader(opts: UploaderOptions): { + upload: ( + key: string, + chunks: readonly unknown[], + options?: UploadOptions, + ) => Promise; + lastStats: () => UploadStats | undefined; +} { + const shared = { + adapter: opts.adapter, + ...(opts.clock ? { clock: opts.clock } : {}), + ...(opts.trace ? { trace: opts.trace } : {}), + }; + const initiate = stitch({ + ...shared, + name: 'multipart.initiate', + url: `${opts.urlTemplate}?uploads`, + method: 'POST', + pick: 'UploadId', + }) as Stitch; + const part = stitch({ + ...shared, + name: 'multipart.part', + url: opts.urlTemplate, + method: 'PUT', + kind: partSurface, + // 500 is S3's own transient error and is NOT in the default `[429,502,503,504]` (C5(b)). + retry: { attempts: opts.attempts ?? 3, on: [429, 500, 502, 503, 504] }, + throttle: { concurrency: opts.concurrency ?? 4 }, + }) as Stitch; + const complete = stitch({ + ...shared, + name: 'multipart.complete', + url: opts.urlTemplate, + method: 'POST', + }); + const abort = stitch({ + ...shared, + name: 'multipart.abort', + url: opts.urlTemplate, + method: 'DELETE', + }); + + let stats: UploadStats | undefined; + + async function upload( + key: string, + chunks: readonly unknown[], + options: UploadOptions = {}, + ): Promise { + const { onProgress, signal } = options; + const uploadId = await initiate({ + params: { key }, + ...(signal ? { signal } : {}), + }); + const sent = new Map(); + let total = 0; + let settled = false; + stats = { uploadId, parts: 0, cleanedUp: false, sent: 0 }; + try { + const parts = await Promise.all( + chunks.map(async (chunk, i) => { + const PartNumber = i + 1; + const ETag = await part({ + params: { key }, + query: { partNumber: PartNumber, uploadId }, + body: chunk, + ...(signal ? { signal } : {}), + onProgress: (p) => { + if (p.direction !== 'upload') return; + if (p.total !== undefined) + total = Math.max( + total, + p.total * chunks.length, + ); + // HIGH-WATER, not `+=`: a retried part replays its ticks (C7(d)). + sent.set( + PartNumber, + Math.max(sent.get(PartNumber) ?? 0, p.loaded), + ); + let done = 0; + for (const v of sent.values()) done += v; + onProgress?.(total ? done / total : 0, done, total); + }, + }); + return { PartNumber, ETag }; + }), + ); + // `Promise.all` resolves in INPUT order regardless of completion order, which is exactly + // the order `complete` demands — no sort needed (C2(c)). + stats.parts = parts.length; + const object = await complete({ + params: { key }, + query: { uploadId }, + body: { Parts: parts }, + ...(signal ? { signal } : {}), + }); + settled = true; + return object; + } finally { + let done = 0; + for (const v of sent.values()) done += v; + stats.sent = done; + if (!settled) { + const cleanup = await abort.safe({ + params: { key }, + query: { uploadId }, + }); + stats.cleanedUp = cleanup.ok; + if (!cleanup.ok) { + const reason = cleanup.error?.message ?? 'unknown'; + if (opts.onCleanupFailure) + opts.onCleanupFailure(uploadId, reason); + else + throw new Error( + `multipart cleanup FAILED for ${uploadId} (${reason}) — parts are still billing`, + ); + } + } + } + } + + return { upload, lastStats: () => stats }; +} diff --git a/docs/scenarios/proofs/n-plus-one-fanout/README.md b/docs/scenarios/proofs/n-plus-one-fanout/README.md new file mode 100644 index 00000000..1a0a7227 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/README.md @@ -0,0 +1,230 @@ +# Proofs — one list, a hundred follow-up calls + +Runnable evidence for the claims in [`../../n-plus-one-fanout.md`](../../n-plus-one-fanout.md). + +**The deciding claim was C2 and it goes the library's way — emphatically.** `cache.coalesce` +**genuinely collapses in-flight duplicates**: 100 concurrent calls over 30 distinct customer ids +made **30 requests**, one per id, from a single `cache: { ttl }` block and no user code. Every call +was in flight simultaneously and not one response had landed, so no read-through cache could have +helped; `coalesce: false` on the same cache put it straight back to 100. 70 of the 100 callers were +served without a request of their own. This is the capability the capture hoped for and most +clients do not have, and it is on by default the moment a `cache` block exists. + +The rest of the fan-out is four decisions, and three of them are also configuration. **Bounded +concurrency works exactly** on one stitch called N times (peak 8 in-flight against a declared 8, +where the unbounded baseline is 100). **The default backoff genuinely de-clusters a herd**: 100 +calls 429ed in the same instant retried over ~98 distinct milliseconds under `expo-jitter`, against +all 100 in ONE millisecond under `expo` and `fixed`. **Partial failure** is `.safe()` and an array +index. The end-to-end answer is **45 executable lines against a hand-rolled control's 87**. + +What the capture did not anticipate, and what a docs page has to say out loud, is that **each of +those wins has a default that undoes it**: + +- A **coalesced FAILURE is not shared.** A failed leader releases its joiners to run independently + (engine.ts:1646-1659), so 100 concurrent calls for one id that 404s made **100 requests in two + waves** — and in the assembled run the deleted customer cost StitchAPI **4 requests to the + hand-rolled control's 1**. +- A vendor that sends **`Retry-After` re-clusters the herd**, because `retry.respect` defaults ON + (engine.ts:748-761): `expo-jitter` stayed configured and all 100 retries landed in one + millisecond. +- Building **one stitch per id** — the only construction `all()` can express this with — multiplies + the concurrency budget by the number of stitches (peak 100 against a declared 8), and adding a + **`store`** silently breaks the one fix (`pool: 'host'`) for it. +- Under coalescing every joiner is handed the **leader's object by reference**, so rows that share + a customer share one mutable object. + +Every script is standalone and offline. The measurement is always one of two numbers: **how many +requests reached the server**, and **the peak number open at once**. Each script prints one +`PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c2-coalesce.ts + +# all of them +for f in docs/scenarios/proofs/n-plus-one-fanout/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. The whole suite takes about twelve seconds: +every wait is on a `manualClock`, and nothing here does real I/O. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/n-plus-one-fanout/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| --------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ | +| `c1-combinators.ts` | can `all()` express N different inputs? | **No — but not for the reason the capture gives.** Runtime length is fine; 100 members + 1 input = **100 requests for ONE id** | +| `c2-coalesce.ts` | **DECIDING** — does `cache.coalesce` dedupe in-flight? | **YES. 100 calls, 30 ids, 30 requests.** And a failed leader releases 99 joiners: **100 requests for one dead id** | +| `c3-bounded-concurrency.ts` | is `throttle: { concurrency }` a real bound? | **On one stitch, exactly** (peak 8/8). On 100 stitches, **peak 100**. A `store` un-fixes `pool: 'host'` | +| `c4-partial-failure.ts` | does one 404 cost the other 99? | **Only without `.safe()`.** Bare `Promise.all`: 0 rows kept, 100 requests spent. `.safe()`: 99 rows, failure at index 49 | +| `c5-thundering-herd.ts` | does `expo-jitter` de-cluster a 429ed burst? | **Yes: ~98 distinct ms vs 1 for `expo`/`fixed`.** `Retry-After` puts all 100 back in one ms | +| `c6-trace.ts` | one tree or a hundred roots? does `linked` help? | **101 roots by default; 1 tree with `linked`** — drawn as a **101-deep chain** over calls that ran concurrently | +| `c7-ordering.ts` | does result order survive concurrency? | **Positional, always** (completion order reversed, results in input order). Coalesced rows **share one object** | +| `c8-assembled.ts` | the whole job, end to end, against a hand-rolled pool | **45 vs 87 executable lines**, identical outcomes — and **44 requests vs 32**, all of the difference on the failure path | + +## Files + +- `fake-vendor.ts` — the vendor. `GET /orders` returns N orders whose `customerId`s round-robin over + a smaller pool (100 orders, 30 customers ⇒ every id 3 or 4 times, a flat and exact duplicate + distribution). `GET /customers/{id}` counts requests **per id**, records each arrival with its + **in-flight count** and its time on the injected clock, and tracks the **peak**. Named ids can 404 + or 429 persistently; `burst429(n)` rate-limits the first `n` requests whatever their id, which is + how a simultaneous cohort gets throttled together. `slow` gives per-id latencies so completion + order can be made to disagree with call order. `echoUrl` and `retryAfter` exist because two + findings turn on whether the transport echoes `res.url` and whether the vendor sends `Retry-After`. +- `harness.ts` — `check` / `checkSeq` / `checkRequests` / `checkPeak` / `checkAtMost` / + `checkAtLeast` / `note` / `heading` / `finish`. `checkRequests` prints requests **and** the + distinct-id floor together, because `100 requests for 30 distinct ids` is the finding and `100` + alone is not; `checkPeak` prints the peak **and** the declared bound for the same reason. +- `trace-probe.ts` — a `TraceSink` recording `(name, type, traceId, spanId, parentSpanId, url)`, + reduced to trace count, root count, **max chain depth** and **max fan-out**. The `url` is there + because one stitch called 100 times emits 100 spans with one name. +- `type-probe.ts` — hands the compiler seven candidate fan-out spellings and reports which compile. + "The combinator can't" and "I couldn't find the spelling" are different findings. +- `fanout.ts` / `hand-rolled.ts` — the assembled answer and the control, both over the same + `Adapter` and the same `Clock`, both delimited by `BEGIN`/`END USER CODE` markers so the line + count is of code someone maintains. +- `virtual-time.ts` — `drain` / `runOut`, so a claim can say "advance past everything" instead of + hand-computing a hundred backoff schedules. + +## Reading the numbers honestly + +- **C2 is the headline and it deserves to be read carefully.** The saving is real and it is + specifically an **in-flight** saving: with all 100 calls issued in the same tick, a TTL cache + cannot help any of them, and `coalesce: false` proved that (100 requests, same cache, same TTL). + What made it 30 is `InflightCoalescer.join` (cache.ts:207-262) handing the first caller per key a + leader claim and everyone else a shared promise, awaited at engine.ts:1634-1663. `'cluster'` is + accepted and **silently degrades to `'process'`** in v1 (cache.ts:396-397). +- **The coalescer is a SUCCESS-path optimisation, and the failure path is its exact inverse.** When + the leader fails, `claim.fail` rejects and each follower's `catch` re-runs the whole chain + independently (engine.ts:1646-1659). Measured: 100 concurrent calls for one 404ing id → **100 + requests in two waves, 1 then 99**. Every caller got its own honest `HTTP 404` (never a + leader-failure artefact), which is the right error at the wrong price. In C8 this is **4 requests + for the deleted customer against the hand-rolled `Map`'s 1**, and it scales with the + number of order rows naming a broken id, not with the number of broken ids. +- **C1 refutes the capture's stated reason while confirming its conclusion.** The capture says a + runtime length "rules out any combinator that takes a fixed list of members". It does not: + `membersFrom` (pipe.ts:226-229) reads a plain array and `all(ids.map(…))` compiles and runs. The + wall is the **input** — `runMember` spreads one `StitchInput` into every member (pipe.ts:75-86), + measured as **100 requests for `cust-001` and 99 wasted**. Of seven candidate spellings only + `all(array)` and `all(a, b, c)` compile; `all(one, inputs)`, `all.map`, `allSettled`, `.safe()` + members and `all(members, { concurrency })` are all compile errors. +- **`all()`'s auto-cancel prevented zero requests.** It aborts the losers on the first failure + (pipe.ts:114-117), but in a fan-out they have all already left: measured 100 requests made, 99 + customers fetched, and every one discarded by the fail-fast. +- **C3's good news is narrow and its trap is the construction C1 forces.** One stitch, 100 calls, + `concurrency: 8` → **peak 8**. One hundred stitches each declaring 8 → **peak 100**, because + `makeStitch` builds a limiter per stitch (stitch.ts:985-988) over closure-local state + (resilience.ts:107). `pool: 'host'` repairs it via the module-level `hostStates` registry + (resilience.ts:84) — and a **`store` breaks the repair again** (peak 100), because + `createStoreThrottle` never reads `opts.pool` and keeps `inFlight` in its own Map + (store.ts:137-151). A **seam** is the construction that survives: 100 members under a seam-level + `concurrency: 8` measured peak 8 (seam.ts:51-69). +- **This proof's own hypothesis about the throttle was wrong, in the library's disfavour.** It + assumed a call sleeping on a retry backoff had released its slot. It has not: the backoff sleep + is inside the `try` whose `finally` releases (engine.ts:760-765, 834-837). Measured with a bound + of 4, a 429ed first wave and a 1s backoff, the **fifth call left at t=1050 rather than t=50** — + four of four slots held by calls that were asleep and issuing nothing, ~95% of the declared + budget idle. And a retry re-queues at the **back** of the FIFO: the first call's retry left at + t=4200, behind every other call's first attempt. +- **The coalescer sits outside the throttle, which is the right layering.** 100 calls over 30 ids at + a bound of 8 fired exactly **22 `throttled` events** (30 real requests minus the first 8), not 92. + The joiners never reach the limiter, so the bound governs **requests**, not callers. +- **C5's default is correct and a well-behaved vendor cancels it.** `expo-jitter` is the default + curve (resilience.ts:45) and it spread a 100-call cohort over ~98 distinct milliseconds across the + full window, all ten 100ms slices occupied; `expo` and `fixed` both put all 100 in **one** + millisecond, because attempt 2 is `base·2^0` and doubling a constant is a constant. A bare + `retry: { attempts: 2 }` de-clusters with nothing configured. But `Retry-After` is preferred over + the computed backoff and `retry.respect` defaults ON (engine.ts:748-761), so a vendor sending + `Retry-After: 2` put all 100 retries back into **one millisecond at t=2000** with `expo-jitter` + still declared. `retry: { respect: false }` restores the spread and is all-or-nothing. +- **The jitter is FULL, not equal** (`Math.random() * computed`, resilience.ts:54): the earliest of + 100 retries measured 2-8ms against a base/2 floor of 500. Aggressive, and correct here. +- **C6 answers the capture's `linked` question yes, with a caveat that matters.** `ScopedRun` takes + the stitch's **own** input per call (pipe.ts:355-361), so `linked` is the only construction that + gets a runtime-length fan-out of **different** ids into one trace: measured 1 traceId, 1 root, 101 + spans over the list and all 100 lookups. The shape is wrong, though — `run` chains each call under + the **previous** one (pipe.ts:360-367), so the same run measured **depth 101, max fan-out 1** while + the calls genuinely ran concurrently (peak 100 in flight). A viewer draws 100 simultaneous lookups + as a queue. `linked(...)` is a `Promise`, not a `Composable`, confirming scenario 10. +- **C7's ordering answer is boring and its aliasing answer is not.** Completion order reversed + end-to-end, results still in input order, under concurrency and under coalescing alike. But every + joiner is handed the leader's object **by reference** (engine.ts:1645,1662): 20 rows over 5 + customers measured **5 distinct objects**, and mutating row 0's customer changed row 5's. The same + fan-out with no cache measured 20 distinct objects, so the aliasing arrives with the optimisation. + A cache **hit** aliases the same way for the whole TTL. +- **C8's 45 vs 87 lines attributes cleanly.** What became configuration: the FIFO pool, the + retry-with-jitter loop, the retryable-status set, the non-2xx throw and the URL assembly. What did + **not** shrink: the per-row `problem` branch and the positional join. Partial failure is user code + on both sides, and it is the only user code the StitchAPI version needs. + +## The footguns + +- **`cache: { ttl: 0 }` caches FOREVER.** It is the obvious spelling for "I want the in-flight + dedupe, not the staleness", and `memoryStore` stores `expires: ttl ? now() + ttl : 0` and treats + `expires === 0` as live (store.ts:15-16,45). Measured: a second fan-out long after the first added + **0 requests**. There is no "coalesce only" spelling. +- **`sensitive: true` silently disables coalescing.** `ensureCache` returns null on it + (engine.ts:1020), so a config that still reads `cache: { ttl: '60s' }` went back to **100 requests + for 30 ids** with nothing warning. The same holds for `.inspect()`, which bypasses the cache by + default (ADR 0016) and therefore runs its own uncoalesced request. +- **Coalescing applies only to the CACHEABLE METHOD set.** A POST-shaped lookup coalesced nothing + (100 requests) until `methods: 'POST'` was named (30). The default is `['GET','HEAD']` + (cache.ts:367-369). +- **A declared `concurrency` is multiplied by the number of stitch objects.** 100 stitches each + declaring 8 measured peak 100. This is not hypothetical: it is the construction C1 shows is the + _only_ way `all()` can express a per-id fan-out. +- **Adding a `store` un-pools `pool: 'host'` concurrency.** The config does not change, the rate + budget becomes cross-process as intended, and the concurrency bound quietly reverts to + per-instance — measured peak 100 against a declared 8. `createStoreThrottle` (store.ts:137-151) + reads `concurrency` and `rate` and never reads `pool`. +- **A backing-off call holds its concurrency slot.** With `concurrency: N` and a long backoff, N + slots can be occupied by N sleeping calls issuing nothing (measured ~95% idle over a 1s backoff). + A retry then re-queues at the back of the FIFO, so a retried call finishes after every call that + started later. +- **`Retry-After` defeats `expo-jitter` by default.** Obeying the server is the right default in + general and the wrong one for a simultaneous cohort, and the two policies cannot be combined: + `retry.respect` is a boolean, so it is obey-and-cluster or ignore-and-spread. +- **Coalesced and cached callers share one mutable object.** Any normalise/enrich step that writes + onto a joined customer writes onto every row sharing it — and, through the cache, onto every later + hit for the TTL. +- **`verdict: { accept: [404] }` does not classify a missing customer, it succeeds on it.** Measured + `ok: true` with `{ error: 'customer_not_found', id }` handed back as the customer, so the join + writes a row with an undefined name and nothing says so. (The same trap was measured on an + idempotency 409 in `unconfirmed-write` and an AWS skew 403 in `expiring-signatures`.) +- **`StitchError.url` comes from the TRANSPORT.** `rebuildError` copies `res.url`, so a custom + adapter that does not echo it leaves the caller unable to say which id failed from the error + alone (measured `undefined`). `fetchAdapter` does set it (http-adapter.ts:98,111,145). The array + index is the only identifier that always works. + +## What is NOT measured here + +- **A batch endpoint.** The capture's own best advice is not to fan out at all where + `GET /customers?ids=…` exists. Nothing here measures that path; the partial-failure semantics you + inherit from it are [`batch-partial-failure`](../batch-partial-failure/). +- **Cross-process coalescing.** `coalesce: 'cluster'` is accepted and degrades to `'process'` + (cache.ts:396-397); no measurement here involves two processes, so nothing establishes what a + real cluster protocol would or would not collapse. +- **A shared `store` behind the cache.** Every measurement uses the default `memoryStore`. Whether a + Redis-backed store changes the coalescing arithmetic (it should not — the coalescer is in-process + by construction) is untested. +- **Real latency distributions.** Holds are exact virtual durations, so the concurrency measurements + are free of the scheduling noise a real transport has. The peak-in-flight numbers are therefore + upper bounds on tidiness, not predictions. +- **Memory.** 100 concurrent calls means 100 live run states, 100 event streams and a joined array; + nothing here measures the footprint. See [`large-response-memory`](../large-response-memory/). +- **`store`-backed cross-process concurrency.** C3 (e) establishes that a store does not pool + concurrency in-process; it does not attempt a two-process test of whether anything could. diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c1-combinators.ts b/docs/scenarios/proofs/n-plus-one-fanout/c1-combinators.ts new file mode 100644 index 00000000..effd0915 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c1-combinators.ts @@ -0,0 +1,297 @@ +// C1 — can `all()` express a runtime-length list of DIFFERENT inputs? +// +// This is the shape the whole scenario is: `GET /orders` returns N rows, each needing its own +// `GET /customers/{id}`. N is not known until the first response lands, and every call needs a +// DIFFERENT id. The capture's hypothesis, inherited from scenario 7, is that `all()` structurally +// cannot do it because it broadcasts ONE `StitchInput` to every member (pipe.ts:75-86). +// +// MEASURED: the hypothesis holds, and the runtime length is not the reason. +// (a) The RUNTIME LENGTH is fine. `all(ids.map(…))` compiles and runs — `membersFrom` +// (pipe.ts:226-229) takes a plain array, and nothing anywhere wants a literal. +// (b) THE INPUT IS THE WALL. 100 members, one input: 100 requests, ONE distinct id, 100 requests +// for the SAME customer. `runMember` builds `{ ...input, signal }` once per member from the +// one input the group was called with (pipe.ts:81). +// (c) The workaround is N stitch objects with the id baked into the URL. It works — 100 distinct +// ids — and it is the construction C3 measures the cost of. +// (d) `all()` BOUNDS NOTHING: peak 100 in-flight over 100 members. Scenario 7 measured peak 8 +// over 8; at 100 the same non-bound is a different-sized problem. +// (e) `all()` is FAIL-FAST and the successes are discarded: one 404 among 100 rejects the whole +// group and hands back no data at all. There is deliberately no `allSettled` (pipe.ts:20-21). +// (f) …and `.safe()` members cannot be composed in, because `Member` is gated on the `__stitch` +// brand (pipe.ts:188-189) and `stitch.safe(input)` is a `Promise`, not a stitch. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c1-combinators.ts +import { stitch } from '../../../../packages/core/src/index'; +import { all } from '../../../../packages/core/src/pipe'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { type Customer, FakeVendor, idsOf } from './fake-vendor'; +import { + check, + checkPeak, + checkRequests, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { accepted, probeSpellings, rejected } from './type-probe'; +import { runOut } from './virtual-time'; + +const BASE = 'https://api.vendor.test'; +const HOLD = 50; + +/** The runtime context every measurement here is read off. */ +function context(opts: { orders: number; customers: number }) { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: opts.orders, + customers: opts.customers, + holdMs: HOLD, + }); + return { clock, vendor }; +} + +async function main(): Promise { + heading( + 'C1 — a runtime-length list of DIFFERENT inputs, through the combinators', + ); + + // ── (a) what the compiler admits ─────────────────────────────────────────────────────────── + // The runtime length is NOT the problem the capture expected it to be: `all` over a `.map()` is + // a legal call. What does not exist is any spelling that varies the INPUT per member. + { + const results = probeSpellings([ + { + label: 'all(ids.map(id => stitch(...))) — runtime-length array of stitches', + code: `void all(many);`, + }, + { + label: 'all(one, one, one) — the bare-argument form', + code: `void all(one, one, one);`, + }, + { + label: 'all(one, inputs) — one stitch, a list of inputs', + code: `void all(one, ids.map((id) => ({ params: { id } })));`, + }, + { + label: 'all.map(one, ids) — a mapping combinator', + code: `void all.map(one, ids);`, + }, + { + label: 'allSettled([...]) — a partial-failure combinator', + code: `void allSettled(many);`, + }, + { + label: 'all(ids.map(id => () => one.safe({ params: { id } }))) — `.safe()` members', + code: `void all(ids.map((id) => () => one.safe({ params: { id } })));`, + }, + { + label: 'all(many, { concurrency: 8 }) — a bound on the fan', + code: `void all(many, { concurrency: 8 });`, + }, + ]); + checkSeq('(a) fan-out spellings that COMPILE', accepted(results), [ + 'all(ids.map(id => stitch(...))) — runtime-length array of stitches', + 'all(one, one, one) — the bare-argument form', + ]); + check( + '(a) spellings the compiler REFUSED', + rejected(results).length, + 5, + ); + note( + '(a) → the runtime LENGTH is not the obstacle', + '`membersFrom` (pipe.ts:226-229) reads a plain array; `all(ids.map(...))` compiles and runs. What is missing is per-member INPUT', + ); + } + + // ── (b) THE WALL: one input, broadcast to every member ───────────────────────────────────── + // The only way to give `all` a per-member id would be through the input, and there is one + // input for the whole group. `runMember` (pipe.ts:75-86) spreads it into every member. + { + const { clock, vendor } = context({ orders: 100, customers: 100 }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + clock, + }); + // 100 members, built at runtime from the list — exactly the shape the scenario wants. + const members = vendor.orders.map(() => fetchCustomer) as Stitch[]; + const group = all(members); + const pending = group({ params: { id: 'cust-001' } }); + await runOut(clock, 5_000, 1_000); + const values = (await pending) as unknown[]; + + check('(b) members in the group', members.length, 100); + check('(b) values returned', values.length, 100); + checkRequests( + '(b) 100 members, ONE input', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + check( + '(b) DISTINCT ids that reached the server', + vendor.distinctIds, + 1, + ); + check('(b) requests for cust-001', vendor.requestsFor('cust-001'), 100); + note( + '(b) → 100 calls, one customer, 99 of them pure waste', + '`runMember` builds `{ ...input, signal }` from the single group input (pipe.ts:81) — every member is handed the same `params`', + ); + } + + // ── (c) the workaround: bake the id into N SEPARATE stitches ─────────────────────────────── + // This does express the scenario. The price is one `stitch()` object per row, which is the + // construction whose resilience cost C3 measures. + { + const { clock, vendor } = context({ orders: 100, customers: 100 }); + const adapter = vendor.adapter(); + const members = idsOf(vendor.orders).map( + (id) => + stitch({ + name: `customer:${id}`, + url: `${BASE}/customers/${id}`, + adapter, + clock, + }) as Stitch, + ); + const pending = all(members)(); + await runOut(clock, 5_000, 1_000); + await pending; + + checkRequests( + '(c) 100 stitches, id baked into each URL', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + check( + '(c) DISTINCT ids that reached the server', + vendor.distinctIds, + 100, + ); + note( + '(c) → it works, and it costs 100 stitch objects', + 'each carries its own throttle/cache/circuit state — the trap C3 measures', + ); + } + + // ── (d) `all()` bounds nothing, at 100 as at 8 ───────────────────────────────────────────── + { + const { clock, vendor } = context({ orders: 100, customers: 100 }); + const adapter = vendor.adapter(); + const members = idsOf(vendor.orders).map( + (id) => + stitch({ + name: `customer:${id}`, + url: `${BASE}/customers/${id}`, + adapter, + clock, + }) as Stitch, + ); + const pending = all(members)(); + await runOut(clock, 5_000, 1_000); + await pending; + checkPeak( + '(d) peak in-flight under `all()`', + vendor.peakInFlight, + undefined, + 100, + ); + note( + '(d) → scenario 7 measured peak 8 over 8 members', + 'the non-bound is the same; at 100 members it is a different-sized problem', + ); + } + + // ── (e) fail-fast: one 404 discards 99 successes ─────────────────────────────────────────── + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: 100, + customers: 100, + holdMs: HOLD, + notFound: ['cust-050'], // one deleted customer among a hundred good ones + }); + const adapter = vendor.adapter(); + const members = idsOf(vendor.orders).map( + (id) => + stitch({ + name: `customer:${id}`, + url: `${BASE}/customers/${id}`, + adapter, + clock, + }) as Stitch, + ); + let failed = false; + let message = ''; + let recovered: unknown = 'nothing'; + const pending = all(members)().then( + (v) => { + recovered = v; + }, + (e: Error) => { + failed = true; + message = e.message; + }, + ); + await runOut(clock, 5_000, 1_000); + await pending; + + check('(e) the group REJECTED', failed, true); + check('(e) the error message', message, 'HTTP 404'); + check( + '(e) successes handed back to the caller', + String(recovered), + 'nothing', + ); + note( + '(e) customers successfully fetched and then discarded', + vendor.customerCalls.filter((c) => c.status === 200).length, + ); + note( + '(e) → `runAllArray` awaits `Promise.all` and rethrows (pipe.ts:133-137)', + 'there is deliberately no `allSettled` variant (pipe.ts:20-21)', + ); + } + + // ── (f) …and you cannot hand `all` a `.safe()` member to get partial results ─────────────── + // The doc comment says "compose `.safe()` members by hand" (pipe.ts:21). `Member` is gated on + // the `__stitch`/`__composable` BRAND (pipe.ts:188-189), and `stitch.safe(input)` returns a + // `Promise` — no brand. So the suggested composition is not a composition at all; + // it is `Promise.allSettled` in user code, which is C4. + { + const results = probeSpellings([ + { + label: 'all(one.safe, one.safe)', + code: `void all(one.safe, one.safe);`, + }, + { + label: 'a plain async function as a member', + code: `void all(async () => 1, async () => 2);`, + }, + ]); + check( + '(f) `.safe()`-flavoured members that COMPILE', + accepted(results).length, + 0, + ); + checkSeq('(f) refused', rejected(results), [ + 'all(one.safe, one.safe)', + 'a plain async function as a member', + ]); + } + + finish( + 'C1', + 'CONFIRMED, and the reason is narrower than the capture says. The RUNTIME LENGTH is not the obstacle: `all(ids.map(...))` compiles and runs, because `membersFrom` (pipe.ts:226-229) takes a plain array. THE INPUT IS. 100 members called with one input made 100 requests for ONE distinct id — 100 fetches of cust-001, 99 of them waste — because `runMember` spreads the single group input into every member (pipe.ts:75-86, `{ ...input, signal }`). Of seven candidate spellings only two compile (`all(array)` and `all(a, b, c)`); `all(one, inputs)`, `all.map`, `allSettled`, `.safe()` members and `all(members, { concurrency })` are all compile errors. The workaround — one `stitch()` per row with the id baked into the URL — does express it (measured: 100 distinct ids) and costs 100 stitch objects, which is the trap C3 measures. Two further non-bounds ride along: `all()` bounded nothing (peak 100 in-flight over 100 members, the same result scenario 7 got at 8), and it is FAIL-FAST — one 404 rejected the group with `HTTP 404` and handed back NOTHING, discarding 99 successful customer fetches. `Member` is brand-gated on `__stitch` (pipe.ts:188-189), so the doc\'s own suggestion to "compose `.safe()` members by hand" does not typecheck', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c2-coalesce.ts b/docs/scenarios/proofs/n-plus-one-fanout/c2-coalesce.ts new file mode 100644 index 00000000..c78c9b4c --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c2-coalesce.ts @@ -0,0 +1,420 @@ +// C2 — THE DECIDING CLAIM. Does `cache.coalesce` collapse IN-FLIGHT duplicates? +// +// 100 orders commonly reference far fewer customers. A cache that only helps AFTER a response +// lands does nothing for a simultaneous fan-out — every one of the 100 calls misses, because none +// of them has finished yet. The fix is single-flight/in-flight coalescing, and most clients do not +// have it. `cache.coalesce` is documented as `'process' | 'cluster' | false` and nothing in this +// pass had exercised it. +// +// MEASURED, and IT IS REAL. 100 concurrent calls over 30 distinct ids: +// (a) no cache at all → 100 requests. The 3.33x quota bill the capture describes. +// (b) `cache: { ttl }` → 30 requests. ONE PER DISTINCT ID, from configuration alone, with all +// 100 calls in flight simultaneously and not one response yet landed. This is in-flight +// coalescing, it is ON BY DEFAULT the moment you add a `cache` block, and it is the finding. +// (c) `coalesce: false` → back to 100. So (b) is the coalescer, not the TTL cache. +// (d) `'cluster'` → 30, identical to `'process'` (cache.ts:396-397 degrades it in v1). +// +// AND THE OTHER DIRECTION, which the capture did not ask about and which is bigger than it looks: +// (e) A COALESCED FAILURE IS NOT SHARED. 100 concurrent calls for ONE id that 404s made 100 +// requests, in two waves — the leader, then 99 followers that each re-ran on their own +// (engine.ts:1647-1659). Coalescing buys exactly nothing for a failing id, and the second +// wave lands as a synchronised burst. +// (f) The joiners do NOT get the leader's error; each gets its own. So one 404 is diagnosed 100 +// times and the vendor is asked 100 times for a resource that does not exist. +// (g) The footgun: `cache: { ttl: 0 }` — the obvious spelling for "coalesce but do not cache" — +// caches FOREVER (store.ts:45, `ttl ? now() + ttl : 0`, and `expires === 0` reads as live). +// (h) …and `sensitive: true` silently turns the whole thing off, coalescing included. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c2-coalesce.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + CacheOptions, + SafeResult, +} from '../../../../packages/core/src/types'; +import { type Customer, FakeVendor, idsOf } from './fake-vendor'; +import { + check, + checkRequests, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { runOut } from './virtual-time'; + +const BASE = 'https://api.vendor.test'; +const ORDERS = 100; +const CUSTOMERS = 30; +const HOLD = 50; +const TTL = '60s'; + +interface Run { + vendor: FakeVendor; + results: SafeResult[]; +} + +/** + * Fire every order's customer lookup CONCURRENTLY through ONE stitch, with `cache` as given, and + * run the clock out. This is the scenario's shape exactly: N calls in the same tick, each with its + * own id, over a smaller pool of distinct ids. + */ +async function fanOut(opts: { + cache?: CacheOptions; + customers?: number; + notFound?: readonly string[]; + sensitive?: boolean; +}): Promise { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: opts.customers ?? CUSTOMERS, + holdMs: HOLD, + ...(opts.notFound ? { notFound: opts.notFound } : {}), + }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + clock, + ...(opts.cache ? { cache: opts.cache } : {}), + ...(opts.sensitive ? { sensitive: true } : {}), + }); + const pending = idsOf(vendor.orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + return { vendor, results: await Promise.all(pending) }; +} + +const okCount = (rs: readonly SafeResult[]): number => + rs.filter((r) => r.ok).length; + +async function main(): Promise { + heading( + `C2 — ${String(ORDERS)} CONCURRENT calls over ${String(CUSTOMERS)} distinct ids: how many requests reach the server?`, + ); + + // ── (a) the baseline: no cache block at all ──────────────────────────────────────────────── + { + const { vendor, results } = await fanOut({}); + checkRequests( + '(a) no `cache` block', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + check('(a) calls that succeeded', okCount(results), 100); + check('(a) distinct ids asked for', vendor.distinctIds, CUSTOMERS); + checkSeq( + '(a) requests for the three most-repeated ids', + [ + vendor.requestsFor('cust-001'), + vendor.requestsFor('cust-002'), + vendor.requestsFor('cust-030'), + ], + [4, 4, 3], + ); + } + + // ── (b) THE MEASUREMENT: a `cache` block, default coalescing ─────────────────────────────── + // Nothing has finished when the 100th call starts, so a read-through TTL cache alone cannot + // help. 30 requests means the duplicates were collapsed WHILE IN FLIGHT. + { + const { vendor, results } = await fanOut({ cache: { ttl: TTL } }); + checkRequests( + '(b) `cache: { ttl }` — default coalescing', + vendor.customerRequests, + vendor.distinctIds, + 30, + ); + check('(b) calls that succeeded', okCount(results), 100); + check('(b) DISTINCT ids asked for', vendor.distinctIds, CUSTOMERS); + checkSeq( + '(b) requests per id — every id exactly once', + [...new Set(vendor.perIdCounts())], + [1], + ); + check( + '(b) callers served without their own request', + ORDERS - vendor.customerRequests, + 70, + ); + note( + '(b) → in-flight coalescing, from one config field', + '`join(key)` returns a leader claim to the first caller and a shared promise to the rest (cache.ts:207-262); the engine awaits it at engine.ts:1634-1663', + ); + } + + // ── (c) turn the coalescer off and the TTL cache alone buys NOTHING here ─────────────────── + // Same cache, same TTL, `coalesce: false`: 100 requests. Every one of the 100 calls missed, + // because a simultaneous fan-out has no completed response to hit. + { + const { vendor } = await fanOut({ + cache: { ttl: TTL, coalesce: false }, + }); + checkRequests( + '(c) `coalesce: false`', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + note( + '(c) → the TTL cache is not what saved (b)', + 'a read-through cache helps the NEXT fan-out; the coalescer helps THIS one', + ); + } + + // ── (d) `'cluster'` is accepted and behaves as `'process'` in v1 ─────────────────────────── + { + const { vendor } = await fanOut({ + cache: { ttl: TTL, coalesce: 'cluster' }, + }); + checkRequests( + "(d) `coalesce: 'cluster'`", + vendor.customerRequests, + vendor.distinctIds, + 30, + ); + note( + '(d) → identical to `process`, by design and silently', + "cache.ts:396-397 — `config.coalesce === false ? false : 'process'`; the cross-process protocol is deferred and nothing warns that you did not get it", + ); + } + + // ── (e) THE OTHER DIRECTION: a coalesced FAILURE is not shared ───────────────────────────── + // 100 concurrent calls for ONE id, which 404s. If the failure were shared this would be 1 + // request. It is 100 — the leader, then 99 followers each re-running independently. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: 1, + holdMs: HOLD, + notFound: ['cust-001'], + }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + cache: { ttl: TTL }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + const results = await Promise.all(pending); + + checkRequests( + '(e) 100 concurrent calls for ONE id that 404s', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + check('(e) calls that failed', results.length - okCount(results), 100); + // Two waves: the leader alone, then the 99 followers it rejected. + const arrivals = vendor.customerCalls.map((c) => c.at); + check('(e) DISTINCT arrival times (waves)', new Set(arrivals).size, 2); + checkSeq( + '(e) wave sizes', + [ + arrivals.filter((a) => a === arrivals[0]).length, + arrivals.filter((a) => a !== arrivals[0]).length, + ], + [1, 99], + ); + note( + '(e) → engine.ts:1646-1649 and 1656-1659', + "`claim.fail(new Error('cache: leader run failed'))`, and each follower's `catch` re-runs the whole chain on its own", + ); + } + + // ── (f) …and every joiner diagnoses the failure for itself ───────────────────────────────── + // The leader's error is never handed on. Each follower gets ITS OWN 404 from ITS OWN request, + // which is why (e) is 100 requests rather than 1 error fanned out to 100 callers. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: 20, + customers: 1, + holdMs: HOLD, + notFound: ['cust-001'], + }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + cache: { ttl: TTL }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + const results = await Promise.all(pending); + const errors = results.flatMap((r) => (r.ok ? [] : [r.error])); + + check( + '(f) requests for 20 concurrent calls, one dead id', + vendor.customerRequests, + 20, + ); + checkSeq( + '(f) distinct error messages seen by the callers', + [...new Set(errors.map((e) => e.message))], + ['HTTP 404'], + ); + checkSeq( + '(f) distinct statuses', + [...new Set(errors.map((e) => e.status))], + [404], + ); + check( + "(f) any caller told 'cache: leader run failed'?", + errors.some((e) => e.message.includes('leader run failed')), + false, + ); + note( + '(f) → the right ERROR, at the wrong PRICE', + 'no caller is ever handed a leader-failure artefact (good), and the cost is that a deterministic 404 is re-asked once per joiner (bad)', + ); + } + + // ── (g) THE FOOTGUN: `ttl: 0` is "cache forever", not "do not cache" ─────────────────────── + // The natural spelling for "I want the dedupe, not the staleness" is `ttl: 0`. `memoryStore` + // stores `expires: ttl ? now() + ttl : 0` and treats `expires === 0` as immortal (store.ts:15-16, + // 45) — so the entry never expires and a LATER fan-out is served entirely from cache. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: CUSTOMERS, + holdMs: HOLD, + }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + cache: { ttl: 0 }, + clock, + }); + const first = idsOf(vendor.orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + await Promise.all(first); + const afterFirst = vendor.customerRequests; + // A SECOND fan-out, long after the first settled. With a real TTL of zero this should + // re-fetch everything. + const second = idsOf(vendor.orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + await Promise.all(second); + + check('(g) requests in the first fan-out', afterFirst, 30); + check( + '(g) requests added by the SECOND fan-out', + vendor.customerRequests - afterFirst, + 0, + ); + note( + '(g) → `ttl: 0` caches FOREVER', + 'store.ts:45 `ttl ? now() + ttl : 0`, store.ts:15-16 `expires === 0` is live — the entry has no expiry at all. There is no "coalesce only" spelling', + ); + note( + '(g) → and the store TTL runs on the WALL clock', + '`memoryStore` calls `now()` (util.ts:4 = `Date.now()`), not the injected `clock`, so a `manualClock` cannot age a cache entry out', + ); + } + + // ── (h) `sensitive: true` turns the whole thing off, coalescing included ─────────────────── + { + const { vendor } = await fanOut({ + cache: { ttl: TTL }, + sensitive: true, + }); + checkRequests( + '(h) `cache` + `sensitive: true`', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + note( + '(h) → `ensureCache` returns null when `cfg.sensitive` (engine.ts:1020)', + 'the config still READS as coalescing; the 3.33x quota bill comes back with no warning', + ); + } + + // ── (i) the coalescing set is the CACHEABLE-METHOD set ───────────────────────────────────── + // `methods` defaults to GET/HEAD (cache.ts:367-369) and coalescing applies to exactly that + // set, so a POST-shaped lookup (a batch-ish `POST /customers/search`) coalesces nothing until + // `methods` names it. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: CUSTOMERS, + holdMs: HOLD, + }); + const post = stitch({ + name: 'customer-post', + method: 'POST', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + cache: { ttl: TTL }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + post({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + await Promise.all(pending); + checkRequests( + '(i) a POST lookup with `cache: { ttl }`', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + + const clock2 = manualClock(); + const vendor2 = new FakeVendor({ + clock: clock2, + orders: ORDERS, + customers: CUSTOMERS, + holdMs: HOLD, + }); + const postCached = stitch({ + name: 'customer-post', + method: 'POST', + url: `${BASE}/customers/{id}`, + adapter: vendor2.adapter(), + cache: { ttl: TTL, methods: 'POST' }, + clock: clock2, + }); + const pending2 = idsOf(vendor2.orders).map((id) => + postCached({ params: { id } }).safe(), + ); + await runOut(clock2, 20_000, 1_000); + await Promise.all(pending2); + checkRequests( + "(i) …the same POST with `methods: 'POST'`", + vendor2.customerRequests, + vendor2.distinctIds, + 30, + ); + } + + finish( + 'C2', + 'YES — `cache.coalesce` GENUINELY COLLAPSES IN-FLIGHT DUPLICATES, and this is the strongest positive result in the pass. 100 concurrent calls over 30 distinct ids made 30 REQUESTS — one per id, exactly the floor — with every call in flight simultaneously and not one response yet landed, from a single `cache: { ttl }` block and no user code. `coalesce: false` on the same cache put it back to 100, so the saving is the coalescer and not the TTL. 70 of the 100 callers were served without a request of their own. `cluster` is accepted and silently degrades to `process` (cache.ts:396-397). THE OTHER DIRECTION IS AS IMPORTANT AND CUTS THE OTHER WAY: a coalesced FAILURE is not shared. 100 concurrent calls for one id that 404s made 100 requests in TWO WAVES — 1 leader, then 99 followers each re-running the whole chain independently (engine.ts:1646-1659) — so a failing id gets no dedupe at all and its retry storm is synchronised. Every joiner got its own honest `HTTP 404` (status 404, never a leader-failure artefact), which is the right ERROR at the wrong PRICE: the correct diagnosis is bought by asking the vendor 100 times for a resource that does not exist. TWO FOOTGUNS. `cache: { ttl: 0 }` is the obvious spelling for "dedupe but do not cache" and it caches FOREVER (store.ts:15-16,45 — `expires === 0` reads as live); a second fan-out much later added 0 requests. And `sensitive: true` silently disables the whole cache including coalescing (engine.ts:1020), taking the fan-out back to 100 requests with the config still reading as if it coalesces. Coalescing also applies only to the CACHEABLE METHOD set: a POST lookup coalesced nothing (100) until `methods: \'POST\'` was named (30)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c3-bounded-concurrency.ts b/docs/scenarios/proofs/n-plus-one-fanout/c3-bounded-concurrency.ts new file mode 100644 index 00000000..7b45a30c --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c3-bounded-concurrency.ts @@ -0,0 +1,356 @@ +// C3 — bounded concurrency. Does `throttle: { concurrency: N }` actually bound a 100-call fan-out, +// and what happens to someone who builds a stitch per id? +// +// The governance on this shape is often concurrency-based rather than request-rate-based, so a +// requests-per-second cap does not protect you: what the vendor counts is how many of your +// connections are open at once. The only honest measurement is PEAK IN-FLIGHT at the server. +// +// MEASURED: the declaration works, on ONE stitch, and every way of spreading the fan-out across +// stitch OBJECTS silently multiplies the budget by the number of objects. +// (a) no throttle → peak 100. `all()` bounded nothing (C1 d) and neither does a bare loop. +// (b) `throttle: { concurrency: 8 }` on ONE stitch called 100 times → PEAK 8. It holds exactly, +// and it is the right construction for this scenario. +// (c) THE TRAP: 100 SEPARATE stitches, each declaring `concurrency: 8` → PEAK 100. Each +// `stitch()` builds its own limiter (stitch.ts:985-988) with closure-local state +// (resilience.ts:107), so "8 at a time" became "8 at a time, 100 times over". +// (d) `pool: 'host'` fixes (c) — peak 8 across 100 separate stitches, via the module-level +// `hostStates` registry (resilience.ts:84,106-107). +// (e) …and ADDING A `store` SILENTLY BREAKS THAT FIX AGAIN. `createStoreThrottle` keeps +// concurrency in a closure-local Map (store.ts:137-151) and never reads `pool` at all, so +// `pool: 'host'` + `store` measured PEAK 100. The store is what you add for cross-process +// rate limiting; it un-pools the concurrency on the way past. +// (f) A SEAM with a seam-level `concurrency` DOES pool across its members — peak 8 over 100 +// member stitches (seam.ts:51-69) — which is the one construction that survives the +// stitch-per-id shape. +// (g) A BACKING-OFF CALL HOLDS ITS SLOT. The backoff sleep is inside the `try` the release's +// `finally` guards (engine.ts:760-765, 834-837), so N slots can be occupied by N calls that +// are asleep and issuing nothing — measured 4 of 4 idle for 95% of the run. +// (h) The coalescer sits OUTSIDE the throttle: 100 calls over 30 ids at a bound of 8 fired 22 +// `throttled` events, not 92, so the joiners never take a slot. The bound is on REQUESTS. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c3-bounded-concurrency.ts +import { seam, stitch } from '../../../../packages/core/src/index'; +import { memoryStore } from '../../../../packages/core/src/store'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { type Customer, FakeVendor, idsOf } from './fake-vendor'; +import { check, checkPeak, checkSeq, finish, heading, note } from './harness'; +import { runOut } from './virtual-time'; + +const ORDERS = 100; +const BOUND = 8; +const HOLD = 50; + +/** A fresh vendor with 100 orders over 100 distinct customers (no duplicates: this is not C2). */ +function context(host: string) { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: ORDERS, + holdMs: HOLD, + }); + return { clock, vendor, base: `https://${host}` }; +} + +async function main(): Promise { + heading( + `C3 — ${String(ORDERS)} calls, a declared bound of ${String(BOUND)}: what was the PEAK IN-FLIGHT?`, + ); + + // ── (a) the unbounded baseline ───────────────────────────────────────────────────────────── + { + const { clock, vendor, base } = context('a.vendor.test'); + const call = stitch({ + name: 'customer', + url: `${base}/customers/{id}`, + adapter: vendor.adapter(), + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 60_000, 1_000); + await Promise.all(pending); + checkPeak('(a) no throttle', vendor.peakInFlight, undefined, 100); + } + + // ── (b) ONE stitch, called 100 times, with a declared bound ──────────────────────────────── + // This is the construction the capture predicted would work, and it does — exactly. + { + const { clock, vendor, base } = context('b.vendor.test'); + const call = stitch({ + name: 'customer', + url: `${base}/customers/{id}`, + adapter: vendor.adapter(), + throttle: { concurrency: BOUND }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 60_000, 1_000); + const results = await Promise.all(pending); + checkPeak( + '(b) ONE stitch, `throttle: { concurrency: 8 }`', + vendor.peakInFlight, + BOUND, + BOUND, + ); + check( + '(b) all 100 calls still completed', + results.filter((r) => r.ok).length, + 100, + ); + check('(b) requests made', vendor.customerRequests, 100); + note( + '(b) → the slot is keyed on `nameOf(cfg)` (engine.ts:265-274)', + 'one stitch called N times is ONE key over ONE limiter — which is exactly this scenario', + ); + } + + // ── (c) THE TRAP: one stitch per id, each declaring the same bound ───────────────────────── + // The construction C1 (c) showed is the only way `all()` can express the scenario. Give each of + // those 100 stitches the bound and the bound evaporates: `makeStitch` builds a limiter per + // stitch, and `createThrottle`'s per-key state is closure-local unless pooled. + { + const { clock, vendor, base } = context('c.vendor.test'); + const adapter = vendor.adapter(); + const calls = idsOf(vendor.orders).map((id) => + stitch({ + name: `customer:${id}`, + url: `${base}/customers/${id}`, + adapter, + throttle: { concurrency: BOUND }, + clock, + }), + ); + const pending = calls.map((c) => c().safe()); + await runOut(clock, 60_000, 1_000); + await Promise.all(pending); + checkPeak( + '(c) 100 SEPARATE stitches, each `concurrency: 8`', + vendor.peakInFlight, + BOUND, + 100, + ); + note( + '(c) → 100 limiters, 8 slots each: a declared budget multiplied by 100', + 'stitch.ts:985-988 builds a throttle per stitch; `createThrottle` keeps state in a closure-local Map (resilience.ts:107)', + ); + } + + // ── (d) `pool: 'host'` re-pools the separate stitches ────────────────────────────────────── + { + const { clock, vendor, base } = context('d.vendor.test'); + const adapter = vendor.adapter(); + const calls = idsOf(vendor.orders).map((id) => + stitch({ + name: `customer:${id}`, + url: `${base}/customers/${id}`, + adapter, + throttle: { concurrency: BOUND, pool: 'host' as const }, + clock, + }), + ); + const pending = calls.map((c) => c().safe()); + await runOut(clock, 60_000, 1_000); + await Promise.all(pending); + checkPeak( + "(d) …the same 100 stitches with `pool: 'host'`", + vendor.peakInFlight, + BOUND, + BOUND, + ); + note( + '(d) → the state moves to a MODULE-level registry', + '`hostStates` (resilience.ts:84) is shared by every host-pooled limiter in the process, and `hostKey` keys on the URL host (engine.ts:265-274)', + ); + } + + // ── (e) …and a `store` silently un-does it ───────────────────────────────────────────────── + // A store is what you add to make the RATE budget cross-process. `createStoreThrottle` does not + // read `pool` at all and keeps `inFlight`/`waiters` in its own closure-local Map, so the + // concurrency bound goes back to per-object — while the config still says `pool: 'host'`. + { + const { clock, vendor, base } = context('e.vendor.test'); + const adapter = vendor.adapter(); + const store = memoryStore(); + const calls = idsOf(vendor.orders).map((id) => + stitch({ + name: `customer:${id}`, + url: `${base}/customers/${id}`, + adapter, + throttle: { concurrency: BOUND, pool: 'host' as const }, + store, + clock, + }), + ); + const pending = calls.map((c) => c().safe()); + await runOut(clock, 60_000, 1_000); + await Promise.all(pending); + checkPeak( + "(e) `pool: 'host'` + a shared `store`", + vendor.peakInFlight, + BOUND, + 100, + ); + note( + '(e) → `createStoreThrottle` never reads `opts.pool` (store.ts:137-151)', + 'the shared store carries the RATE window (`rl:` keys) and nothing else; concurrency stays a per-instance Map, so adding a store for cross-process rate limiting un-pools the concurrency', + ); + } + + // ── (f) a SEAM pools its members, which is the construction that survives ────────────────── + { + const { clock, vendor, base } = context('f.vendor.test'); + const s = seam({ + baseUrl: base, + adapter: vendor.adapter(), + throttle: { concurrency: BOUND }, + clock, + }); + const calls = idsOf(vendor.orders).map((id) => + s.stitch({ + name: `customer:${id}`, + path: `/customers/${id}`, + }), + ); + const pending = calls.map((c) => c().safe()); + await runOut(clock, 60_000, 1_000); + await Promise.all(pending); + checkPeak( + '(f) 100 seam MEMBERS under a seam-level `concurrency: 8`', + vendor.peakInFlight, + BOUND, + BOUND, + ); + note( + '(f) → `seamBucket` re-keys every acquire onto `seam:` (seam.ts:51-69)', + 'one bucket for every member, whatever its name — the only construction here that bounds a stitch-per-id fan-out', + ); + } + + // ── (g) A BACKING-OFF CALL HOLDS ITS SLOT ────────────────────────────────────────────────── + // The retry-backoff sleep (engine.ts:760-765) is INSIDE the `try` whose `finally` releases + // (engine.ts:834-837), so a call that is asleep still occupies one of the N slots. 16 calls, + // bound 4, every first attempt 429ed, a 1s fixed backoff: if the slot were released at the + // 429 the 5th call would leave at t=50. It leaves at t=1050 — the whole backoff later. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: 16, + customers: 16, + holdMs: HOLD, + }); + vendor.burst429(16); // every first attempt is rate-limited; every retry succeeds + const call = stitch({ + name: 'customer', + url: 'https://g.vendor.test/customers/{id}', + adapter: vendor.adapter(), + throttle: { concurrency: 4 }, + retry: { attempts: 2, backoff: { curve: 'fixed', base: '1s' } }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 60_000, 1_000); + const results = await Promise.all(pending); + const first = vendor.customerCalls.filter((c) => c.status === 429); + checkPeak( + '(g) peak in-flight with a retry in the mix', + vendor.peakInFlight, + 4, + 4, + ); + check( + '(g) requests (16 calls x 2 attempts)', + vendor.customerRequests, + 32, + ); + check( + '(g) calls that eventually succeeded', + results.filter((r) => r.ok).length, + 16, + ); + checkSeq( + '(g) when each wave of 4 FIRST attempts left', + [...new Set(first.map((c) => c.at))], + [0, 1050, 2100, 3150], + ); + note( + '(g) → wave 2 left at t=1050, not t=50', + 'the 429 landed at t=50 and the slot stayed held for the whole 1000ms backoff — 4 of 4 slots occupied by calls doing nothing, ~95% of the declared budget idle', + ); + checkSeq( + '(g) when the first call`s RETRY finally left', + vendor.customerCalls + .filter((c) => c.id === 'cust-001') + .map((c) => c.at), + [0, 4200], + ); + note( + '(g) → and a retry re-queues at the BACK of the FIFO', + '`continue` (engine.ts:765) runs the `finally` release, which hands the slot to the next WAITER (resilience.ts:162-164); the retrying call then re-acquires behind every fresh call', + ); + } + + // ── (h) the coalescer sits OUTSIDE the throttle ──────────────────────────────────────────── + // The cache lookup is outermost over the expensive chain (engine.ts:1713-1718) and the throttle + // is acquired inside `attemptLoop` — so a coalesced JOINER never takes a slot. The decisive + // measurement is the `throttled` progress event, which fires once per acquire that actually + // BLOCKED (engine.ts:636-643): 100 calls over 30 ids at a bound of 8 should block at most + // 30 - 8 = 22 times, not 100 - 8 = 92. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: 100, + customers: 30, + holdMs: HOLD, + }); + let throttled = 0; + const call = stitch({ + name: 'customer', + url: 'https://h.vendor.test/customers/{id}', + adapter: vendor.adapter(), + throttle: { concurrency: BOUND }, + cache: { ttl: '60s' }, + trace: { + handle(event) { + if ( + event.type === 'progress' && + event.phase === 'throttled' + ) + throttled += 1; + }, + }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 60_000, 1_000); + const results = await Promise.all(pending); + check( + '(h) requests made (100 calls, 30 ids)', + vendor.customerRequests, + 30, + ); + check('(h) calls completed', results.filter((r) => r.ok).length, 100); + checkPeak('(h) peak in-flight', vendor.peakInFlight, BOUND, BOUND); + check('(h) `throttled` events — acquires that BLOCKED', throttled, 22); + note( + '(h) → 22 = the 30 real requests minus the first 8', + 'the 70 coalesced joiners never reached the limiter at all, so the bound applies to REQUESTS and not to callers — which is what you want, and it means a fan-out over duplicates finishes far faster than its call count suggests', + ); + } + + finish( + 'C3', + 'YES on one stitch, NO on any construction that spreads the fan-out across stitch OBJECTS — and the failure is silent every time. `throttle: { concurrency: 8 }` on ONE stitch called 100 times measured PEAK 8 in-flight exactly, against an unthrottled baseline of 100, with all 100 calls completing. That is the right construction for this scenario and it needs no user code. THE TRAP IS REAL AND IT IS THE CONSTRUCTION C1 FORCES: the only way `all()` can express a per-id fan-out is one stitch per id, and 100 separate stitches each declaring `concurrency: 8` measured PEAK 100 — `makeStitch` builds a limiter per stitch (stitch.ts:985-988) over closure-local state (resilience.ts:107), so a declared budget of 8 became 800. `pool: \'host\'` repairs it (peak 8 over 100 stitches, via the module-level `hostStates` registry, resilience.ts:84) — AND ADDING A `store` SILENTLY BREAKS THE REPAIR: `createStoreThrottle` never reads `opts.pool` and keeps `inFlight` in a closure-local Map (store.ts:137-151), so `pool: "host"` + `store` measured PEAK 100 again. The store is exactly what you add to make the RATE budget cross-process, and it un-pools the CONCURRENCY on the way past with the config unchanged. The one construction that survives a stitch-per-id shape is a SEAM: 100 members under a seam-level `concurrency: 8` measured peak 8, because `seamBucket` re-keys every acquire onto one `seam:` (seam.ts:51-69). ONE MORE, AGAINST AN ASSUMPTION THIS PROOF MADE AND HAD TO CORRECT: a BACKING-OFF call HOLDS its slot. The retry sleep sits inside the `try` the release `finally` guards (engine.ts:760-765, 834-837), so with a bound of 4, a 429ed first wave and a 1s backoff, the fifth call left at t=1050 rather than t=50 — 4 of 4 slots occupied by calls that were asleep and issuing nothing, ~95% of the declared budget idle. And a retry re-queues at the BACK of the FIFO (the `continue` releases to the next waiter, resilience.ts:162-164): the first call`s retry left at t=4200, behind every other call`s first attempt. ONE CLEAN WIN TO END ON: the coalescer sits OUTSIDE the throttle (engine.ts:1713-1718), so 100 calls over 30 ids at a bound of 8 fired exactly 22 `throttled` events — 30 real requests minus the first 8 — proving the 70 joiners never reached the limiter. The bound applies to REQUESTS, not to callers', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c4-partial-failure.ts b/docs/scenarios/proofs/n-plus-one-fanout/c4-partial-failure.ts new file mode 100644 index 00000000..5e9281c6 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c4-partial-failure.ts @@ -0,0 +1,346 @@ +// C4 — one customer was deleted. Do the other 99 survive, and can you tell WHICH one failed? +// +// `Promise.all` rejects on the first failure AND discards the results that already succeeded. That +// is the standard warning, and this scenario is where it costs the most: the 99 good rows were +// fetched, paid for, and thrown away because a hundredth id 404ed. +// +// MEASURED: `.safe()` per call fixes it completely, and the failing id is identifiable four ways — +// but two of the four depend on things outside the config. +// (a) bare `Promise.all` over throwing calls → 1 rejection, 0 rows kept, and ALL 100 requests +// still went out. The quota was spent and nothing was retained. +// (b) `.safe()` per call → 99 rows kept, 1 error, and the array INDEX still lines up with the +// input, so the failing order row is identifiable with no extra bookkeeping. +// (c) `Promise.allSettled` over throwing calls is the same outcome with more ceremony. +// (d) The error is well-furnished: `status: 404`, `attempts: 1`, `body` carrying the vendor's +// `{ error, id }`, and `url` naming the exact resource. +// (e) …but `url` is COPIED OFF THE ADAPTER RESPONSE (stitch.ts `rebuildError`), so a transport +// that does not echo it leaves `error.url` undefined and the error self-identifies only if +// the vendor's BODY happens to name the id. +// (f) `all()`'s auto-cancel bought nothing: the losers had already left. 100 requests, 0 rows. +// (g) `verdict: { accept: [404] }` does not classify the miss — it SUCCEEDS on it, and hands the +// error envelope back as the customer. +// (h) A 429 id burns its whole retry budget while the other 99 are long done: 3 requests for the +// doomed id, and `.safe()` still keeps the 99. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c4-partial-failure.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + SafeResult, + StitchError, +} from '../../../../packages/core/src/types'; +import { type Customer, FakeVendor, idsOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { runOut } from './virtual-time'; + +const BASE = 'https://api.vendor.test'; +const ORDERS = 100; +const HOLD = 50; +/** The customer someone deleted. Position 49 in the list, so "index 49" is a real coordinate. */ +const DEAD = 'cust-050'; + +function context( + opts: { echoUrl?: boolean; notFound?: readonly string[] } = {}, +) { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: ORDERS, + holdMs: HOLD, + notFound: opts.notFound ?? [DEAD], + ...(opts.echoUrl === undefined ? {} : { echoUrl: opts.echoUrl }), + }); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + clock, + }); + return { clock, vendor, call }; +} + +async function main(): Promise { + heading( + `C4 — ${String(ORDERS)} lookups, one of them a deleted customer (${DEAD})`, + ); + + // ── (a) the one-liner everyone writes ────────────────────────────────────────────────────── + { + const { clock, vendor, call } = context(); + let kept = 0; + let failure = ''; + const pending = Promise.all( + idsOf(vendor.orders).map((id) => call({ params: { id } })), + ).then( + (rows) => { + kept = rows.length; + }, + (e: Error) => { + failure = e.message; + }, + ); + await runOut(clock, 20_000, 1_000); + await pending; + + check('(a) `Promise.all` rejected with', failure, 'HTTP 404'); + check('(a) rows the caller kept', kept, 0); + check( + '(a) requests that still reached the server', + vendor.customerRequests, + 100, + ); + check( + '(a) customers successfully fetched and discarded', + vendor.customerCalls.filter((c) => c.status === 200).length, + 99, + ); + } + + // ── (b) `.safe()` per call ───────────────────────────────────────────────────────────────── + // The whole fix, and it is one method call. `SafeResult` is a discriminated union, so the + // survivors narrow to `Customer` without a cast. + { + const { clock, vendor, call } = context(); + const ids = idsOf(vendor.orders); + const pending = ids.map((id) => call({ params: { id } }).safe()); + await runOut(clock, 20_000, 1_000); + const results: SafeResult[] = await Promise.all(pending); + const failedAt = results.flatMap((r, i) => (r.ok ? [] : [i])); + + check('(b) rows kept', results.filter((r) => r.ok).length, 99); + check( + '(b) failures', + results.length - results.filter((r) => r.ok).length, + 1, + ); + checkSeq('(b) INDEX of the failure', failedAt, [49]); + check( + '(b) …which the caller maps back to an id with the input array', + ids[failedAt[0] ?? -1], + DEAD, + ); + check('(b) requests made', vendor.customerRequests, 100); + note( + '(b) → order is positional and preserved (see C7)', + 'so `results[i]` belongs to `orders[i]` and no correlation key is needed', + ); + } + + // ── (c) `Promise.allSettled` over the throwing form ──────────────────────────────────────── + { + const { clock, vendor, call } = context(); + const pending = Promise.allSettled( + idsOf(vendor.orders).map((id) => call({ params: { id } })), + ); + await runOut(clock, 20_000, 1_000); + const settled = await pending; + check( + '(c) fulfilled', + settled.filter((s) => s.status === 'fulfilled').length, + 99, + ); + checkSeq( + '(c) index of the rejection', + settled.flatMap((s, i) => (s.status === 'rejected' ? [i] : [])), + [49], + ); + check( + '(c) the rejection reason is a StitchError', + settled + .flatMap((s) => + s.status === 'rejected' ? [s.reason as Error] : [], + ) + .map((e) => e.name) + .join(), + 'StitchError', + ); + void vendor; + } + + // ── (d) is the failing id identifiable FROM THE ERROR? ───────────────────────────────────── + { + const { clock, vendor, call } = context(); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + const results = await Promise.all(pending); + const err = results.flatMap((r) => + r.ok ? [] : [r.error], + )[0] as StitchError; + + check('(d) error.status', err.status, 404); + check('(d) error.attempts', err.attempts, 1); + check('(d) error.url', err.url, `${BASE}/customers/${DEAD}`); + check( + '(d) error.body names the id', + (err.body as { id?: string }).id, + DEAD, + ); + check('(d) error.message', err.message, 'HTTP 404'); + note( + '(d) → the message alone is useless and everything else is enough', + '`HTTP 404` is identical for all 100; `url`, `body` and the array index each name the row', + ); + } + + // ── (e) …but `url` comes from the TRANSPORT, not the engine ─────────────────────────────── + // `rebuildError` copies `url: res.url` off the adapter response. `fetchAdapter` sets it + // (http-adapter.ts:98,111,145); a hand-written adapter, a mock, or a custom transport may not. + { + const { clock, vendor, call } = context({ echoUrl: false }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + const results = await Promise.all(pending); + const err = results.flatMap((r) => + r.ok ? [] : [r.error], + )[0] as StitchError; + + check( + '(e) error.url with a transport that omits it', + err.url, + undefined, + ); + check( + '(e) error.body still names the id (vendor-dependent)', + (err.body as { id?: string }).id, + DEAD, + ); + check('(e) rows still kept', results.filter((r) => r.ok).length, 99); + void vendor; + note( + '(e) → two of the four identifiers are outside your control', + '`url` needs the ADAPTER to echo it and `body` needs the VENDOR to name the id; the array index is the only one that always works', + ); + } + + // ── (f) the combinator's auto-cancel does not save the requests ──────────────────────────── + // `all()` aborts the losers on the first failure (pipe.ts:114-117). In a fan-out they have all + // already left, so the cancel saves nothing and the fail-fast still costs the 99 rows. + { + const { clock, vendor } = context(); + const adapter = vendor.adapter(); + const members = idsOf(vendor.orders).map((id) => + stitch({ + name: `customer:${id}`, + url: `${BASE}/customers/${id}`, + adapter, + clock, + }), + ); + let kept = 0; + let failure = ''; + const { all } = await import('../../../../packages/core/src/pipe'); + const pending = all(members)().then( + (rows) => { + kept = (rows as unknown[]).length; + }, + (e: Error) => { + failure = e.message; + }, + ); + await runOut(clock, 20_000, 1_000); + await pending; + check('(f) `all()` rejected with', failure, 'HTTP 404'); + check('(f) rows kept', kept, 0); + check( + '(f) requests the auto-cancel prevented', + 100 - vendor.customerRequests, + 0, + ); + } + + // ── (g) the `verdict.accept` trap ────────────────────────────────────────────────────────── + // The reflex for "a 404 is not really an error here" is `verdict: { accept: [404] }`. It does + // not classify the miss; it makes it a SUCCESS whose `data` is the vendor's error envelope. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: ORDERS, + holdMs: HOLD, + notFound: [DEAD], + }); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + verdict: { accept: [404] }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + const results = await Promise.all(pending); + const row = results[49]; + + check( + '(g) calls reported as failures', + results.filter((r) => !r.ok).length, + 0, + ); + check('(g) the deleted customer reported as ok', row?.ok, true); + check( + '(g) …and its `data` is the vendor error envelope', + JSON.stringify(row?.ok === true ? row.data : null), + JSON.stringify({ error: 'customer_not_found', id: DEAD }), + ); + note( + '(g) → `customer.name` is now `undefined` and nothing said so', + 'the join writes a row with a missing name rather than a row flagged as missing', + ); + } + + // ── (h) a rate-limited id burns its retry budget alone ───────────────────────────────────── + // 429 IS in the default `retry.on` set (engine.ts:612), so one permanently-throttled id costs + // its full attempt budget while the other 99 finished on their first try. `.safe()` still keeps + // the 99, which is the point. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: ORDERS, + holdMs: HOLD, + rateLimited: [DEAD], + }); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + retry: { attempts: 3, backoff: { curve: 'fixed', base: '1s' } }, + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 60_000, 1_000); + const results = await Promise.all(pending); + + check('(h) rows kept', results.filter((r) => r.ok).length, 99); + check('(h) requests for the throttled id', vendor.requestsFor(DEAD), 3); + check( + '(h) requests for a healthy id', + vendor.requestsFor('cust-001'), + 1, + ); + check('(h) total requests', vendor.customerRequests, 102); + check( + '(h) attempts on the surfaced error', + results.flatMap((r) => (r.ok ? [] : [r.error.attempts]))[0], + 3, + ); + } + + finish( + 'C4', + "SOLVED, by `.safe()`, and the failing row is identifiable four ways. Bare `Promise.all` behaved exactly as advertised: rejected with `HTTP 404`, kept ZERO rows — and all 100 requests still reached the server, so 99 customers were fetched, paid for and discarded. `.safe()` per call kept 99 rows and 1 error with no other change, and because the results array is positional the failure is at INDEX 49, which maps straight back to the order row. `Promise.allSettled` over the throwing form gives the same outcome with more ceremony (the reason is a real `StitchError`). The error is well-furnished — `status: 404`, `attempts: 1`, `body: { error: 'customer_not_found', id: 'cust-050' }`, `url: .../customers/cust-050` — while the MESSAGE is the useless `HTTP 404`, identical for all hundred. TWO OF THE FOUR IDENTIFIERS ARE NOT THE LIBRARY'S TO GIVE: `url` is copied off the ADAPTER response (`rebuildError`, stitch.ts), so a transport that does not echo it measured `error.url === undefined`, and `body` only names the id because this vendor does. The array index is the only identifier that always holds. Three things worth stating plainly. `all()`'s auto-cancel (pipe.ts:114-117) prevented ZERO requests — the losers had already left — so fail-fast costs the 99 rows and saves nothing. `verdict: { accept: [404] }` does NOT classify the miss: it reported the deleted customer as `ok: true` with the vendor's error envelope as `data`, so the join silently writes a row with no name. And a permanently-429ed id burns its full budget alone (3 requests against 1 for every healthy id, 102 total, `attempts: 3` on the error) while `.safe()` keeps the other 99", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c5-thundering-herd.ts b/docs/scenarios/proofs/n-plus-one-fanout/c5-thundering-herd.ts new file mode 100644 index 00000000..f3ababc8 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c5-thundering-herd.ts @@ -0,0 +1,358 @@ +// C5 — THE THUNDERING HERD. 100 calls leave together, 100 calls are 429ed together. When do the +// retries arrive? +// +// This matters more here than anywhere else, precisely because the calls started together. A +// deterministic backoff adds the SAME number to the SAME instant and re-clusters the burst exactly +// as it was; only jitter breaks it. `backoff: 'expo-jitter'` is the default (resilience.ts:45), so +// the question is whether the default actually de-clusters — quantified, not asserted. +// +// The measurement is the ARRIVAL TIME at the server. `manualClock` reports a timer's EXACT due +// time (test-clock.ts:90), so a backoff of 372.4ms is recorded as 372.4 and the bucketing is done +// by this proof, at a stated width. `largestBucket(arrivals, 1)` is how many retries share their +// most crowded MILLISECOND: 1 means fully de-clustered, 100 means the burst re-formed intact. A +// 100ms width is also reported, because that is closer to the scale a rate window cares about. +// +// MEASURED: the default works, and it is the only one of the three that does. +// (a) `'fixed'` → 100 retries in ONE millisecond. The burst re-formed exactly. +// (b) `'expo'` → 100 retries in ONE millisecond. Identical: doubling a constant is a constant. +// (c) `'expo-jitter'` (default) → ~100 distinct milliseconds, worst millisecond holds 1-2, and +// at a 100ms width the 100 retries are spread over all 10 slices of the window, ~10 each. +// This is the whole reason it is the default. +// (d) THE ONE THAT UNDOES IT, and it is on by DEFAULT: a vendor that sends `Retry-After` gets +// obeyed verbatim (engine.ts:748-751), and every retry is then scheduled at the SAME +// absolute instant — 100 in one bucket, WITH `expo-jitter` configured. `retry: { respect: +// false }` restores the jitter and disobeys the server. +// (e) The jitter is FULL, not equal: `Math.random() * computed` (resilience.ts:54), so a retry +// can land arbitrarily early. That is the aggressive-but-correct choice for de-clustering. +// (f) Coalescing does NOT protect a herd: C2 measured that a failed leader releases its +// followers to run independently, so a 429ed cohort of duplicates re-fans at full width. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c5-thundering-herd.ts +import { stitch } from '../../../../packages/core/src/index'; +import { + type ManualClock, + manualClock, +} from '../../../../packages/core/src/testing'; +import type { BackoffCurve, Stitch } from '../../../../packages/core/src/types'; +import { + type Customer, + FakeVendor, + distinctBuckets, + idsOf, + largestBucket, +} from './fake-vendor'; +import { + check, + checkAtLeast, + checkAtMost, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { runOut } from './virtual-time'; + +const HERD = 100; +/** Long enough that a 1ms bucket is a fine-grained reading of the jitter window. */ +const BASE = '1s'; +const BASE_MS = 1000; + +interface Spread { + /** How many retries shared their most crowded MILLISECOND. */ + largest: number; + /** How many distinct milliseconds the retries occupied. */ + buckets: number; + /** The same, at a 100ms width — a tenth of the jitter window. */ + largestCoarse: number; + coarseBuckets: number; + first: number; + last: number; + /** Every retry arrival, for the raw record. */ + arrivals: number[]; +} + +/** Round for printing: the raw arrivals are fractional and unreadable at full precision. */ +const ms = (x: number): string => x.toFixed(1); + +/** + * Fire `HERD` calls in one tick, 429 every one of them, and report WHEN the retries arrived. + * + * The vendor is told to 429 exactly the first `HERD` requests, so the whole cohort is throttled + * simultaneously and every retry succeeds — which isolates the backoff curve as the only thing + * deciding arrival time. + */ +async function herd(opts: { + /** A curve AND a base, both present: the config surface refuses an all-optional envelope. */ + backoff: { curve: BackoffCurve; base: string }; + retryAfter?: string; + /** `retry.respect` — defaults ON in the library, so this mirrors it. */ + respect?: boolean; +}): Promise { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: HERD, + customers: HERD, + holdMs: 0, // the 429 comes back immediately: every call is throttled in the same instant + ...(opts.retryAfter ? { retryAfter: opts.retryAfter } : {}), + }); + vendor.burst429(HERD); + const call = stitch({ + name: 'customer', + url: 'https://api.vendor.test/customers/{id}', + adapter: vendor.adapter(), + retry: { + attempts: 2, + backoff: opts.backoff, + respect: opts.respect ?? true, + }, + clock, + }); + return measure(clock, vendor, call); +} + +/** Drive one cohort to completion and reduce its retry arrivals to a {@link Spread}. */ +async function measure( + clock: ManualClock, + vendor: FakeVendor, + call: Stitch, +): Promise { + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 4 * BASE_MS, 250); + await Promise.all(pending); + const arrivals = vendor.arrivalsAfter(HERD); // everything after the 429ed first wave + return { + largest: largestBucket(arrivals, 1), + buckets: distinctBuckets(arrivals, 1), + largestCoarse: largestBucket(arrivals, 100), + coarseBuckets: distinctBuckets(arrivals, 100), + first: Math.min(...arrivals), + last: Math.max(...arrivals), + arrivals, + }; +} + +const report = (label: string, s: Spread): void => { + note( + `${label} spread`, + `${String(s.buckets)} distinct ms over [${ms(s.first)}, ${ms(s.last)}]ms; worst ms holds ${String(s.largest)}, worst 100ms slice holds ${String(s.largestCoarse)} across ${String(s.coarseBuckets)} slices`, + ); +}; + +async function main(): Promise { + heading( + `C5 — ${String(HERD)} calls 429ed in the same instant: when do the retries land?`, + ); + + // ── (a) `'fixed'` — the burst re-forms exactly ───────────────────────────────────────────── + { + const s = await herd({ backoff: { curve: 'fixed', base: BASE } }); + check('(a) `fixed` retries that arrived', s.arrivals.length, HERD); + check( + '(a) `fixed` — retries in the WORST millisecond', + s.largest, + HERD, + ); + check('(a) `fixed` — distinct milliseconds', s.buckets, 1); + checkSeq( + '(a) `fixed` — [first, last] arrival', + [s.first, s.last], + [BASE_MS, BASE_MS], + ); + report('(a) `fixed`', s); + } + + // ── (b) `'expo'` — identical, because attempt 2 is base·2^0 ──────────────────────────────── + { + const s = await herd({ backoff: { curve: 'expo', base: BASE } }); + check('(b) `expo` — retries in the WORST millisecond', s.largest, HERD); + check('(b) `expo` — distinct milliseconds', s.buckets, 1); + report('(b) `expo`', s); + note( + '(b) → doubling a constant is still a constant', + 'every member of the cohort computes the same delay from the same instant (resilience.ts:48-55); the burst arrives as one packet whatever the exponent', + ); + } + + // ── (c) `'expo-jitter'`, the DEFAULT — the burst is broken up ────────────────────────────── + // `Math.random()` is real randomness, so the exact numbers move run to run. The assertions are + // therefore bounds, set far from both the ~1-2 per millisecond a birthday estimate gives for + // 100 continuous draws over a 1000ms window and the 100 the deterministic curves produce. + { + const s = await herd({ backoff: { curve: 'expo-jitter', base: BASE } }); + check( + '(c) `expo-jitter` — retries that arrived', + s.arrivals.length, + HERD, + ); + checkAtMost( + '(c) `expo-jitter` — retries in the WORST millisecond', + s.largest, + 4, + ); + checkAtLeast( + '(c) `expo-jitter` — distinct milliseconds', + s.buckets, + 80, + ); + checkAtMost( + '(c) `expo-jitter` — retries in the WORST 100ms slice', + s.largestCoarse, + 25, + ); + check( + '(c) `expo-jitter` — 100ms slices of the window touched (of 10)', + s.coarseBuckets, + 10, + ); + checkAtMost('(c) earliest retry (ms)', s.first, 200); + checkAtLeast('(c) latest retry (ms)', s.last, 800); + report('(c) `expo-jitter`', s); + + // …and the same measurement with NOTHING declared but `attempts`, to establish that the + // curve above really is what a bare `retry` gets. The base is then the built-in 100ms + // (resilience.ts:46), so the window is a tenth as wide and the millisecond buckets crowd + // proportionally — the point is only that the arrivals are SPREAD rather than identical. + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: HERD, + customers: HERD, + holdMs: 0, + }); + vendor.burst429(HERD); + const bare = await measure( + clock, + vendor, + stitch({ + name: 'customer', + url: 'https://api.vendor.test/customers/{id}', + adapter: vendor.adapter(), + retry: { attempts: 2 }, // no `backoff` at all + clock, + }), + ); + checkAtLeast( + '(c) a BARE `retry: { attempts: 2 }` — distinct milliseconds', + bare.buckets, + 50, + ); + checkAtMost( + '(c) …latest retry (ms), against the 100ms default base', + bare.last, + 100, + ); + report('(c) bare `retry: { attempts: 2 }`', bare); + note( + '(c) → `expo-jitter` really is the default', + '`kind = policy.curve ?? "expo-jitter"` (resilience.ts:45) with `base ?? 100` (resilience.ts:46); nothing has to be configured to de-cluster', + ); + } + + // ── (d) …and the default vendor behaviour undoes it ──────────────────────────────────────── + // A well-behaved 429 carries `Retry-After`. The engine prefers it over the computed backoff + // (`ra ?? backoffDelay(...)`, engine.ts:748-761) and `retry.respect` defaults ON — so every + // member of the cohort is told the same number and the herd re-forms, with `expo-jitter` still + // configured and doing nothing. + { + const s = await herd({ + backoff: { curve: 'expo-jitter', base: BASE }, + retryAfter: '2', + }); + check('(d) with `Retry-After: 2` — worst millisecond', s.largest, HERD); + check('(d) distinct milliseconds', s.buckets, 1); + checkSeq('(d) [first, last] arrival', [s.first, s.last], [2000, 2000]); + report('(d) `expo-jitter` + `Retry-After`', s); + + const restored = await herd({ + backoff: { curve: 'expo-jitter', base: BASE }, + retryAfter: '2', + respect: false, + }); + checkAtMost( + '(d) …with `retry: { respect: false }` — worst millisecond', + restored.largest, + 4, + ); + checkAtLeast('(d) …distinct milliseconds', restored.buckets, 80); + report('(d) `respect: false`', restored); + note( + '(d) → the fix disobeys the server', + '`respect: false` is all-or-nothing: there is no "honour the header, then jitter around it" — the two policies cannot be combined in configuration', + ); + } + + // ── (e) the jitter is FULL, not equal ────────────────────────────────────────────────────── + // `delay = Math.random() * computed` (resilience.ts:54) — the whole window, from 0. An "equal + // jitter" scheme (half fixed, half random) would floor at base/2. Measuring the earliest + // arrival across a large cohort distinguishes them. + { + const s = await herd({ backoff: { curve: 'expo-jitter', base: BASE } }); + checkAtMost( + '(e) earliest of 100 retries, against base/2 = 500ms', + s.first, + 400, + ); + note( + '(e) → full jitter, so a retry may land almost immediately', + 'aggressive for de-clustering and correct for it; it does mean the effective minimum wait is 0, not base', + ); + } + + // ── (f) coalescing does not protect the herd ─────────────────────────────────────────────── + // C2 (e) measured the mechanism; here is what it costs a throttled fan-out. 100 calls over 20 + // distinct ids, all 429ed: a cache collapses the SUCCESSES, but the failed leaders release + // their followers, so the cohort re-fans at full width. + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: HERD, + customers: 20, + holdMs: 0, + retryAfter: '1', + }); + vendor.burst429(20); // only the 20 LEADERS are throttled + const call = stitch({ + name: 'customer', + url: 'https://api.vendor.test/customers/{id}', + adapter: vendor.adapter(), + cache: { ttl: '60s' }, + retry: { attempts: 1 }, // no retry: isolate the follower re-run from a retry + clock, + }); + const pending = idsOf(vendor.orders).map((id) => + call({ params: { id } }).safe(), + ); + await runOut(clock, 4 * BASE_MS, 250); + const results = await Promise.all(pending); + check( + '(f) requests for 100 calls over 20 ids when the leaders are 429ed', + vendor.customerRequests, + 100, + ); + check( + '(f) calls that succeeded', + results.filter((r) => r.ok).length, + 80, + ); + check( + '(f) the 20 leaders failed, the 80 followers re-ran and succeeded', + results.filter((r) => !r.ok).length, + 20, + ); + note( + '(f) → the coalescer is a SUCCESS-path optimisation', + 'engine.ts:1646-1659 — a failed leader hands its followers nothing, so exactly the cohort that just tripped a rate limit is the cohort that fans back out at full width', + ); + } + + finish( + 'C5', + "THE DEFAULT DOES DE-CLUSTER, and the default VENDOR behaviour cancels it. 100 calls 429ed in the same instant, over a 1s backoff window: `'fixed'` put all 100 retries in ONE MILLISECOND (t=1000.0, 1 distinct arrival time), `'expo'` did exactly the same (doubling a constant is still a constant — attempt 2 is base·2^0), and `'expo-jitter'` — the default, resilience.ts:45 — spread them over ~98 distinct milliseconds from ~4ms to ~990ms, with the worst millisecond holding 2 and all ten 100ms slices of the window occupied (worst slice ~14-16). A bare `retry: { attempts: n }` therefore already breaks the herd, with nothing to configure. THE TRAP IS THAT A WELL-BEHAVED VENDOR UNDOES IT: a 429 carrying `Retry-After` is obeyed verbatim (`ra ?? backoffDelay(...)`, engine.ts:748-761) and `retry.respect` defaults ON, so `Retry-After: 2` put all 100 retries back into ONE millisecond at exactly t=2000 with `expo-jitter` still configured and contributing nothing. `retry: { respect: false }` restored the ~98-millisecond spread and is all-or-nothing — there is no \"honour the header, then jitter around it\", so the choice is obey-and-cluster or ignore-and-spread. Two riders. The jitter is FULL (`Math.random() * computed`, resilience.ts:54), not equal, so the earliest of 100 retries measured 2-8ms against a base/2 floor of 500 — aggressive, and correct for this. And COALESCING DOES NOT PROTECT A HERD: with 100 calls over 20 ids and only the 20 leaders 429ed, the run made 100 requests and 80 followers re-fanned at full width, because a failed leader releases its joiners (engine.ts:1646-1659) — the cohort that just tripped the limit is exactly the cohort that fans back out", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c6-trace.ts b/docs/scenarios/proofs/n-plus-one-fanout/c6-trace.ts new file mode 100644 index 00000000..5dac6980 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c6-trace.ts @@ -0,0 +1,237 @@ +// C6 — is a 100-call fan-out ONE trace tree or a hundred unrelated roots? Does `linked` help when +// the members are created at runtime? +// +// A fan-out that shows up as 101 disconnected root spans is unreadable: you cannot ask "what did +// this order sync do", only "what did this one lookup do". The scenario's whole shape is `list → +// fan out → join`, and the trace should say so. +// +// MEASURED: a hundred roots by default, ONE tree with `linked` — and the shape `linked` draws is a +// 101-DEEP CHAIN, not a fan, which is a faithful record of nothing that happened. +// (a) 100 bare calls after a list call → 101 distinct traceIds, 101 roots. Nothing relates them. +// (b) `all()` → ONE traceId and a 100-wide fan. It is the right SHAPE and it is unusable here, +// because the members share one input (C1). +// (c) `linked` DOES take per-call input — `run(stitch, { params: { id } })` — so it is the only +// construction that gives a runtime-length fan-out of DIFFERENT inputs one trace tree. +// Measured: 1 traceId, 1 root, 101 spans. +// (d) …and the shape is a CHAIN of depth 101, max fan-out 1, because `run` chains each call under +// the PREVIOUS one (pipe.ts:360-367) whatever the concurrency. The calls really did run +// concurrently (measured: peak 100 in flight) and the trace draws them as a queue. +// (e) `linked` returns a `Promise`, not a `Composable` — confirming scenario 10 — so a fan-out +// written this way cannot itself be a member of anything. +// (f) Every span is named after the STITCH (`customer`, x100). The only per-call identity a sink +// gets is the `url` on the `start` event (engine.ts:1071-1085); the `name` is useless here. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c6-trace.ts +import { stitch } from '../../../../packages/core/src/index'; +import { all, linked } from '../../../../packages/core/src/pipe'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { type Customer, FakeVendor, type Order, idsOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { idFromUrl, recordingSink } from './trace-probe'; +import { runOut } from './virtual-time'; + +const BASE = 'https://api.vendor.test'; +const ORDERS = 100; +const HOLD = 50; + +function context() { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: ORDERS, + holdMs: HOLD, + }); + const trace = recordingSink(); + const adapter = vendor.adapter(); + const listOrders = stitch<{ data: Order[] }>({ + name: 'orders', + url: `${BASE}/orders`, + adapter, + trace, + clock, + }); + const fetchCustomer = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter, + trace, + clock, + }); + return { clock, vendor, trace, listOrders, fetchCustomer, adapter }; +} + +async function main(): Promise { + heading('C6 — the trace over `list -> fan out -> join`'); + + // ── (a) the default: one root per call ───────────────────────────────────────────────────── + { + const { clock, trace, listOrders, fetchCustomer } = context(); + const listing = listOrders(); + await runOut(clock, 5_000, 1_000); + const orders = (await listing).data; + const pending = idsOf(orders).map((id) => + fetchCustomer({ params: { id } }).safe(), + ); + await runOut(clock, 20_000, 1_000); + await Promise.all(pending); + + check('(a) spans', trace.starts().length, 101); + check('(a) DISTINCT trace trees', trace.traceIds().length, 101); + check('(a) ROOT spans', trace.roots().length, 101); + check('(a) max fan-out under any parent', trace.maxFanout(), 101); + note( + '(a) → every call mints its own root run', + '`newRunContext()` with no parent (stitch.ts) — a fan-out is 101 unrelated traces, and nothing ties the lookups to the list that produced them', + ); + } + + // ── (b) `all()` draws the right shape and cannot carry the inputs ────────────────────────── + { + const { clock, trace, fetchCustomer } = context(); + const members = Array.from({ length: ORDERS }, () => fetchCustomer); + const pending = all(members as Stitch[])({ + params: { id: 'cust-001' }, + }); + await runOut(clock, 20_000, 1_000); + await pending; + + check('(b) spans', trace.starts().length, 100); + check('(b) DISTINCT trace trees', trace.traceIds().length, 1); + check('(b) max fan-out under one parent', trace.maxFanout(), 100); + check( + '(b) DISTINCT ids in those 100 spans', + new Set(trace.starts().map(idFromUrl)).size, + 1, + ); + check('(b) ROOT spans among the members', trace.roots().length, 0); + note( + '(b) → the fan is real and the parent never emits', + 'the group run belongs to a `Composable`, which is not a stitch and produces no span; the 100 members all name a `parentSpanId` that appears nowhere in the trace', + ); + } + + // ── (c) `linked` gives per-call input AND one tree ───────────────────────────────────────── + // `ScopedRun` is `(stitch: Stitch, ...args)` (pipe.ts:355-361) — the stitch's OWN + // input, per call. So this is the only construction that gets a runtime-length fan-out of + // different inputs into a single trace. + { + const { clock, vendor, trace, listOrders, fetchCustomer } = context(); + const done = linked(async (run) => { + const { data } = await run(listOrders); + const rows = idsOf(data).map((id) => + run(fetchCustomer, { params: { id } }), + ); + return Promise.all(rows); + }); + await runOut(clock, 30_000, 1_000); + const customers = await done; + + check('(c) rows joined', customers.length, 100); + check('(c) spans', trace.starts().length, 101); + check('(c) DISTINCT trace trees', trace.traceIds().length, 1); + check('(c) ROOT spans', trace.roots().length, 1); + check( + '(c) DISTINCT ids in the customer spans', + new Set( + trace + .starts() + .filter((r) => r.name === 'customer') + .map(idFromUrl), + ).size, + 100, + ); + note( + '(c) → one traceId over the list AND all 100 lookups', + 'the whole `list -> fan out -> join` is one queryable tree, with each lookup carrying its own id', + ); + void vendor; + } + + // ── (d) …and the shape it draws is a chain, not a fan ────────────────────────────────────── + // `run` sets `prev` to the context it just minted (pipe.ts:362-367), so call i's parent is call + // i-1 — whatever order they actually execute in. The calls here are all started before any is + // awaited, so they genuinely ran concurrently. + { + const { clock, vendor, trace, listOrders, fetchCustomer } = context(); + const done = linked(async (run) => { + const { data } = await run(listOrders); + const rows = idsOf(data).map((id) => + run(fetchCustomer, { params: { id } }), + ); + return Promise.all(rows); + }); + await runOut(clock, 30_000, 1_000); + await done; + + check('(d) max chain DEPTH', trace.maxDepth(), 101); + check('(d) max FAN-OUT under any parent', trace.maxFanout(), 1); + check( + '(d) …while the calls really were concurrent: peak in-flight', + vendor.peakInFlight, + 100, + ); + note( + '(d) → the trace says A->B->C->…, 101 deep', + 'a viewer renders 100 simultaneous lookups as a sequential queue; the depth is an artefact of CALL ORDER (pipe.ts:360-367), not of any dependency', + ); + } + + // ── (e) `linked` is a Promise, not a Composable ──────────────────────────────────────────── + // Confirms scenario 10. It cannot be nested as a member, cannot be re-called, and cannot be + // handed a different input later — it has already run by the time you hold it. + { + const { clock, listOrders } = context(); + const result = linked((run) => run(listOrders)); + await runOut(clock, 5_000, 1_000); + await result; + check('(e) `linked(...)` is thenable', typeof result.then, 'function'); + check( + '(e) `linked(...)` carries the `__composable` brand?', + (result as unknown as { __composable?: true }).__composable, + undefined, + ); + check( + '(e) `linked(...)` is callable?', + typeof (result as unknown), + 'object', + ); + } + + // ── (f) what identifies a span ───────────────────────────────────────────────────────────── + // One stitch called 100 times gives 100 spans with ONE name. `TraceContext.name` is therefore + // no help; the `url` on the `start` event is the per-call identity, and it is only there + // because the id is in the path. + { + const { clock, trace, fetchCustomer } = context(); + const pending = Array.from({ length: 5 }, (_, i) => + fetchCustomer({ + params: { id: `cust-00${String(i + 1)}` }, + }).safe(), + ); + await runOut(clock, 20_000, 1_000); + await Promise.all(pending); + checkSeq( + '(f) span NAMES', + [...new Set(trace.starts().map((r) => r.name))], + ['customer'], + ); + checkSeq( + '(f) ids recoverable from the start event url', + trace.starts().map(idFromUrl), + ['cust-001', 'cust-002', 'cust-003', 'cust-004', 'cust-005'], + ); + note( + '(f) → an id carried in a QUERY STRING or a BODY would also be on `start`', + '`startEvt` stamps `url` and the full `input` (engine.ts:1071-1085), so the sink can always recover the discriminator — it is just never the span NAME', + ); + } + + finish( + 'C6', + 'A HUNDRED ROOTS BY DEFAULT; ONE TREE WITH `linked`, WHICH THEN DRAWS THE WRONG SHAPE. A list call plus 100 bare lookups measured 101 spans in 101 DISTINCT TRACES, all roots — nothing relates a lookup to the list that produced its id. `all()` produces the right shape (1 trace, a 100-wide fan, 0 member roots) and cannot carry the scenario, because its members share one input: the 100 spans named ONE distinct id. `linked` IS the answer to the capture`s question, and the answer is yes with a caveat: `ScopedRun` takes the stitch`s OWN input per call (pipe.ts:355-361), so a runtime-length fan-out of DIFFERENT ids measured 1 traceId, 1 root and 101 spans covering the list and every lookup. THE CAVEAT IS THE SHAPE. `run` chains each call under the PREVIOUS one (pipe.ts:360-367), so the same run measured DEPTH 101 and MAX FAN-OUT 1 — a 101-deep chain — while the calls were genuinely concurrent (peak 100 in flight). A trace viewer renders 100 simultaneous lookups as a sequential queue, and the depth is an artefact of call order rather than of any dependency. Two riders. `linked(...)` is a `Promise`, not a `Composable` (no `__composable` brand), confirming scenario 10 — so a fan-out written this way cannot nest inside any combinator. And every span of a one-stitch fan-out carries the SAME name (`customer`, x100): the only per-call identity a sink gets is the `url`/`input` stamped on the `start` event (engine.ts:1071-1085)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c7-ordering.ts b/docs/scenarios/proofs/n-plus-one-fanout/c7-ordering.ts new file mode 100644 index 00000000..587ca2dc --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c7-ordering.ts @@ -0,0 +1,238 @@ +// C7 — does the result order match the input order under concurrency? +// +// The join at the end of a fan-out is `orders[i]` to `customers[i]`. If the result order followed +// COMPLETION order instead, every row would be attached to the wrong customer — the quietest +// possible data-corruption bug, because nothing errors and every field is populated. +// +// MEASURED: positional, always, and the one hazard is aliasing rather than ordering. +// (a) With per-id latencies that scramble completion order completely (last id first), the +// result array is still in INPUT order. `Promise.all` is positional; nothing in the library +// changes that. +// (b) Bounded concurrency does not perturb it either. +// (c) Neither does coalescing, where 70 of 100 callers are resolved out of a shared promise. +// (d) THE HAZARD IS ALIASING, NOT ORDER. Under coalescing the joiners are handed the LEADER'S +// OBJECT — the same reference (engine.ts:1645,1662). 4 order rows sharing a customer got 4 +// references to ONE object, so mutating a joined row mutates the other three. +// (e) The same is true of a CACHE HIT: the stored value is handed out by reference, so a mutation +// persists into every later hit for the TTL. +// (f) Without a cache each caller gets its own object, so the hazard appears exactly when you +// turn on the optimisation C2 recommends. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c7-ordering.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { CacheOptions } from '../../../../packages/core/src/types'; +import { type Customer, FakeVendor, idsOf } from './fake-vendor'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { runOut } from './virtual-time'; + +const BASE = 'https://api.vendor.test'; +const HOLD = 10; + +/** Fire one lookup per order and return the results IN CALL ORDER, plus the vendor. */ +async function fanOut(opts: { + orders: number; + customers: number; + slow?: Record; + cache?: CacheOptions; + concurrency?: number; +}) { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: opts.orders, + customers: opts.customers, + holdMs: HOLD, + ...(opts.slow ? { slow: opts.slow } : {}), + }); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + clock, + ...(opts.cache ? { cache: opts.cache } : {}), + ...(opts.concurrency + ? { throttle: { concurrency: opts.concurrency } } + : {}), + }); + const ids = idsOf(vendor.orders); + const pending = ids.map((id) => call({ params: { id } }).safe()); + await runOut(clock, 60_000, 1_000); + const results = await Promise.all(pending); + return { vendor, ids, results }; +} + +/** The id each positional result actually came back with — the join's correctness, as a list. */ +const joinedIds = ( + results: readonly Awaited>['results'][number][], +): string[] => results.map((r) => (r.ok ? r.data.id : '')); + +async function main(): Promise { + heading( + 'C7 — 20 concurrent lookups whose completion order is deliberately reversed', + ); + + // ── (a) completion order reversed, result order intact ───────────────────────────────────── + // Id N is held for (21-N)·100ms, so the LAST request completes FIRST and the first completes + // last. If anything anywhere ordered by completion, this is where it would show. + { + const slow = Object.fromEntries( + Array.from({ length: 20 }, (_, i) => [ + `cust-${String(i + 1).padStart(3, '0')}`, + (20 - i) * 100, + ]), + ); + const { vendor, ids, results } = await fanOut({ + orders: 20, + customers: 20, + slow, + }); + checkSeq( + '(a) COMPLETION order at the server (first 4)', + vendor.completions.slice(0, 4), + ['cust-020', 'cust-019', 'cust-018', 'cust-017'], + ); + checkSeq( + '(a) RESULT order handed to the caller (first 4)', + joinedIds(results).slice(0, 4), + ['cust-001', 'cust-002', 'cust-003', 'cust-004'], + ); + checkSeq('(a) result order === input order', joinedIds(results), ids); + check( + '(a) …and it is the exact REVERSE of completion order', + JSON.stringify([...vendor.completions].reverse()) === + JSON.stringify(ids), + true, + ); + } + + // ── (b) bounded concurrency does not perturb it ──────────────────────────────────────────── + { + const slow = Object.fromEntries( + Array.from({ length: 20 }, (_, i) => [ + `cust-${String(i + 1).padStart(3, '0')}`, + (20 - i) * 100, + ]), + ); + const { ids, results } = await fanOut({ + orders: 20, + customers: 20, + slow, + concurrency: 4, + }); + checkSeq('(b) with `concurrency: 4`', joinedIds(results), ids); + } + + // ── (c) coalescing does not perturb it ───────────────────────────────────────────────────── + // 20 orders over 5 customers: 15 of the 20 results come out of a shared promise rather than + // their own response, and the array is still positional. + { + const { ids, results } = await fanOut({ + orders: 20, + customers: 5, + cache: { ttl: '60s' }, + }); + checkSeq('(c) with coalescing', joinedIds(results), ids); + checkSeq( + '(c) …which is 4 repeats of 5 ids', + joinedIds(results).slice(0, 6), + [ + 'cust-001', + 'cust-002', + 'cust-003', + 'cust-004', + 'cust-005', + 'cust-001', + ], + ); + } + + // ── (d) THE HAZARD: coalesced callers share ONE object ───────────────────────────────────── + // `claim.settle({ data: out.value, … })` hands every follower the leader's value by reference + // (engine.ts:1645) and `resultEvt(shared.data, …)` passes it straight through (engine.ts:1662). + { + const { results } = await fanOut({ + orders: 20, + customers: 5, + cache: { ttl: '60s' }, + }); + const rows = results.flatMap((r) => (r.ok ? [r.data] : [])); + const a = rows[0]; + const b = rows[5]; // the next order for the same customer + check('(d) same customer id', a?.id === b?.id, true); + check('(d) SAME OBJECT REFERENCE', a === b, true); + if (a) a.tier = 'mutated-by-row-0'; + check( + '(d) …so mutating row 0 changed row 5', + b?.tier, + 'mutated-by-row-0', + ); + const distinct = new Set(rows).size; + check('(d) distinct customer OBJECTS across 20 rows', distinct, 5); + note( + '(d) → the aliasing arrives WITH the optimisation', + 'a per-row `transform`, a normalise step, or anything that writes onto the joined customer now writes onto every row that shares it', + ); + } + + // ── (e) a cache HIT aliases too, and for the whole TTL ───────────────────────────────────── + { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: 4, + customers: 1, + holdMs: HOLD, + }); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + cache: { ttl: '60s' }, + clock, + }); + const one = call({ params: { id: 'cust-001' } }).safe(); + await runOut(clock, 10_000, 1_000); + const first = await one; + if (first.ok) first.data.tier = 'mutated-before-the-hit'; + const two = call({ params: { id: 'cust-001' } }).safe(); + await runOut(clock, 10_000, 1_000); + const second = await two; + + check('(e) requests made', vendor.customerRequests, 1); + check( + '(e) the SECOND call was served the mutated object', + second.ok ? second.data.tier : '', + 'mutated-before-the-hit', + ); + check( + '(e) same reference across the cache hit', + first.ok && second.ok && first.data === second.data, + true, + ); + } + + // ── (f) with no cache, every caller gets its own object ──────────────────────────────────── + { + const { results } = await fanOut({ orders: 20, customers: 5 }); + const rows = results.flatMap((r) => (r.ok ? [r.data] : [])); + check( + '(f) distinct customer OBJECTS across 20 rows', + new Set(rows).size, + 20, + ); + check( + '(f) rows 0 and 5 share an id', + rows[0]?.id === rows[5]?.id, + true, + ); + check('(f) …and are DIFFERENT objects', rows[0] === rows[5], false); + } + + finish( + 'C7', + 'ORDER IS POSITIONAL AND SAFE; THE REAL HAZARD IS ALIASING. With per-id latencies chosen so the LAST request completes FIRST — a completion order that is the exact reverse of the call order — the result array came back in INPUT order, so `orders[i]` joins to `results[i]` with no correlation key. That held under bounded concurrency and under coalescing, where 15 of 20 results are resolved out of a shared promise rather than their own response. THE FINDING IS ELSEWHERE: under coalescing every joiner is handed the LEADER`S OBJECT BY REFERENCE (`claim.settle({ data: out.value })` at engine.ts:1645, passed through at engine.ts:1662). 20 rows over 5 customers measured 5 DISTINCT OBJECTS — mutating the customer on row 0 changed row 5 — where the same fan-out with no cache measured 20 distinct objects. A cache HIT aliases the same way and for the whole TTL: a value mutated after the first call was handed unchanged to the second. So the aliasing appears exactly when you turn on the optimisation C2 recommends, and any normalise/enrich step that writes onto a joined customer will silently write onto every row sharing it', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/c8-assembled.ts b/docs/scenarios/proofs/n-plus-one-fanout/c8-assembled.ts new file mode 100644 index 00000000..1e02c29d --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/c8-assembled.ts @@ -0,0 +1,338 @@ +// C8 — the best available answer, run end to end, against the same job written by hand. +// +// 100 orders over 30 customers, one of them deleted, one of them permanently rate-limited. Both +// implementations run over the SAME fake vendor and the SAME `manualClock`, so the comparison is +// of the code, not of the wire. +// +// MEASURED: the two agree on every outcome, and StitchAPI is shorter by roughly the pool, the +// retry loop and the dedupe map — the three things the capture says you would otherwise reach for +// `p-limit` and a hand-written backoff for. +// (a) The StitchAPI answer: 1 list request, 30 customer requests for 100 orders, peak in-flight +// 8, 100 rows joined in input order, the 2 bad ids flagged and identifiable, 1 trace tree. +// (b) The hand-rolled control: the same numbers, from ~1.9x the executable lines. +// (c) THE ONE PLACE THE LIBRARY IS WORSE, and it is the failure path: a coalesced FAILURE is not +// shared (C2 e), so the deleted customer costs one request per ORDER that references it. The +// hand-rolled dedupe map shares the rejection and asks ONCE. +// (d) The control for the whole exercise: `Promise.all` with no `.safe()`, no pool and no cache +// — 0 rows, unbounded peak, and the quota spent anyway. +// +// pnpm exec tsx docs/scenarios/proofs/n-plus-one-fanout/c8-assembled.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { + type Customer, + FakeVendor, + type OrderWithCustomer, + idsOf, +} from './fake-vendor'; +import { ordersWithCustomers } from './fanout'; +import { handRolledOrdersWithCustomers } from './hand-rolled'; +import { + check, + checkPeak, + checkRequests, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { recordingSink } from './trace-probe'; +import { runOut } from './virtual-time'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const BASE = 'https://api.vendor.test'; +const ORDERS = 100; +const CUSTOMERS = 30; +const HOLD = 50; +const BOUND = 8; +/** Deleted: 4 of the 100 orders reference it (ids round-robin, so 001-010 appear 4 times). */ +const DEAD = 'cust-005'; +/** Permanently throttled: also referenced 4 times. */ +const THROTTLED = 'cust-007'; + +/** + * Executable lines — imports (however they wrap), blanks and comment-only lines removed on BOTH + * sides, so the number is the code someone actually writes and maintains. + */ +function executableLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .replace(/^import[\s\S]*?;$/gm, '') + .replace(/^export interface[\s\S]*?^}$/gm, '') // the options type, identical on both sides + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +function context() { + const clock = manualClock(); + const vendor = new FakeVendor({ + clock, + orders: ORDERS, + customers: CUSTOMERS, + holdMs: HOLD, + notFound: [DEAD], + rateLimited: [THROTTLED], + }); + return { clock, vendor }; +} + +/** How many rows carry a problem, and which ids they belong to. */ +const flagged = (rows: readonly OrderWithCustomer[]): string[] => [ + ...new Set(rows.filter((r) => r.problem !== null).map((r) => r.customerId)), +]; + +async function main(): Promise { + heading( + `C8 — ${String(ORDERS)} orders over ${String(CUSTOMERS)} customers, one deleted (${DEAD}), one throttled (${THROTTLED})`, + ); + + // ── (a) the StitchAPI answer ─────────────────────────────────────────────────────────────── + { + const { clock, vendor } = context(); + const trace = recordingSink(); + const done = ordersWithCustomers({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + concurrency: BOUND, + ttl: '60s', + attempts: 3, + trace, + }); + await runOut(clock, 120_000, 1_000); + const rows = await done; + + check('(a) list requests', vendor.listCalls, 1); + check('(a) rows joined', rows.length, ORDERS); + checkPeak('(a) peak in-flight', vendor.peakInFlight, BOUND, BOUND); + check( + '(a) requests for a HEALTHY duplicated id (4 orders reference it)', + vendor.requestsFor('cust-001'), + 1, + ); + checkSeq('(a) ids flagged as a problem', flagged(rows).sort(), [ + DEAD, + THROTTLED, + ]); + check( + '(a) rows flagged', + rows.filter((r) => r.problem !== null).length, + 8, // 4 orders for the deleted id + 4 for the throttled one + ); + check( + '(a) rows joined successfully', + rows.filter((r) => r.problem === null).length, + 92, + ); + checkSeq( + '(a) row order === order order', + rows.slice(0, 3).map((r) => r.orderId), + ['ord-0001', 'ord-0002', 'ord-0003'], + ); + check( + '(a) the deleted customer is identifiable', + rows.find((r) => r.customerId === DEAD)?.problem, + '404: HTTP 404', + ); + check('(a) DISTINCT trace trees', trace.traceIds().length, 1); + checkRequests( + '(a) total customer requests', + vendor.customerRequests, + vendor.distinctIds, + 44, + ); + checkSeq( + '(a) requests for [healthy, deleted, throttled]', + [ + vendor.requestsFor('cust-001'), + vendor.requestsFor(DEAD), + vendor.requestsFor(THROTTLED), + ], + [1, 4, 12], + ); + note( + '(a) → 28 healthy ids cost 1 request each; the 2 bad ones cost 16', + 'the coalescer collapses duplicates on the SUCCESS path only (C2 e), so each of the 4 orders naming a broken id runs its own attempts — and the throttled one runs 3 of them', + ); + } + + // ── (b) the hand-rolled control ──────────────────────────────────────────────────────────── + { + const { clock, vendor } = context(); + const done = handRolledOrdersWithCustomers({ + baseUrl: BASE, + adapter: vendor.adapter(), + clock, + concurrency: BOUND, + attempts: 3, + }); + await runOut(clock, 120_000, 1_000); + const rows = await done; + + check('(b) rows joined', rows.length, ORDERS); + checkPeak('(b) peak in-flight', vendor.peakInFlight, BOUND, BOUND); + check( + '(b) requests for a HEALTHY duplicated id', + vendor.requestsFor('cust-001'), + 1, + ); + checkSeq('(b) ids flagged as a problem', flagged(rows).sort(), [ + DEAD, + THROTTLED, + ]); + check( + '(b) rows joined successfully', + rows.filter((r) => r.problem === null).length, + 92, + ); + checkSeq( + '(b) row order === order order', + rows.slice(0, 3).map((r) => r.orderId), + ['ord-0001', 'ord-0002', 'ord-0003'], + ); + checkRequests( + '(b) total customer requests', + vendor.customerRequests, + vendor.distinctIds, + 32, + ); + checkSeq( + '(b) requests for [healthy, deleted, throttled]', + [ + vendor.requestsFor('cust-001'), + vendor.requestsFor(DEAD), + vendor.requestsFor(THROTTLED), + ], + [1, 1, 3], + ); + } + + // ── (c) where the two DIVERGE: the failure path ──────────────────────────────────────────── + // The only behavioural difference between the two implementations, and it goes against the + // library. A `Map` shares the rejection with every joiner; the engine's coalescer + // releases them to re-run (engine.ts:1646-1659). + { + const stitched = context(); + const stitchedDone = ordersWithCustomers({ + baseUrl: BASE, + adapter: stitched.vendor.adapter(), + clock: stitched.clock, + concurrency: BOUND, + ttl: '60s', + attempts: 1, // no retry, so the count is purely about coalescing + }); + await runOut(stitched.clock, 120_000, 1_000); + await stitchedDone; + + const rolled = context(); + const rolledDone = handRolledOrdersWithCustomers({ + baseUrl: BASE, + adapter: rolled.vendor.adapter(), + clock: rolled.clock, + concurrency: BOUND, + attempts: 1, + }); + await runOut(rolled.clock, 120_000, 1_000); + await rolledDone; + + check( + '(c) StitchAPI — requests for the DELETED id (4 orders reference it)', + stitched.vendor.requestsFor(DEAD), + 4, + ); + check( + '(c) hand-rolled — requests for the same id', + rolled.vendor.requestsFor(DEAD), + 1, + ); + check( + '(c) StitchAPI — total customer requests', + stitched.vendor.customerRequests, + 36, + ); + check( + '(c) hand-rolled — total customer requests', + rolled.vendor.customerRequests, + 30, + ); + note( + '(c) → 6 extra requests here; at scale it is one per duplicate REFERENCE', + 'a list where 40 of 100 orders name one deleted customer costs 40 requests for that id under coalescing and 1 under a Map', + ); + } + + // ── (d) the naive baseline, for the size of the gap ──────────────────────────────────────── + { + const { clock, vendor } = context(); + const call = stitch({ + name: 'customer', + url: `${BASE}/customers/{id}`, + adapter: vendor.adapter(), + clock, + }); + let rows = 0; + let failure = ''; + const pending = Promise.all( + idsOf(vendor.orders).map((id) => call({ params: { id } })), + ).then( + (v) => { + rows = v.length; + }, + (e: Error) => { + failure = e.message; + }, + ); + await runOut(clock, 120_000, 1_000); + await pending; + check('(d) `Promise.all(ids.map(call))` — rows joined', rows, 0); + check('(d) …rejected with', failure, 'HTTP 404'); + checkRequests( + '(d) …requests spent anyway', + vendor.customerRequests, + vendor.distinctIds, + 100, + ); + checkPeak('(d) …peak in-flight', vendor.peakInFlight, undefined, 100); + } + + // ── (e) the line count ───────────────────────────────────────────────────────────────────── + { + const mine = executableLines('fanout.ts'); + const theirs = executableLines('hand-rolled.ts'); + check( + '(e) StitchAPI answer (`fanout.ts`) — executable lines', + mine, + 45, + ); + check( + '(e) hand-rolled (`hand-rolled.ts`) — executable lines', + theirs, + 87, + ); + note( + '(e) → what became configuration', + 'the FIFO pool (~14 lines), the retry-with-jitter loop (~9), the retryable-status set, the non-2xx throw and the URL assembly', + ); + note( + '(e) → what did NOT shrink', + 'the per-row `problem` branch and the positional join are the same on both sides — partial failure is user code either way', + ); + } + + finish( + 'C8', + 'ACHIEVABLE WITH USER CODE, and the user code is the partial-failure branch. Both implementations settle the same job identically over the same fake vendor and the same virtual clock: 1 list request, 100 rows joined in input order, peak in-flight 8 against a declared 8, ONE request for each healthy id even though four orders reference it, 92 rows joined and 8 flagged, and both bad ids identifiable (`404: HTTP 404` on the deleted one). The StitchAPI answer buys concurrency, duplicate collapsing, jittered retry and a single trace tree in four config fields, in 45 EXECUTABLE LINES against the control`s 87 — the difference is exactly the FIFO pool, the retry-with-jitter loop, the retryable-status set, the non-2xx throw and the URL assembly, all of which became configuration. What did NOT shrink is the per-row `problem` branch and the positional join: partial failure is user code on both sides. THE ONE PLACE IT LOSES IS THE FAILURE PATH. Full run: 44 customer requests against the hand-rolled 32, split [healthy 1, deleted 4, throttled 12] against [1, 1, 3]. With retry off so the count is purely about coalescing: 36 against 30, and the deleted customer cost FOUR requests against ONE. A failed leader releases its joiners to re-run (engine.ts:1646-1659) where a `Map` shares the rejection, so the library spends one wasted request per duplicate REFERENCE to a broken id — exactly the shape a dead foreign key takes. The naive baseline is the size of the whole gap: `Promise.all(ids.map(call))` joined ZERO rows, spent all 100 requests anyway, and ran at peak 100 in flight', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/fake-vendor.ts b/docs/scenarios/proofs/n-plus-one-fanout/fake-vendor.ts new file mode 100644 index 00000000..945ac08a --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/fake-vendor.ts @@ -0,0 +1,270 @@ +// The vendor this scenario fans out against: a list endpoint and a per-id detail endpoint, as a +// plain `Adapter` over an injected {@link Clock}. No network, no timers of its own. +// +// Everything a claim here needs to say is a read off this object: +// +// • `GET /orders` returns N orders, each carrying a `customerId` drawn from a SMALLER pool — the +// duplicate-id shape C2 exists for (100 orders, 30 distinct customers). +// • `GET /customers/{id}` counts requests PER ID (`perId`), so "did it fetch cust-7 four times?" +// is a number rather than an impression. +// • It tracks PEAK IN-FLIGHT: the adapter increments on entry and decrements on exit, and +// `peakInFlight` is the high-water mark. This is the ONLY honest way to answer "was the +// concurrency actually bounded" — a request-count says nothing about how many were open at once. +// • Named ids can be made to 404 or 429 persistently, and `burst429(n)` rate-limits the first `n` +// customer requests whatever their id — which is how a simultaneous fan-out gets throttled as a +// BURST, so the retry clustering in C5 is measurable. +// +// The hold (`holdMs`) matters more than it looks: with a zero-duration handler every request opens +// and closes inside one microtask and the peak is trivially 1. Every concurrency measurement here +// runs with a hold, so overlap is real. +import type { + Adapter, + AdapterRequest, + Clock, +} from '../../../../packages/core/src/types'; + +/** One order from the list endpoint — the only field that matters is the foreign key. */ +export interface Order { + id: string; + customerId: string; + total: number; +} + +/** One customer, as the detail endpoint returns it. */ +export interface Customer { + id: string; + name: string; + tier: string; +} + +/** The joined row C8 assembles: the order plus whatever the customer lookup produced. */ +export interface OrderWithCustomer { + orderId: string; + customerId: string; + customerName: string | null; + /** `null` on success; the reason string on a failed lookup — the identifiability C4 measures. */ + problem: string | null; +} + +/** One recorded customer request: which id, what came back, and WHEN on the injected clock. */ +export interface CustomerCall { + id: string; + status: number; + /** Virtual ms at which the request reached the vendor — the arrival spread C5 measures. */ + at: number; + /** In-flight count INCLUDING this request, at the moment it arrived. */ + inFlight: number; +} + +export interface FakeVendorOptions { + clock: Clock; + /** How many orders the list endpoint returns. Default 100. */ + orders?: number; + /** How many DISTINCT customers those orders reference. Default = `orders` (no duplicates). */ + customers?: number; + /** Virtual ms each customer request is held open. Default 0 (no hold — peak is then trivially 1). */ + holdMs?: number; + /** + * Per-id hold override, so completion order can be made to disagree with request order — which + * is the only way to test that the result order is positional rather than incidental. + */ + slow?: Readonly>; + /** Ids that always 404 — the deleted customer C4 is about. */ + notFound?: readonly string[]; + /** Ids that always 429. */ + rateLimited?: readonly string[]; + /** + * Echo the request URL on the response, as `fetchAdapter` does (http-adapter.ts:98,111,145). + * Default true. `StitchError.url` is copied straight off `res.url` (stitch.ts `rebuildError`), + * so a transport that does not set it leaves the caller unable to say WHICH id failed from the + * error alone — which C4 measures both ways. + */ + echoUrl?: boolean; + /** + * `Retry-After` (delta-seconds) to send with every 429 — what a well-behaved vendor does. The + * engine PREFERS it over the computed backoff by default (`retry.respect !== false`, + * engine.ts:748-751), so it is the one thing that can undo `expo-jitter`. C5 measures both. + */ + retryAfter?: string; +} + +/** + * The vendor. `adapter()` is what a stitch/seam is handed; every number in this directory is read + * off `perId`, `customerCalls` or `peakInFlight`. + */ +export class FakeVendor { + /** The list the fan-out starts from. Deterministic, so every claim sees the same duplicates. */ + readonly orders: readonly Order[]; + /** Every customer request, in arrival order. */ + readonly customerCalls: CustomerCall[] = []; + /** Requests per customer id — the duplicate ledger C2 is a statement about. */ + readonly perId = new Map(); + /** How many times the LIST endpoint was called (should be 1 in every claim here). */ + listCalls = 0; + /** High-water mark of simultaneously-open customer requests. THE number C3 measures. */ + peakInFlight = 0; + + /** Ids whose responses landed, in COMPLETION order — the scrambling C7 measures against. */ + readonly completions: string[] = []; + + private readonly clock: Clock; + private readonly holdMs: number; + private readonly slow: Map; + private readonly notFound: Set; + private readonly rateLimited: Set; + private readonly echoUrl: boolean; + private readonly retryAfter: string | undefined; + private inFlight = 0; + /** Remaining requests to answer 429 regardless of id — the simultaneous-burst throttle. */ + private burstLeft = 0; + + constructor(opts: FakeVendorOptions) { + this.clock = opts.clock; + this.holdMs = opts.holdMs ?? 0; + this.slow = new Map(Object.entries(opts.slow ?? {})); + this.notFound = new Set(opts.notFound ?? []); + this.rateLimited = new Set(opts.rateLimited ?? []); + this.echoUrl = opts.echoUrl ?? true; + this.retryAfter = opts.retryAfter; + const orderCount = opts.orders ?? 100; + const customerCount = opts.customers ?? orderCount; + this.orders = Array.from({ length: orderCount }, (_, i) => ({ + id: `ord-${String(i + 1).padStart(4, '0')}`, + // Round-robin so the duplicate distribution is flat and exact: with 100 orders over 30 + // customers every id appears 3 or 4 times, and `100 / 30` is the ratio C2 reports. + customerId: `cust-${String((i % customerCount) + 1).padStart(3, '0')}`, + total: 1000 + i, + })); + } + + /** Rate-limit the next `n` customer requests whatever their id — one simultaneous burst, 429ed. */ + burst429(n: number): void { + this.burstLeft = n; + } + + /** Total customer requests that reached the server. */ + get customerRequests(): number { + return this.customerCalls.length; + } + + /** How many DISTINCT customer ids the fan-out asked for. */ + get distinctIds(): number { + return this.perId.size; + } + + /** Requests made for one id. */ + requestsFor(id: string): number { + return this.perId.get(id) ?? 0; + } + + /** The per-id request counts, descending — `[4,4,4,3,3,…]` reads as a duplicate ledger. */ + perIdCounts(): number[] { + return [...this.perId.values()].sort((a, b) => b - a); + } + + /** Arrival times of the customer requests after the first `skip` — i.e. the RETRIES. */ + arrivalsAfter(skip: number): number[] { + return this.customerCalls.slice(skip).map((c) => c.at); + } + + adapter(): Adapter { + return async (req: AdapterRequest) => { + const path = new URL(req.url).pathname; + if (path === '/orders') { + this.listCalls += 1; + return { + status: 200, + headers: {}, + body: { data: this.orders }, + }; + } + const id = path.slice('/customers/'.length); + const status = + this.burstLeft > 0 + ? ((this.burstLeft -= 1), 429) + : this.notFound.has(id) + ? 404 + : this.rateLimited.has(id) + ? 429 + : 200; + this.inFlight += 1; + if (this.inFlight > this.peakInFlight) + this.peakInFlight = this.inFlight; + // Recorded on ARRIVAL, before the hold: `at` is when the request left the process, + // which is the number every timing claim measures. + this.customerCalls.push({ + id, + status, + at: this.clock.now(), + inFlight: this.inFlight, + }); + this.perId.set(id, (this.perId.get(id) ?? 0) + 1); + const echo = this.echoUrl ? { url: req.url } : {}; + const hold = this.slow.get(id) ?? this.holdMs; + try { + if (hold > 0) await this.clock.sleep(hold); + this.completions.push(id); + if (status === 404) + return { + status, + headers: {}, + body: { error: 'customer_not_found', id }, + ...echo, + }; + if (status === 429) + return { + status, + headers: this.retryAfter + ? { 'retry-after': this.retryAfter } + : {}, + body: { error: 'rate_limited', id }, + ...echo, + }; + return { + status: 200, + headers: {}, + body: { + id, + name: `Customer ${id.slice(-3)}`, + tier: 'standard', + } satisfies Customer, + ...echo, + }; + } finally { + this.inFlight -= 1; + } + }; + } +} + +/** The ids the fan-out will ask for, in list order (WITH duplicates — that is the point). */ +export const idsOf = (orders: readonly Order[]): string[] => + orders.map((o) => o.customerId); + +// `manualClock.advance` sets `now` to a timer's EXACT due time (test-clock.ts:90), so an arrival +// recorded after a 372.4ms backoff really is 372.4 — the clock adds no quantisation of its own. +// Bucketing is therefore done here, deliberately, at a stated width: "how many retries landed in +// the same MILLISECOND" is `width = 1`, and a coarser width answers the same question at the scale +// a server's rate window actually cares about. +const bucketOf = (x: number, width: number): number => Math.floor(x / width); + +/** + * How many values in `xs` share their most-crowded bucket of `width` ms. `1` means every arrival + * is alone in its bucket (fully de-clustered); `xs.length` means the whole burst re-arrived + * together. + */ +export function largestBucket(xs: readonly number[], width = 1): number { + const buckets = new Map(); + let biggest = 0; + for (const x of xs) { + const k = bucketOf(x, width); + const n = (buckets.get(k) ?? 0) + 1; + buckets.set(k, n); + if (n > biggest) biggest = n; + } + return biggest; +} + +/** How many DISTINCT `width`-ms buckets a set of arrival times occupies — the spread, as one number. */ +export const distinctBuckets = (xs: readonly number[], width = 1): number => + new Set(xs.map((x) => bucketOf(x, width))).size; diff --git a/docs/scenarios/proofs/n-plus-one-fanout/fanout.ts b/docs/scenarios/proofs/n-plus-one-fanout/fanout.ts new file mode 100644 index 00000000..7c6e5857 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/fanout.ts @@ -0,0 +1,88 @@ +// The best available StitchAPI answer to `list -> fan out -> join`, as a caller would write it. +// +// Every decision the capture names is made here, and four of the five are made in CONFIGURATION: +// +// • CONCURRENCY — a seam-level `throttle: { concurrency }`. C3 measured this as the only +// construction that survives; the member could declare its own, but the seam bucket is what +// holds if a second member is ever added. +// • DUPLICATES — `cache: { ttl }` on the member. C2 measured 100 concurrent calls over 30 ids +// collapsing to 30 requests, and the coalescer sits OUTSIDE the throttle (engine.ts:1713-1718), +// so the 70 joiners never take a concurrency slot either. +// • THE HERD — `retry: { attempts }` and nothing else: `expo-jitter` is the default curve +// (resilience.ts:45) and C5 measured it spreading a 100-call cohort across ~98 milliseconds. +// • THE TRACE — `linked`, whose `run` takes the stitch's OWN input per call, so the list and +// all 100 lookups land in ONE trace (C6). The cost is that the trace draws a chain. +// +// The fifth — PARTIAL FAILURE — is the user code, and it is the `settle` helper plus the two +// branches of the row builder. `linked`'s `run` returns a rejecting Promise, and `all()` cannot +// take `.safe()` members (C1 f), so keeping 99 rows when one 404s is written by hand every time. +import { seam } from '../../../../packages/core/src/index'; +import { linked } from '../../../../packages/core/src/pipe'; +import type { + Adapter, + Clock, + StitchError, + TraceSink, +} from '../../../../packages/core/src/types'; +import type { Customer, Order, OrderWithCustomer } from './fake-vendor'; + +export interface FanOutOptions { + baseUrl: string; + adapter: Adapter; + clock: Clock; + /** Simultaneous open requests allowed against the vendor. */ + concurrency: number; + /** How long a customer stays cached — and, incidentally, what turns coalescing on. */ + ttl: string; + attempts: number; + trace?: TraceSink; +} + +// >>> BEGIN USER CODE +/** One order joined to its customer, or to the reason the customer could not be fetched. */ +export function ordersWithCustomers( + opts: FanOutOptions, +): Promise { + const api = seam({ + baseUrl: opts.baseUrl, + adapter: opts.adapter, + clock: opts.clock, + throttle: { concurrency: opts.concurrency }, + ...(opts.trace ? { trace: opts.trace } : {}), + }); + const listOrders = api.stitch<{ data: Order[] }>({ + name: 'orders', + path: '/orders', + }); + const fetchCustomer = api.stitch({ + name: 'customer', + path: '/customers/{id}', + cache: { ttl: opts.ttl }, + retry: { attempts: opts.attempts }, + }); + return linked(async (run) => { + const { data: orders } = await run(listOrders); + return Promise.all( + orders.map(async (order): Promise => { + const found = await run(fetchCustomer, { + params: { id: order.customerId }, + }).then( + (customer) => ({ + customerName: customer.name, + problem: null, + }), + (e: StitchError) => ({ + customerName: null, + problem: `${String(e.status ?? 'transport')}: ${e.message}`, + }), + ); + return { + orderId: order.id, + customerId: order.customerId, + ...found, + }; + }), + ); + }); +} +// <<< END USER CODE diff --git a/docs/scenarios/proofs/n-plus-one-fanout/hand-rolled.ts b/docs/scenarios/proofs/n-plus-one-fanout/hand-rolled.ts new file mode 100644 index 00000000..b300d81f --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/hand-rolled.ts @@ -0,0 +1,121 @@ +// The control: the same `list -> fan out -> join`, written the way the capture says the state of +// the art is written — `Promise.allSettled` + a hand-rolled concurrency pool + an in-flight dedupe +// map + a jittered retry loop. +// +// It runs against the SAME `Adapter` and the SAME `Clock` as `fanout.ts`, so the two are compared +// on identical wire behaviour and identical virtual time — the only difference is who wrote the +// resilience. Deliberately no dependencies: `p-limit` and `Bottleneck` are the real answer, and +// vendoring the ~15 lines they contribute is the honest way to count what the library replaces +// (an `npm i p-limit` is not zero lines, it is a dependency). +import type { + Adapter, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; +import type { Customer, Order, OrderWithCustomer } from './fake-vendor'; + +export interface HandRolledOptions { + baseUrl: string; + adapter: Adapter; + clock: Clock; + concurrency: number; + attempts: number; +} + +// >>> BEGIN USER CODE +/** Statuses worth another go — the set StitchAPI's `retry.on` defaults to (engine.ts:612). */ +const RETRYABLE = new Set([429, 502, 503, 504]); + +/** A FIFO concurrency pool: at most `limit` bodies running at once. What `p-limit` is. */ +function pool(limit: number) { + let active = 0; + const waiting: (() => void)[] = []; + return async function run(fn: () => Promise): Promise { + if (active >= limit) + await new Promise((resolve) => waiting.push(resolve)); + else active += 1; + try { + return await fn(); + } finally { + const next = waiting.shift(); + if (next) next(); + else active -= 1; + } + }; +} + +/** One request with a bounded, FULL-jitter exponential backoff on the retryable statuses. */ +async function request( + adapter: Adapter, + clock: Clock, + url: string, + attempts: number, +): Promise { + for (let attempt = 1; ; attempt += 1) { + const res = await adapter({ method: 'GET', url, headers: {} }); + if (!RETRYABLE.has(res.status) || attempt >= attempts) return res; + await clock.sleep( + Math.random() * Math.min(100 * 2 ** (attempt - 1), 10_000), + ); + } +} + +/** Non-2xx becomes a throw, so the pool's callers can be `allSettled`. */ +async function json( + adapter: Adapter, + clock: Clock, + url: string, + attempts: number, +): Promise { + const res = await request(adapter, clock, url, attempts); + if (res.status < 200 || res.status >= 300) + throw Object.assign(new Error(`HTTP ${String(res.status)}`), { + status: res.status, + }); + return res.body as T; +} + +export async function handRolledOrdersWithCustomers( + opts: HandRolledOptions, +): Promise { + const limit = pool(opts.concurrency); + const inFlight = new Map>(); + const customer = (id: string): Promise => { + const joined = inFlight.get(id); + if (joined) return joined; + const started = limit(() => + json( + opts.adapter, + opts.clock, + `${opts.baseUrl}/customers/${id}`, + opts.attempts, + ), + ); + inFlight.set(id, started); + void started.catch(() => undefined); + return started; + }; + const { data: orders } = await json<{ data: Order[] }>( + opts.adapter, + opts.clock, + `${opts.baseUrl}/orders`, + opts.attempts, + ); + const settled = await Promise.allSettled( + orders.map((o) => customer(o.customerId)), + ); + return orders.map((order, i): OrderWithCustomer => { + const row = settled[i]; + const base = { orderId: order.id, customerId: order.customerId }; + if (row === undefined || row.status === 'rejected') { + const e = row?.reason as { status?: number; message?: string }; + return { + ...base, + customerName: null, + problem: `${String(e?.status ?? 'transport')}: ${String(e?.message)}`, + }; + } + return { ...base, customerName: row.value.name, problem: null }; + }); +} +// <<< END USER CODE diff --git a/docs/scenarios/proofs/n-plus-one-fanout/harness.ts b/docs/scenarios/proofs/n-plus-one-fanout/harness.ts new file mode 100644 index 00000000..d611a683 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/harness.ts @@ -0,0 +1,158 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is REQUESTS THAT REACHED THE SERVER and PEAK IN-FLIGHT. Every claim +// here is ultimately a statement about one of those two numbers — "100 calls, 30 distinct ids, how +// many requests?" and "you asked for 8 at a time, how many were actually open?" — so both have a +// dedicated assertion that prints the comparison, not just the value. `100 requests for 30 ids` +// and `peak 100 in-flight under concurrency: 8` have to be readable out of context. +// +// Everything else follows `unconfirmed-write/harness.ts`: `check` for an exact value, `checkSeq` +// for a measured sequence, `note` for a reported-but-not-asserted number, `finish` for the verdict. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-id request spine + * (`[1,1,1,4,1]`) and the retry-arrival spine (`[100,100,100]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * THE assertion of C2: how many requests reached the server, against how many DISTINCT resources + * were asked for. + * + * Both numbers, because the pair is the finding. `100 requests for 30 distinct ids` is a fan-out + * that paid 3.3x its quota; `30 requests for 30 distinct ids` is in-flight coalescing working. A + * bare "30" says neither. + */ +export function checkRequests( + label: string, + requests: number, + distinct: number, + expected: number, +): void { + checks++; + const ok = requests === expected; + if (!ok) failures++; + const ratio = (requests / distinct).toFixed(2); + const verdict = + requests === distinct + ? ' <- one per distinct id' + : ` <- ${ratio}x the distinct-id floor`; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: REQUESTS ${String(requests)} for ${String(distinct)} distinct ids${verdict}${ + ok ? '' : ` (expected ${String(expected)})` + }`, + ); +} + +/** + * THE assertion of C3: the peak number of requests open at the server at once, against the bound + * the caller declared. + * + * Printed together for the same reason `checkRequests` prints both sides: `peak 100, declared + * bound 8` is the footgun this scenario exists to catch, and it is unreadable as a bare `100`. + * Pass `bound: undefined` for the unbounded baseline. + */ +export function checkPeak( + label: string, + peak: number, + bound: number | undefined, + expected: number, +): void { + checks++; + const ok = peak === expected; + if (!ok) failures++; + const against = + bound === undefined + ? ' (no bound declared)' + : peak <= bound + ? `, declared bound ${String(bound)} <- HELD` + : `, declared bound ${String(bound)} <- BREACHED by ${String(peak - bound)}`; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: PEAK IN-FLIGHT ${String(peak)}${against}${ + ok ? '' : ` (expected ${String(expected)})` + }`, + ); +} + +/** Assert a measured number is at most `bound`. */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (<= ${String(bound)})` : ` (expected <= ${String(bound)})`}`, + ); +} + +/** Assert a measured number is at least `bound` — the de-clustering floor in C5. */ +export function checkAtLeast( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual >= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (>= ${String(bound)})` : ` (expected >= ${String(bound)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C2'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Some claims here PASS by measuring the library doing something genuinely good (C2's in-flight + * coalescing) and some by measuring an unbounded fan-out that looked bounded, so the verdict + * statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/n-plus-one-fanout/trace-probe.ts b/docs/scenarios/proofs/n-plus-one-fanout/trace-probe.ts new file mode 100644 index 00000000..bf18477a --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/trace-probe.ts @@ -0,0 +1,110 @@ +// A {@link TraceSink} that records every event with the run identity it arrived under. +// +// C6's question is whether a 100-call fan-out is ONE tree or a hundred unrelated roots, so this +// records `(name, type, traceId, spanId, parentSpanId)` plus the `url` off each `start` — the url +// is the only thing that says WHICH id a span was for, because one stitch called 100 times emits +// 100 spans all named `customer`. +// +// The reductions the claims assert on: how many distinct trace trees, how many roots, how deep the +// deepest chain is, and how wide the widest fan is. +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; + +export interface TraceRecord { + /** The stitch's `name` — the only identity a sink gets for free. */ + name: string; + type: StitchEvent['type']; + traceId?: string; + spanId?: string; + parentSpanId?: string; + /** Present on `start` only: the resolved request URL, which carries the id. */ + url?: string; +} + +export interface RecordingSink extends TraceSink { + readonly records: TraceRecord[]; + /** Every `start` event, in order. One per call. */ + starts(): TraceRecord[]; + /** Distinct `traceId`s — 1 means the whole fan-out is one tree, 100 means a hundred roots. */ + traceIds(): string[]; + /** Starts with no `parentSpanId` — the ROOT runs. */ + roots(): TraceRecord[]; + /** + * Longest ancestor chain among the recorded spans. 1 = a flat fan; N = an N-deep chain, which + * is what `linked` produces when it is used to give each call its own input. + */ + maxDepth(): number; + /** Largest number of spans sharing one `parentSpanId` — the widest fan. */ + maxFanout(): number; + reset(): void; +} + +export function recordingSink(): RecordingSink { + const records: TraceRecord[] = []; + return { + records, + handle(event: StitchEvent, ctx: TraceContext): void { + const rec: TraceRecord = { name: ctx.name, type: event.type }; + if (ctx.traceId !== undefined) rec.traceId = ctx.traceId; + if (ctx.spanId !== undefined) rec.spanId = ctx.spanId; + if (ctx.parentSpanId !== undefined) + rec.parentSpanId = ctx.parentSpanId; + if (event.type === 'start') rec.url = event.url; + records.push(rec); + }, + starts() { + return records.filter((r) => r.type === 'start'); + }, + traceIds() { + return [ + ...new Set( + records + .map((r) => r.traceId) + .filter((t): t is string => t !== undefined), + ), + ]; + }, + roots() { + return this.starts().filter((r) => r.parentSpanId === undefined); + }, + maxDepth() { + const parent = new Map(); + for (const r of this.starts()) + if (r.spanId) parent.set(r.spanId, r.parentSpanId); + let deepest = 0; + for (const span of parent.keys()) { + let d = 1; + let cur = parent.get(span); + // The chain is walked against the KNOWN spans only: a `parentSpanId` naming a span + // that never emitted (a combinator's group run) terminates the walk. + while (cur !== undefined && parent.has(cur)) { + d += 1; + cur = parent.get(cur); + } + if (d > deepest) deepest = d; + } + return deepest; + }, + maxFanout() { + const byParent = new Map(); + let widest = 0; + for (const r of this.starts()) { + const key = r.parentSpanId ?? ''; + const n = (byParent.get(key) ?? 0) + 1; + byParent.set(key, n); + if (n > widest) widest = n; + } + return widest; + }, + reset() { + records.length = 0; + }, + }; +} + +/** The customer id a `start` record was for, read off its url. */ +export const idFromUrl = (r: TraceRecord): string => + r.url ? (r.url.split('/').pop() ?? '') : ''; diff --git a/docs/scenarios/proofs/n-plus-one-fanout/type-probe.ts b/docs/scenarios/proofs/n-plus-one-fanout/type-probe.ts new file mode 100644 index 00000000..2fe10f18 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/type-probe.ts @@ -0,0 +1,126 @@ +// Ask the COMPILER which fan-out spellings exist, instead of grepping for them. +// +// "The combinators can't express this" and "I couldn't find the spelling" are different findings, +// and only one of them is the library's problem. So the honest way to establish that there is no +// `all(stitch, inputs)`, no `map`, and no `allSettled` is to hand the compiler each candidate and +// read back its diagnostics. A line that compiles is a spelling that EXISTS; a line that doesn't is +// one the vocabulary refuses. +// +// The fixture is written to a temp dir (not into the repo) and deleted afterwards, so this leaves +// nothing behind and never lands in `prettier --check`. It imports core by ABSOLUTE path, which is +// why it can live outside the tree. +// +// `typescript` is loaded through a `require` ANCHORED AT `packages/core`, which is the workspace +// package that declares it. A bare `import ts from 'typescript'` resolves under `tsx` and NOT under +// plain Node from this directory (pnpm gives `docs/` no `node_modules`), so the bare form would be +// a script that runs one way and typechecks another. The compiler surface used is tiny, so it is +// declared structurally here rather than imported as a type. +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +/** Absolute path to core's barrel, so the fixture can be compiled from anywhere. */ +export const CORE = join(HERE, '../../../../packages/core/src/index'); +/** Absolute path to the `stitchapi/pipe` module — where the combinators live. */ +export const PIPE = join(HERE, '../../../../packages/core/src/pipe'); + +/** The slice of the TypeScript compiler API this probe uses. */ +interface TsCompiler { + readonly ScriptTarget: Record; + readonly ModuleKind: Record; + readonly ModuleResolutionKind: Record; + createProgram( + rootNames: readonly string[], + options: Record, + ): unknown; + getPreEmitDiagnostics(program: unknown): readonly { + code: number; + start?: number | undefined; + file?: + | { + fileName: string; + getLineAndCharacterOfPosition(pos: number): { line: number }; + } + | undefined; + }[]; +} + +const ts = createRequire(join(HERE, '../../../../packages/core/package.json'))( + 'typescript', +) as TsCompiler; + +export interface Candidate { + /** What a reader would call this spelling — printed in the report. */ + label: string; + /** One statement. Compiles ⇒ the spelling exists. */ + code: string; +} + +export interface ProbeResult extends Candidate { + compiles: boolean; + /** First diagnostic code, e.g. 2769 (no overload matches) or 2345 (argument not assignable). */ + diagnostic?: number; +} + +/** + * Typecheck each candidate as its own statement in one program and report which compile. + * Diagnostics are attributed by LINE, so each candidate must be a single line. + * + * The header declares the props a fan-out candidate needs — a stitch, a runtime-length id list, and + * an array of stitches built from it — so a candidate line is only ever about the COMBINATOR. + */ +export function probeSpellings( + candidates: readonly Candidate[], +): ProbeResult[] { + const dir = mkdtempSync(join(tmpdir(), 'stitch-fanout-probe-')); + const file = join(dir, 'probe.ts'); + const header = [ + `import { stitch } from ${JSON.stringify(CORE)};`, + `import { all, any, linked, race } from ${JSON.stringify(PIPE)};`, + `const one = stitch<{ id: string }>({ url: 'https://x.test/customers/{id}' });`, + `const ids: string[] = ['a', 'b'];`, + `const many = ids.map((id) => stitch<{ id: string }>({ url: \`https://x.test/customers/\${id}\` }));`, + `void [all, any, race, linked, one, ids, many];`, + ]; + try { + writeFileSync( + file, + [...header, ...candidates.map((c) => c.code)].join('\n'), + ); + const program = ts.createProgram([file], { + target: ts.ScriptTarget['ES2022'], + module: ts.ModuleKind['ESNext'], + moduleResolution: ts.ModuleResolutionKind['Bundler'], + strict: true, + noEmit: true, + skipLibCheck: true, + exactOptionalPropertyTypes: true, + noUncheckedIndexedAccess: true, + }); + const byLine = new Map(); + for (const d of ts.getPreEmitDiagnostics(program)) { + if (d.file?.fileName !== file || d.start === undefined) continue; + const { line } = d.file.getLineAndCharacterOfPosition(d.start); + if (!byLine.has(line)) byLine.set(line, d.code); + } + return candidates.map((c, i) => { + const diagnostic = byLine.get(header.length + i); + return diagnostic === undefined + ? { ...c, compiles: true } + : { ...c, compiles: false, diagnostic }; + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The spellings that compiled — the vocabulary that actually exists. */ +export const accepted = (results: readonly ProbeResult[]): string[] => + results.filter((r) => r.compiles).map((r) => r.label); + +/** The spellings the compiler refused. */ +export const rejected = (results: readonly ProbeResult[]): string[] => + results.filter((r) => !r.compiles).map((r) => r.label); diff --git a/docs/scenarios/proofs/n-plus-one-fanout/virtual-time.ts b/docs/scenarios/proofs/n-plus-one-fanout/virtual-time.ts new file mode 100644 index 00000000..119a82c2 --- /dev/null +++ b/docs/scenarios/proofs/n-plus-one-fanout/virtual-time.ts @@ -0,0 +1,56 @@ +// Driving a `manualClock` when the code under test does a little real async work. +// +// `manualClock.advance(ms)` fires every timer due before the target and drains the MICROTASK queue +// between fires. That is enough for code whose only asynchrony is the clock, and almost everything +// here is that code — the fake vendor is pure in-memory logic with no I/O. +// +// `runOut` exists for the two places it is not. A retry backoff is armed only after the failing +// attempt settles, and a coalescing follower's continuation lands on the microtask queue behind a +// leader that may itself be sleeping. Advancing in slices with a macrotask turn between them means +// a timer armed during slice N is fired by slice N+1 rather than being missed — so a claim can say +// "advance past everything" instead of hand-computing a hundred backoff schedules. +// +// SLICE SIZE IS THE MEASUREMENT RESOLUTION for C5. Arrival times are read off `clock.now()`, so a +// request whose backoff lands at 37.4ms is recorded at the first slice boundary at or after it. At +// `stepMs = 1` that is 1ms buckets, which is exactly the granularity "how many retries land in the +// same millisecond" is asking about. +import type { ManualClock } from '../../../../packages/core/src/testing'; + +// Yield one full turn of the event loop. `setImmediate` fires in the check phase and costs +// microseconds; `setTimeout(…, 0)` is clamped to ~1ms by Node and would make a deep drain slow. +const macrotask: () => Promise = + typeof setImmediate === 'function' + ? () => + new Promise((resolve) => { + setImmediate(resolve); + }) + : () => + new Promise((resolve) => { + setTimeout(resolve, 0); + }); + +/** Yield `turns` times, so anything sitting on the macrotask queue settles. */ +export async function drain(turns = 5): Promise { + for (let i = 0; i < turns; i++) await macrotask(); +} + +/** + * Advance `clock` by `totalMs` in `stepMs` slices, draining real macrotasks before the first slice + * and after every one. + * + * `stepMs` only has to be smaller than the smallest interval being measured; it does not have to + * divide anything evenly. + */ +export async function runOut( + clock: ManualClock, + totalMs: number, + stepMs = 1_000, +): Promise { + await drain(); + for (let left = totalMs; left > 0;) { + const slice = Math.min(stepMs, left); + await clock.advance(slice); + await drain(); + left -= slice; + } +} diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/README.md b/docs/scenarios/proofs/oauth2-refresh-token-rotation/README.md new file mode 100644 index 00000000..73853096 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/README.md @@ -0,0 +1,63 @@ +# Proofs — OAuth2 rotating refresh tokens under concurrency + +Runnable evidence for the claims in +[`../../oauth2-refresh-token-rotation.md`](../../oauth2-refresh-token-rotation.md). + +Every script is standalone, offline, and deterministic about the thing it measures: it injects a +fake in-memory OAuth2 provider through StitchAPI's `adapter` seams (`stitch({ adapter })` for the +resource server, `oauth2({ adapter })` for the token endpoint) and **counts token-endpoint calls +exactly**. Nothing touches the network. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c1-cold-start-single-flight.ts + +# all of them +for f in docs/scenarios/proofs/oauth2-refresh-token-rotation/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, +so they test the working tree, not the published bundle. + +## What each script establishes + +| Script | Question | Measured | +| -------------------------------- | ----------------------------------------------------------- | ----------------------------------------------------------------------------- | +| `c1-cold-start-single-flight.ts` | 20 concurrent cold callers through one `oauth2()` stitch | **1** token request | +| `c2-concurrent-401-refresh.ts` | 20 concurrent 401s on a cached token | **1** refresh when the 401s are simultaneous; **10** when they land 8ms apart | +| `c3-two-workers-shared-store.ts` | two workers sharing `store` + `key`, both cold, concurrent | **2** token requests — the store is a cache, not a lock | +| `c4-params-escape-hatch.ts` | can `oauth2()` do `grant_type=refresh_token` with rotation? | **No.** Redemption #2 replays the consumed token and the family is revoked | +| `c5-custom-auth-strategy.ts` | can a user write it as a custom `AuthStrategy`? | **Yes**, 79 lines, 0 replays | +| `c6-cross-process-lock.ts` | can user code close C3's cross-process gap? | **Yes**, +42 lines on top of C5, using `increment()` + `set(key, undefined)` | +| `c7-cookiesession-hooks.ts` | is `cookieSession.onRefresh` a seam for rotating state? | **No** — the hook gets `{ ok, status }`; the login body never reaches it | + +## Files + +- `fake-provider.ts` — the Atlassian/Asana-style provider: single-use rotating refresh token, + replay detection, token-family revocation. Exposes both endpoints as `Adapter`s and records + every call. +- `harness.ts` — `check` / `note` / `heading` / `finish`. No test framework. +- `rotating-refresh-strategy.ts` — **user code** for C5: rotation + durable persistence + + in-process single-flight. +- `locked-refresh-strategy.ts` — **user code** for C6: the above plus a store-backed lock scoped + to the connected account. + +## Reading the numbers honestly + +- **C2b's "10 refreshes" is not flaky-looking noise, it is the shape of the thing.** `singleFlight` + coalesces callers that arrive _while a redemption is in flight_. Callers whose 401 lands after + that window start a new one. The exact count depends on the stagger (8ms) versus the token-request + latency (10ms); the assertions only claim `1 < refreshes < N`. +- **C6 shares one `memoryStore` in one process.** It proves the lock _logic_ is expressible with the + primitives StitchAPI exposes. It does not prove `memoryStore` is a distributed lock — + `verifyStoreContract` only requires `increment` to be atomic _within_ a process + (`packages/core/src/testing.ts:160`). A real deployment needs a backend whose increment is atomic + across processes (Redis `INCR`). +- **C4b/C4c show a hack that works, in one process only.** A `params` getter plus a response-capturing + `adapter` does rotate correctly when redemptions are serialised. C4d shows it revoking the account + across two workers, and C4e shows why it cannot be fixed: a `params` value must be produced + synchronously, and every `StitchStore` read is async. diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c1-cold-start-single-flight.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c1-cold-start-single-flight.ts new file mode 100644 index 00000000..8f694014 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c1-cold-start-single-flight.ts @@ -0,0 +1,62 @@ +// C1 — N concurrent calls through ONE `oauth2()` stitch, cold cache: how many token requests fire? +// +// `oauth2()` builds one `singleFlight` per strategy instance (auth.ts:479) and `tokenFor` +// (auth.ts:572) routes a cache miss through it, so every concurrent caller of the same key should +// await ONE shared token fetch. This measures it. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c1-cold-start-single-flight.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note, okCount } from './harness'; + +const N = 20; + +async function main(): Promise { + heading( + `C1 — ${N} concurrent calls, cold token cache, one oauth2() stitch`, + ); + + const provider = new FakeRotatingProvider(); + const api = stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + // The resource server is slow enough that all N calls are genuinely in flight together. + adapter: provider.resourceAdapter({ delayMs: 10 }), + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + adapter: provider.tokenAdapter({ delayMs: 10 }), + }), + }); + + const results = await Promise.all(Array.from({ length: N }, () => api())); + + check('token endpoint calls', provider.tokenCalls, 1); + check('resource calls', provider.resourceRequests.length, N); + check('calls that succeeded', okCount(results), N); + check( + 'distinct bearer tokens sent to the resource server', + new Set(provider.resourceRequests).size, + 1, + ); + note('grant sent', provider.tokenRequests[0]?.grant_type); + + // Control: the same N calls with NO overlap (each awaited) must also fire exactly one token + // request — that path is the store cache, not single-flight. + const before = provider.tokenCalls; + for (let i = 0; i < 5; i++) await api(); + check( + 'extra token calls when serialised (cache hit)', + provider.tokenCalls - before, + 0, + ); + + finish( + 'C1', + `${N} concurrent cold callers coalesce into ONE token request`, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c2-concurrent-401-refresh.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c2-concurrent-401-refresh.ts new file mode 100644 index 00000000..8165e6f7 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c2-concurrent-401-refresh.ts @@ -0,0 +1,103 @@ +// C2 — a CACHED token the resource server now rejects with 401, hit by N concurrent calls. +// How many refreshes fire: 1, or N? +// +// The engine forces at most one refresh per call (engine.ts:707-725) and `oauth2().refresh` +// routes through the same per-strategy `singleFlight` (auth.ts:599-604). Phase A measures the +// simultaneous case. Phase B staggers WHEN each 401 lands, because single-flight only coalesces +// callers that arrive while a redemption is still in flight — that window is the real boundary. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c2-concurrent-401-refresh.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { Adapter, Stitch } from '../../../../packages/core/src/types'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note, okCount } from './harness'; + +const N = 20; + +const build = (provider: FakeRotatingProvider, resource: Adapter): Stitch => + stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + adapter: resource, + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + adapter: provider.tokenAdapter({ delayMs: 10 }), + }), + }); + +/** Phase A: every 401 lands at the same moment. */ +async function simultaneous(): Promise { + heading(`C2a — ${N} concurrent calls whose 401s land SIMULTANEOUSLY`); + const provider = new FakeRotatingProvider(); + const api = build(provider, provider.resourceAdapter({ delayMs: 10 })); + + await api(); // prime the cache — one token fetch, one 200 + const primed = provider.tokenCalls; + provider.expireCurrentAccessToken(); // the resource server now 401s the cached token + + const results = await Promise.all(Array.from({ length: N }, () => api())); + + check('token calls during priming', primed, 1); + check( + 'refreshes triggered by N concurrent 401s', + provider.tokenCalls - primed, + 1, + ); + check('calls that recovered', okCount(results), N); + check( + 'resource hits (N stale + N retried)', + provider.resourceRequests.length, + 1 + N * 2, + ); +} + +/** + * Phase B: all N callers send the stale token at once, but their 401s come back SPREAD OVER TIME + * (call i responds after i*8ms). A refresh takes ~10ms, so late arrivals miss the in-flight window. + */ +async function staggered(): Promise { + heading( + `C2b — ${N} concurrent calls whose 401s land STAGGERED (8ms apart)`, + ); + const provider = new FakeRotatingProvider(); + const inner = provider.resourceAdapter(); + let i = 0; + const staggeredResource: Adapter = async (req) => { + const wait = i++ * 8; + await new Promise((r) => setTimeout(r, wait)); + return inner(req); + }; + const api = build(provider, staggeredResource); + + await api(); + const primed = provider.tokenCalls; + provider.expireCurrentAccessToken(); + + const results = await Promise.all(Array.from({ length: N }, () => api())); + const refreshes = provider.tokenCalls - primed; + + note('refreshes triggered by N staggered 401s', refreshes); + note('calls that recovered', okCount(results)); + // The claim under test is only that it is not N — one per caller. Anything above 1 is the + // honest cost of a time-windowed coalesce. + check('refreshes < N (not one per caller)', refreshes < N, true); + check( + 'refreshes > 1 (coalescing is time-windowed, not identity-scoped)', + refreshes > 1, + true, + ); +} + +async function main(): Promise { + await simultaneous(); + await staggered(); + finish( + 'C2', + 'simultaneous 401s coalesce to ONE refresh; staggered 401s do not (measured above)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c3-two-workers-shared-store.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c3-two-workers-shared-store.ts new file mode 100644 index 00000000..b6fa8836 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c3-two-workers-shared-store.ts @@ -0,0 +1,142 @@ +// C3 — TWO independently constructed stitches sharing one `store` + `key` (two worker processes), +// both cold, called concurrently. One token request between them, or two? +// +// HOW THIS MODELS TWO PROCESSES, precisely: +// `oauth2()` closes over a FRESH `singleFlight()` per call (auth.ts:479). Two separate +// `oauth2({...})` invocations therefore have two separate in-flight maps and share NOTHING in +// process memory. The only object they share is the `StitchStore` — which is exactly what a +// shared Redis is to two pods. Phase A is the control that proves the difference is real: give +// both stitches the SAME strategy object (one process) and the count changes. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c3-two-workers-shared-store.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { + AuthStrategy, + StitchStore, +} from '../../../../packages/core/src/types'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note } from './harness'; + +const KEY = 'acct-42'; // the connected account — the correct scope of mutual exclusion +const VAULT_KEY = `vault:oauth2:${KEY}`; // vaultView() prefix + oauth2 baseKey (auth.ts:475, store.ts:71) + +const newStrategy = (provider: FakeRotatingProvider): AuthStrategy => + oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + adapter: provider.tokenAdapter({ delayMs: 15 }), + }); + +const worker = ( + provider: FakeRotatingProvider, + store: StitchStore, + auth: AuthStrategy, +) => + stitch({ + url: 'https://api.example.com/issues', + store, + adapter: provider.resourceAdapter({ delayMs: 5 }), + auth, + }); + +/** Control: ONE process — two stitches, one shared strategy object, one shared store. */ +async function oneProcess(): Promise { + heading( + 'C3a — CONTROL: two stitches sharing ONE oauth2() object (one process)', + ); + const provider = new FakeRotatingProvider(); + const store = memoryStore(); + const shared = newStrategy(provider); + const a = worker(provider, store, shared); + const b = worker(provider, store, shared); + + await Promise.all([a(), b()]); + check('token calls (shared singleFlight)', provider.tokenCalls, 1); +} + +/** The real question: TWO processes — two oauth2() objects, same store + same key, both cold. */ +async function twoProcessesConcurrent(): Promise { + heading( + 'C3b — TWO independently constructed oauth2() objects, shared store + key, cold, concurrent', + ); + const provider = new FakeRotatingProvider(); + const store = memoryStore(); + const a = worker(provider, store, newStrategy(provider)); // "pod A" + const b = worker(provider, store, newStrategy(provider)); // "pod B" + + check( + 'store is cold before the call', + await store.get(VAULT_KEY), + undefined, + ); + await Promise.all([a(), b()]); + + note('token calls', provider.tokenCalls); + check('token calls between two cold workers', provider.tokenCalls, 2); + // Prove the "2" is a missing LOCK, not a key mismatch: both wrote the same vault key. + const cached = (await store.get(VAULT_KEY)) as + { token: string } | undefined; + check('both workers used the same vault key', cached !== undefined, true); + note('vault key', VAULT_KEY); + note('token cached under it', cached?.token); +} + +/** Sequential: worker A finishes before worker B starts. Does the store serve B? */ +async function twoProcessesSequential(): Promise { + heading( + 'C3c — the same two workers, but SEQUENTIAL (A completes, then B starts)', + ); + const provider = new FakeRotatingProvider(); + const store = memoryStore(); + const a = worker(provider, store, newStrategy(provider)); + const b = worker(provider, store, newStrategy(provider)); + + await a(); + await b(); + check( + 'token calls when the write lands before B reads', + provider.tokenCalls, + 1, + ); +} + +/** Scale: does the count grow with callers, or with workers? */ +async function scale(): Promise { + heading( + 'C3d — 2 workers x 10 concurrent calls each, shared store + key, cold', + ); + const provider = new FakeRotatingProvider(); + const store = memoryStore(); + const a = worker(provider, store, newStrategy(provider)); + const b = worker(provider, store, newStrategy(provider)); + + await Promise.all([ + ...Array.from({ length: 10 }, () => a()), + ...Array.from({ length: 10 }, () => b()), + ]); + note( + 'token calls for 20 concurrent callers across 2 workers', + provider.tokenCalls, + ); + check( + 'token calls == number of workers, not number of callers', + provider.tokenCalls, + 2, + ); +} + +async function main(): Promise { + await oneProcess(); + await twoProcessesConcurrent(); + await twoProcessesSequential(); + await scale(); + finish( + 'C3', + 'the shared store is a CACHE, not a lock: concurrent cold workers fire one token request EACH', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c4-params-escape-hatch.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c4-params-escape-hatch.ts new file mode 100644 index 00000000..97b131a4 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c4-params-escape-hatch.ts @@ -0,0 +1,359 @@ +// C4 — can `oauth2()` run `grant_type=refresh_token` where the RESPONSE carries a new +// `refresh_token` that must be used for the NEXT refresh? +// +// `params` (auth.ts:393) is merged into the token-request body (auth.ts:518) and can override +// `grant_type`, so the REQUEST half is expressible. The response half is the question: `fetchToken` +// reads only `access_token` and `expires_in` (auth.ts:549-552) and caches `{token, expiresAt}` +// (auth.ts:562-568). This measures what actually goes on the wire on the SECOND redemption. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c4-params-escape-hatch.ts +import { oauth2 } from '../../../../packages/core/src/auth'; +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { Adapter, StitchStore } from '../../../../packages/core/src/types'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note } from './harness'; + +const KEY = 'acct-42'; +const VAULT_KEY = `vault:oauth2:${KEY}`; + +/** C4a — the documented escape hatch, used exactly as documented: a STATIC params record. */ +async function staticParams(): Promise { + heading( + 'C4a — params as a static Record (the documented shape)', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store: StitchStore = memoryStore(); + const api = stitch({ + url: 'https://api.example.com/issues', + store, + adapter: provider.resourceAdapter(), + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + // The escape hatch: override the grant and carry the stored refresh token. + params: { grant_type: 'refresh_token', refresh_token: 'RT-0' }, + adapter: provider.tokenAdapter(), + }), + }); + + // First redemption: this WORKS. The request half of the grant is expressible. + await api(); + check('1st redemption succeeded', provider.tokenCalls, 1); + check('grant sent', provider.tokenRequests[0]?.grant_type, 'refresh_token'); + check( + 'refresh token sent', + provider.tokenRequests[0]?.refresh_token, + 'RT-0', + ); + check( + 'provider rotated to a NEW refresh token', + provider.activeRefreshToken, + 'RT-1', + ); + + // What did StitchAPI keep from that response? Only the access token — the rotated + // `refresh_token` is dropped on the floor. + const cached = (await store.get(VAULT_KEY)) as Record; + note('cached vault entry', JSON.stringify(cached)); + check( + 'rotated refresh_token persisted anywhere by oauth2()', + 'refresh_token' in cached, + false, + ); + check( + 'vault entry fields', + Object.keys(cached).sort().join(','), + 'expiresAt,token', + ); + + // Second redemption, forced by a 401. `params` is static, so RT-0 goes out AGAIN. + provider.expireCurrentAccessToken(); + let threw = false; + try { + await api(); + } catch { + threw = true; + } + + check( + '2nd redemption presented the SAME (consumed) token', + provider.tokenRequests[1]?.refresh_token, + 'RT-0', + ); + check( + 'rotated RT-1 was never sent', + provider.tokenRequests.some((r) => r.refresh_token === 'RT-1'), + false, + ); + check('replay detections', provider.replayDetections, 1); + check( + 'token family revoked (account is dead)', + provider.familyRevoked, + true, + ); + check('the call failed', threw, true); +} + +/** + * C4b — the only way to close the loop with `oauth2()`: make `params` DYNAMIC with a getter (an + * object with a getter still satisfies `Record`, and `Object.assign` invokes it on + * every token request), and capture the rotated token out of the response with a wrapping `adapter`. + * Both are user-authored side channels; nothing in the option surface carries the value. + */ +async function getterParamsPlusCapturingAdapter(): Promise { + heading('C4b — dynamic params (getter) + a user-written capturing adapter'); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const persisted: StitchStore = memoryStore(); + await persisted.set('refresh:acct-42', 'RT-0'); + let current = 'RT-0'; + + const inner = provider.tokenAdapter(); + const capturing: Adapter = async (req) => { + const res = await inner(req); + const body = res.body as { refresh_token?: string }; + // Persist the rotated token BEFORE it is used. (The old one is already spent server-side by + // the time this line runs — write-before-use is the best a client can do.) + if (res.status < 400 && body?.refresh_token) { + current = body.refresh_token; + await persisted.set('refresh:acct-42', current); + } + return res; + }; + + const api = stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + adapter: provider.resourceAdapter(), + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + params: { + grant_type: 'refresh_token', + get refresh_token(): string { + return current; + }, + }, + adapter: capturing, + }), + }); + + // Five sequential rotations. + await api(); + for (let i = 0; i < 4; i++) { + provider.expireCurrentAccessToken(); + await api(); + } + + check('redemptions performed', provider.tokenCalls, 5); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); + check( + 'every redemption sent a DISTINCT refresh token', + new Set(provider.tokenRequests.map((r) => r.refresh_token)).size, + 5, + ); + check( + 'durably persisted token matches the provider', + await persisted.get('refresh:acct-42'), + provider.activeRefreshToken, + ); +} + +/** + * C4c — the C4b hack under STAGGERED concurrent 401s in ONE process (the C2b race that produced + * 10 refreshes). It SURVIVES: `singleFlight` serialises redemptions per key, and the capturing + * adapter updates `current` before the flight settles, so no two redemptions ever read the same + * value. This is measured, not assumed — the in-process story is genuinely safe. + */ +async function getterParamsInProcessConcurrency(): Promise { + heading( + 'C4c — the C4b hack under STAGGERED concurrent 401s, ONE process (20 callers)', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + let current = 'RT-0'; + const innerToken = provider.tokenAdapter({ delayMs: 10 }); + const capturing: Adapter = async (req) => { + const res = await innerToken(req); + const body = res.body as { refresh_token?: string }; + if (res.status < 400 && body?.refresh_token) + current = body.refresh_token; + return res; + }; + const innerResource = provider.resourceAdapter(); + let i = 0; + const staggered: Adapter = async (req) => { + await new Promise((r) => setTimeout(r, i++ * 8)); + return innerResource(req); + }; + + const api = stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + adapter: staggered, + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + params: { + grant_type: 'refresh_token', + get refresh_token(): string { + return current; + }, + }, + adapter: capturing, + }), + }); + + await api(); // prime + provider.expireCurrentAccessToken(); + const settled = await Promise.allSettled( + Array.from({ length: 20 }, () => api()), + ); + + note('redemptions attempted', provider.tokenCalls); + note( + 'calls rejected', + settled.filter((s) => s.status === 'rejected').length, + ); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); +} + +/** + * C4d — the same hack across TWO workers (two `oauth2()` objects, shared store + key — the C3 + * setup). Each worker's getter can only read its OWN process memory, so both present RT-0. + */ +async function getterParamsTwoWorkers(): Promise { + heading( + 'C4d — the C4b hack across TWO workers (shared store + key, cold, concurrent)', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store: StitchStore = memoryStore(); // the shared Redis both pods see + await store.set('refresh:acct-42', 'RT-0'); + + // Each "process" gets its own in-memory mirror of the persisted token, seeded from the store. + const makeWorker = () => { + let current = 'RT-0'; + const innerToken = provider.tokenAdapter({ delayMs: 10 }); + const capturing: Adapter = async (req) => { + const res = await innerToken(req); + const body = res.body as { refresh_token?: string }; + if (res.status < 400 && body?.refresh_token) { + current = body.refresh_token; + await store.set('refresh:acct-42', current); + } + return res; + }; + return stitch({ + url: 'https://api.example.com/issues', + store, + adapter: provider.resourceAdapter({ delayMs: 5 }), + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + params: { + grant_type: 'refresh_token', + get refresh_token(): string { + return current; + }, + }, + adapter: capturing, + }), + }); + }; + + const a = makeWorker(); + const b = makeWorker(); + const settled = await Promise.allSettled([a(), b()]); + + note('redemptions attempted', provider.tokenCalls); + note( + 'refresh tokens presented', + provider.tokenRequests.map((r) => r.refresh_token).join(', '), + ); + note( + 'calls rejected', + settled.filter((s) => s.status === 'rejected').length, + ); + check('replay detections', provider.replayDetections, 1); + check( + 'token family revoked (account is dead)', + provider.familyRevoked, + true, + ); +} + +/** + * C4e — why a worker cannot fix C4d by reading the SHARED store inside the getter: a `params` value + * must be produced SYNCHRONOUSLY (`Record`), and every `StitchStore` verb is async. + * Returning the promise instead puts a `Promise` on the wire. + */ +async function getterCannotBeAsync(): Promise { + heading( + 'C4e — a params getter cannot read the shared store (StitchStore is async)', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store: StitchStore = memoryStore(); + await store.set('refresh:acct-42', 'RT-0'); + + const api = stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + adapter: provider.resourceAdapter(), + auth: oauth2({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + params: { + grant_type: 'refresh_token', + get refresh_token(): string { + // The only durable/shared read StitchAPI offers is `store.get`, which is a + // Promise. The cast is the lie a user would have to write to make it compile. + return store.get('refresh:acct-42') as unknown as string; + }, + }, + adapter: provider.tokenAdapter(), + }), + }); + + await Promise.allSettled([api()]); + const sent = provider.tokenRequests[0]?.refresh_token as unknown; + note( + 'value that reached the token endpoint', + Object.prototype.toString.call(sent), + ); + check( + 'the shared-store read arrived as a Promise, not a token', + sent instanceof Promise, + true, + ); + check( + 'provider accepted it', + provider.tokenCalls > 0 && + provider.replayDetections === 0 && + provider.currentAccessToken !== undefined, + false, + ); +} + +async function main(): Promise { + await staticParams(); + await getterParamsPlusCapturingAdapter(); + await getterParamsInProcessConcurrency(); + await getterParamsTwoWorkers(); + await getterCannotBeAsync(); + finish( + 'C4', + 'oauth2() CANNOT do rotation as configured (C4a: replay on redemption #2). A getter+adapter hack rotates safely in ONE process (C4b/C4c) but revokes the family across two (C4d), and cannot be fixed because a params getter is synchronous (C4e)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c5-custom-auth-strategy.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c5-custom-auth-strategy.ts new file mode 100644 index 00000000..24f67180 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c5-custom-auth-strategy.ts @@ -0,0 +1,195 @@ +// C5 — since `oauth2()` cannot do rotation (C4), can a user implement it as a custom +// `AuthStrategy`? This runs the strategy in ./rotating-refresh-strategy.ts against the same fake +// provider and checks the three properties that matter: rotation persisted, concurrent callers +// coalesced, replay never occurring. It also prints the SIZE of that user code. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c5-custom-auth-strategy.ts +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { + Adapter, + Stitch, + StitchStore, +} from '../../../../packages/core/src/types'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note, okCount } from './harness'; +import { rotatingRefresh } from './rotating-refresh-strategy'; + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const KEY = 'acct-42'; +const REFRESH_VAULT_KEY = `vault:rr:${KEY}:refresh`; +const N = 20; + +const build = ( + provider: FakeRotatingProvider, + store: StitchStore, + resource: Adapter, + tokenDelayMs = 10, +): Stitch => + stitch({ + url: 'https://api.example.com/issues', + store, + adapter: resource, + auth: rotatingRefresh({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + seedRefreshToken: 'RT-0', + adapter: provider.tokenAdapter({ delayMs: tokenDelayMs }), + }), + }); + +/** A: N concurrent cold callers — one redemption, and the rotated token lands in the vault. */ +async function coldStart(): Promise { + heading(`C5a — ${N} concurrent cold callers`); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store = memoryStore(); + const api = build( + provider, + store, + provider.resourceAdapter({ delayMs: 10 }), + ); + + const results = await Promise.all(Array.from({ length: N }, () => api())); + + check('redemptions', provider.tokenCalls, 1); + check('grant sent', provider.tokenRequests[0]?.grant_type, 'refresh_token'); + check('calls that succeeded', okCount(results), N); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); + check( + 'ROTATED refresh token durably persisted in the vault', + await store.get(REFRESH_VAULT_KEY), + provider.activeRefreshToken, + ); + note('persisted token', await store.get(REFRESH_VAULT_KEY)); +} + +/** B: N simultaneous 401s on a cached token — one redemption, no replay. */ +async function simultaneous401(): Promise { + heading(`C5b — ${N} concurrent calls whose 401s land SIMULTANEOUSLY`); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const api = build( + provider, + memoryStore(), + provider.resourceAdapter({ delayMs: 10 }), + ); + + await api(); + const primed = provider.tokenCalls; + provider.expireCurrentAccessToken(); + const results = await Promise.all(Array.from({ length: N }, () => api())); + + check( + 'redemptions triggered by N concurrent 401s', + provider.tokenCalls - primed, + 1, + ); + check('calls that recovered', okCount(results), N); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); +} + +/** C: the C2b race that defeats coalescing — churn is expected; a REPLAY is not. */ +async function staggered401(): Promise { + heading( + `C5c — ${N} concurrent calls whose 401s land STAGGERED (8ms apart)`, + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const inner = provider.resourceAdapter(); + let i = 0; + const staggered: Adapter = async (req) => { + await new Promise((r) => setTimeout(r, i++ * 8)); + return inner(req); + }; + const store = memoryStore(); + const api = build(provider, store, staggered); + + await api(); + const primed = provider.tokenCalls; + provider.expireCurrentAccessToken(); + const settled = await Promise.allSettled( + Array.from({ length: N }, () => api()), + ); + + note( + 'redemptions (extra rotations are churn, not corruption)', + provider.tokenCalls - primed, + ); + note( + 'calls rejected', + settled.filter((s) => s.status === 'rejected').length, + ); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); + check( + 'persisted token still matches the provider', + await store.get(REFRESH_VAULT_KEY), + provider.activeRefreshToken, + ); +} + +/** D: many sequential rotations — the token chain never desynchronises. */ +async function sequentialRotations(): Promise { + heading('C5d — 10 sequential rotations'); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store = memoryStore(); + const api = build(provider, store, provider.resourceAdapter(), 0); + + await api(); + for (let i = 0; i < 9; i++) { + provider.expireCurrentAccessToken(); + await api(); + } + + check('redemptions', provider.tokenCalls, 10); + check( + 'every redemption sent a DISTINCT refresh token', + new Set(provider.tokenRequests.map((r) => r.refresh_token)).size, + 10, + ); + check('replay detections', provider.replayDetections, 0); + check( + 'persisted token matches the provider', + await store.get(REFRESH_VAULT_KEY), + provider.activeRefreshToken, + ); +} + +/** How much user code was that? Reported so the "achievable but awkward" verdict has a number. */ +function sizeOfUserCode(): void { + heading('C5e — size of the user-authored strategy'); + const src = readFileSync( + join(__dirname, 'rotating-refresh-strategy.ts'), + 'utf8', + ); + const lines = src.split('\n'); + const code = lines.filter((l) => { + const t = l.trim(); + return ( + t !== '' && + !t.startsWith('//') && + !t.startsWith('*') && + !t.startsWith('/*') + ); + }); + note('total lines (with comments + types)', lines.length); + note('non-blank, non-comment lines', code.length); + check('it fits in one file a reviewer can read', code.length < 100, true); +} + +async function main(): Promise { + await coldStart(); + await simultaneous401(); + await staggered401(); + await sequentialRotations(); + sizeOfUserCode(); + finish( + 'C5', + 'a custom AuthStrategy DOES implement rotation + durability + in-process single-flight with zero replays', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c6-cross-process-lock.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c6-cross-process-lock.ts new file mode 100644 index 00000000..5656cd33 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c6-cross-process-lock.ts @@ -0,0 +1,154 @@ +// C6 (extension of C5) — C3 measured that a shared `store` is a CACHE, not a lock. Can a user +// close that gap themselves with only the `StitchStore` primitives StitchAPI exposes? +// +// Same two-worker model as C3: TWO independently constructed strategy objects (two separate +// in-process single-flights), sharing one store + one key. +// +// HONEST LIMIT of this simulation: the two "workers" share one `memoryStore` inside one Node +// process, so the atomicity the lock relies on is the in-process atomicity `verifyStoreContract` +// guarantees (testing.ts:291). A real deployment needs a backend whose `increment` is atomic +// across processes. What this proves is that the LOGIC is expressible with the primitives on +// offer — not that `memoryStore` is a distributed lock. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c6-cross-process-lock.ts +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { Stitch, StitchStore } from '../../../../packages/core/src/types'; +import { FakeRotatingProvider } from './fake-provider'; +import { check, finish, heading, note, okCount } from './harness'; +import { lockedRotatingRefresh } from './locked-refresh-strategy'; + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const KEY = 'acct-42'; +const REFRESH_VAULT_KEY = `vault:lr:${KEY}:refresh`; +const LOCK_VAULT_KEY = `vault:lr:${KEY}:lock`; + +/** One "worker process": its own strategy object (own single-flight), the shared store. */ +const makeWorker = ( + provider: FakeRotatingProvider, + store: StitchStore, +): Stitch => + stitch({ + url: 'https://api.example.com/issues', + store, + adapter: provider.resourceAdapter({ delayMs: 5 }), + auth: lockedRotatingRefresh({ + tokenUrl: 'https://auth.example.com/token', + clientId: 'cid', + clientSecret: 'csecret', + key: KEY, + seedRefreshToken: 'RT-0', + adapter: provider.tokenAdapter({ delayMs: 15 }), + pollMs: 2, + }), + }); + +/** A: the exact C3b setup — two cold workers, concurrent. C3 measured 2; the lock should give 1. */ +async function twoColdWorkers(): Promise { + heading( + 'C6a — two independently constructed workers, shared store + key, cold, concurrent', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store = memoryStore(); + const a = makeWorker(provider, store); + const b = makeWorker(provider, store); + + const results = await Promise.all([a(), b()]); + + check('token requests between two cold workers', provider.tokenCalls, 1); + check('calls that succeeded', okCount(results), 2); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); + check( + 'rotated token persisted', + await store.get(REFRESH_VAULT_KEY), + provider.activeRefreshToken, + ); + check('lock released', await store.get(LOCK_VAULT_KEY), undefined); +} + +/** B: scale — 3 workers x 10 concurrent callers each. */ +async function threeWorkersManyCallers(): Promise { + heading('C6b — 3 workers x 10 concurrent callers each, cold'); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store = memoryStore(); + const workers = [0, 1, 2].map(() => makeWorker(provider, store)); + + const results = await Promise.all( + workers.flatMap((w) => Array.from({ length: 10 }, () => w())), + ); + + note('token requests for 30 callers across 3 workers', provider.tokenCalls); + check('token requests', provider.tokenCalls, 1); + check('calls that succeeded', okCount(results), 30); + check('replay detections', provider.replayDetections, 0); +} + +/** C: the dangerous case — every worker's cached token 401s at the same instant. */ +async function simultaneous401AcrossWorkers(): Promise { + heading( + 'C6c — 3 workers x 10 callers, all holding a token the server now rejects', + ); + const provider = new FakeRotatingProvider({ refreshToken: 'RT-0' }); + const store = memoryStore(); + const workers = [0, 1, 2].map(() => makeWorker(provider, store)); + + await workers[0]!(); // prime one shared token + const primed = provider.tokenCalls; + provider.expireCurrentAccessToken(); + + const settled = await Promise.allSettled( + workers.flatMap((w) => Array.from({ length: 10 }, () => w())), + ); + + note('redemptions triggered', provider.tokenCalls - primed); + note( + 'calls rejected', + settled.filter((s) => s.status === 'rejected').length, + ); + check('redemptions', provider.tokenCalls - primed, 1); + check( + 'calls that recovered', + settled.filter((s) => s.status === 'fulfilled').length, + 30, + ); + check('replay detections', provider.replayDetections, 0); + check('token family alive', provider.familyRevoked, false); +} + +/** D: how much user code the lock adds on top of C5's strategy. */ +function sizeOfUserCode(): void { + heading('C6d — size of the user-authored locked strategy'); + const count = (file: string): number => + readFileSync(join(__dirname, file), 'utf8') + .split('\n') + .filter((l) => { + const t = l.trim(); + return ( + t !== '' && + !t.startsWith('//') && + !t.startsWith('*') && + !t.startsWith('/*') + ); + }).length; + const plain = count('rotating-refresh-strategy.ts'); + const locked = count('locked-refresh-strategy.ts'); + note('C5 strategy, non-blank non-comment lines', plain); + note('C6 locked strategy, non-blank non-comment lines', locked); + note('lines the cross-process lock adds', locked - plain); + check('still one reviewable file', locked < 150, true); +} + +async function main(): Promise { + await twoColdWorkers(); + await threeWorkersManyCallers(); + await simultaneous401AcrossWorkers(); + sizeOfUserCode(); + finish( + 'C6', + 'a store-backed lock built from increment()+set(undefined) collapses N workers to ONE redemption with zero replays', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/c7-cookiesession-hooks.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c7-cookiesession-hooks.ts new file mode 100644 index 00000000..5b6c8ae4 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/c7-cookiesession-hooks.ts @@ -0,0 +1,62 @@ +// C7 — the research capture wondered whether `cookieSession`'s `onRefresh` / `RefreshResult` +// hooks are "a richer seam for carrying rotating state". They are not, and this measures it: the +// login response here carries BOTH a `Set-Cookie` and a rotated `refresh_token` in its JSON body, +// and we record exactly what the host hook is handed. +// +// pnpm exec tsx docs/scenarios/proofs/oauth2-refresh-token-rotation/c7-cookiesession-hooks.ts +import { cookieSession } from '../../../../packages/core/src/auth'; +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { check, finish, heading, note } from './harness'; + +async function main(): Promise { + heading('C7 — what cookieSession hands its onRefresh hook'); + + // A login endpoint that sets a cookie AND returns a rotated refresh token in the body. + const loginAdapter: Adapter = async () => ({ + status: 200, + headers: { 'set-cookie': 'sid=S1; Path=/; HttpOnly' }, + body: { refresh_token: 'RT-1', access_token: 'AT-1', expires_in: 3600 }, + }); + const login = stitch({ + url: 'https://auth.example.com/login', + method: 'POST', + adapter: loginAdapter, + }); + + const seen: unknown[] = []; + const api = stitch({ + url: 'https://api.example.com/issues', + store: memoryStore(), + adapter: async () => ({ status: 200, headers: {}, body: { ok: true } }), + auth: cookieSession({ + login, + cookie: 'sid', + tenancy: 'app', // standalone stitch: no principal to bind + onRefresh: (result) => { + seen.push(result); + }, + }), + }); + + await api(); + + check('onRefresh fired once', seen.length, 1); + const result = seen[0] as Record; + note('what the hook received', JSON.stringify(result)); + check( + 'fields handed to the host', + Object.keys(result).sort().join(','), + 'ok,status', + ); + // The rotated token was IN the login response and reached nothing the host can read. + check('login body reachable from the hook', 'body' in result, false); + check('rotated refresh_token reachable', 'refresh_token' in result, false); + + finish( + 'C7', + 'cookieSession reports only {ok, status} — the login RESPONSE BODY is never surfaced, so it cannot carry a rotated token', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/fake-provider.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/fake-provider.ts new file mode 100644 index 00000000..e14a849c --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/fake-provider.ts @@ -0,0 +1,172 @@ +// A fake, in-memory OAuth2 provider that behaves like Atlassian / Asana: the refresh token is +// SINGLE-USE and ROTATES, and presenting an already-redeemed one is treated as theft — the whole +// token family is revoked (RFC 6819 §5.2.2.3 replay detection). +// +// Nothing here touches the network. Both endpoints are exposed as StitchAPI `Adapter`s so a proof +// can inject them via `stitch({ adapter })` (resource server) and `oauth2({ adapter })` (token +// endpoint) and COUNT every call precisely. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +const sleep = (ms: number): Promise => + new Promise((r) => setTimeout(r, ms)); + +/** One recorded hit on the token endpoint — the exact form body StitchAPI sent. */ +export interface TokenRequest { + grant_type?: string; + refresh_token?: string; + client_id?: string; + scope?: string; + [k: string]: string | undefined; +} + +export interface ProviderOptions { + /** Seed refresh token for the connected account (the one a real integration has on disk). */ + refreshToken?: string; + /** `expires_in` (seconds) returned with every access token. Default 3600. */ + expiresIn?: number; +} + +export class FakeRotatingProvider { + /** Every token-endpoint hit, in order. `tokenRequests.length` IS the call count. */ + readonly tokenRequests: TokenRequest[] = []; + /** Every resource-server hit, in order (the Authorization header sent). */ + readonly resourceRequests: (string | undefined)[] = []; + /** How many times a CONSUMED refresh token was presented — i.e. the bug we are hunting. */ + replayDetections = 0; + /** Once true the account is dead: every later grant fails, the user must re-authorize. */ + familyRevoked = false; + + /** The refresh token the provider will currently accept. Rotates on every redemption. */ + activeRefreshToken: string; + /** Refresh tokens already spent. Presenting one of these trips replay detection. */ + readonly consumedRefreshTokens = new Set(); + + /** The access token minted most recently (what the resource server accepts). */ + currentAccessToken: string | undefined; + /** Access tokens the resource server now rejects with 401 (simulates server-side expiry). */ + readonly rejectedAccessTokens = new Set(); + + private seq = 0; + private readonly expiresIn: number; + + constructor(opts: ProviderOptions = {}) { + this.activeRefreshToken = opts.refreshToken ?? 'RT-0'; + this.expiresIn = opts.expiresIn ?? 3600; + } + + /** Count of token-endpoint hits. */ + get tokenCalls(): number { + return this.tokenRequests.length; + } + + /** Mark the currently-issued access token as no longer accepted by the resource server. */ + expireCurrentAccessToken(): void { + if (this.currentAccessToken) + this.rejectedAccessTokens.add(this.currentAccessToken); + } + + private mintAccessToken(): string { + const t = `AT-${++this.seq}`; + this.currentAccessToken = t; + return t; + } + + /** + * The token endpoint. Handles `client_credentials` (no refresh token in play) and + * `refresh_token` (single-use + rotating, with replay detection). + */ + tokenAdapter(opts: { delayMs?: number } = {}): Adapter { + return async (req: AdapterRequest): Promise => { + const body = (req.body ?? {}) as TokenRequest; + this.tokenRequests.push({ ...body }); + if (opts.delayMs) await sleep(opts.delayMs); + + // A revoked family is permanently dead — exactly what makes this failure expensive. + if (this.familyRevoked) + return { + status: 400, + headers: {}, + body: { + error: 'invalid_grant', + error_description: 'token family revoked', + }, + }; + + const grant = body.grant_type ?? 'client_credentials'; + + if (grant === 'refresh_token') { + const presented = body.refresh_token ?? ''; + if (this.consumedRefreshTokens.has(presented)) { + // REPLAY. A real provider reads this as token theft and kills the family. + this.replayDetections++; + this.familyRevoked = true; + return { + status: 400, + headers: {}, + body: { + error: 'invalid_grant', + error_description: + 'refresh token already used — token family revoked', + }, + }; + } + if (presented !== this.activeRefreshToken) + return { + status: 400, + headers: {}, + body: { error: 'invalid_grant' }, + }; + // Valid redemption: consume the old token, ROTATE to a new one, mint an access token. + this.consumedRefreshTokens.add(presented); + this.activeRefreshToken = `RT-${this.seq + 1}`; + return { + status: 200, + headers: {}, + body: { + access_token: this.mintAccessToken(), + token_type: 'Bearer', + expires_in: this.expiresIn, + refresh_token: this.activeRefreshToken, // the NEXT one to use + }, + }; + } + + // client_credentials (what `oauth2()` sends by default): no refresh token at all. + return { + status: 200, + headers: {}, + body: { + access_token: this.mintAccessToken(), + token_type: 'Bearer', + expires_in: this.expiresIn, + }, + }; + }; + } + + /** + * The resource server. 200 for the access token it currently accepts, 401 for anything + * expired/rejected — the wall that makes N concurrent callers all decide to refresh at once. + */ + resourceAdapter(opts: { delayMs?: number } = {}): Adapter { + return async (req: AdapterRequest): Promise => { + const auth = req.headers['authorization']; + this.resourceRequests.push(auth); + // The delay is what makes N calls genuinely OVERLAP: every caller is dispatched and + // waiting before the first response lands, which is the real-world race. + if (opts.delayMs) await sleep(opts.delayMs); + const token = auth?.replace(/^Bearer /, '') ?? ''; + if (!token || this.rejectedAccessTokens.has(token)) + return { + status: 401, + headers: {}, + body: { error: 'invalid_token' }, + }; + return { status: 200, headers: {}, body: { ok: true, token } }; + }; + } +} diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/harness.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/harness.ts new file mode 100644 index 00000000..47aaaf85 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/harness.ts @@ -0,0 +1,42 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED number either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** How many results came back as the fake resource server's `{ ok: true }` success body. */ +export function okCount(results: readonly unknown[]): number { + return results.filter((r) => (r as { ok?: boolean } | null)?.ok === true) + .length; +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); +} diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/locked-refresh-strategy.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/locked-refresh-strategy.ts new file mode 100644 index 00000000..8b82bdf7 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/locked-refresh-strategy.ts @@ -0,0 +1,155 @@ +// USER CODE, part 2 — everything in rotating-refresh-strategy.ts PLUS cross-process mutual +// exclusion, built on the only two `StitchStore` primitives that can express a lock: +// `increment(key, ttl)` (atomic; the winner is whoever gets `1`) and `set(key, undefined)` +// (a delete, part of the documented store contract — see `verifyStoreContract`, testing.ts:224). +// +// Losers do NOT redeem. They poll the vault until the winner publishes a new access token, which +// is what keeps a single-use refresh token from being presented twice. +// +// CAVEAT the proof cannot check: `verifyStoreContract` only requires `increment` to be atomic +// WITHIN a process (testing.ts:160-162, 291). A real deployment needs a backend whose increment is +// atomic ACROSS processes (Redis `INCR`, `UPDATE ... RETURNING`). That is a stronger guarantee +// than the store contract demands. +import type { + Adapter, + AuthContext, + AuthStrategy, +} from '../../../../packages/core/src/types'; + +export interface LockedRotatingRefreshOptions { + tokenUrl: string; + clientId: string; + clientSecret: string; + /** Vault namespace — the connected account. Every worker must pass the SAME value. */ + key: string; + seedRefreshToken: string; + adapter: Adapter; + /** Treat the access token as stale this long before its stated expiry. Default 30s. */ + skewMs?: number; + /** Lock lease. Must exceed a redemption's worst-case latency; a crash frees it after this. */ + lockTtlMs?: number; + /** How long a loser waits for the winner's token before giving up. Default 5s. */ + waitMs?: number; + /** Loser poll interval. Default 25ms. */ + pollMs?: number; +} + +interface CachedAccess { + token: string; + expiresAt: number; +} + +interface TokenResponse { + access_token?: string; + refresh_token?: string; + expires_in?: number; +} + +const sleep = (ms: number): Promise => + new Promise((r) => setTimeout(r, ms)); + +export function lockedRotatingRefresh( + opts: LockedRotatingRefreshOptions, +): AuthStrategy { + const skew = opts.skewMs ?? 30_000; + const lockTtl = opts.lockTtlMs ?? 10_000; + const waitMs = opts.waitMs ?? 5_000; + const pollMs = opts.pollMs ?? 25; + const accessKey = `lr:${opts.key}:access`; + const refreshKey = `lr:${opts.key}:refresh`; + const lockKey = `lr:${opts.key}:lock`; + let inFlight: Promise | undefined; + + const fresh = (c: CachedAccess | undefined): boolean => + !!c && (c.expiresAt === 0 || Date.now() < c.expiresAt - skew); + + /** Redeem the stored refresh token ONCE: rotate, persist the new one, cache the access token. */ + const redeem = async (ctx: AuthContext): Promise => { + const stored = (await ctx.vault.get(refreshKey)) as string | undefined; + const res = await opts.adapter({ + url: opts.tokenUrl, + method: 'POST', + headers: { accept: 'application/json' }, + body: { + grant_type: 'refresh_token', + refresh_token: stored ?? opts.seedRefreshToken, + client_id: opts.clientId, + client_secret: opts.clientSecret, + }, + bodyType: 'form', + }); + const body = (res.body ?? {}) as TokenResponse; + if (res.status >= 400 || !body.access_token) + throw new Error( + `refresh_token grant failed: HTTP ${res.status}. The account may need re-authorization.`, + ); + // Durability ordering: the rotated token lands BEFORE the access token is published. + if (body.refresh_token) + await ctx.vault.set(refreshKey, body.refresh_token); + const ttl = body.expires_in ? body.expires_in * 1000 : undefined; + await ctx.vault.set( + accessKey, + { token: body.access_token, expiresAt: ttl ? Date.now() + ttl : 0 }, + ttl, + ); + }; + + /** + * Redeem under a store-backed lock scoped to the account. `previous` is the token the caller + * was holding — a loser is done as soon as the vault shows something different. + */ + const withLock = async ( + ctx: AuthContext, + previous: string | undefined, + ): Promise => { + const deadline = Date.now() + waitMs; + for (;;) { + if ((await ctx.vault.increment(lockKey, lockTtl)) === 1) { + try { + await redeem(ctx); + } finally { + await ctx.vault.set(lockKey, undefined); // release + } + return; + } + if (Date.now() >= deadline) + throw new Error( + `timed out waiting ${waitMs}ms for another worker to refresh ${opts.key}`, + ); + await sleep(pollMs); + const cached = (await ctx.vault.get(accessKey)) as + CachedAccess | undefined; + if (cached && cached.token !== previous) return; // the winner published + } + }; + + /** In-process coalescing IN FRONT of the lock, so one process makes one lock attempt. */ + const redeemOnce = ( + ctx: AuthContext, + previous: string | undefined, + ): Promise => + (inFlight ??= withLock(ctx, previous).finally(() => { + inFlight = undefined; + })); + + return { + name: 'lockedRotatingRefresh', + async apply(req, ctx) { + let cached = (await ctx.vault.get(accessKey)) as + CachedAccess | undefined; + if (!fresh(cached)) { + await redeemOnce(ctx, cached?.token); + cached = (await ctx.vault.get(accessKey)) as + CachedAccess | undefined; + } + if (!cached) throw new Error(`no access token for ${opts.key}`); + req.headers['authorization'] = `Bearer ${cached.token}`; + }, + shouldRefresh: (res) => res.status === 401, + async refresh(ctx) { + const cached = (await ctx.vault.get(accessKey)) as + CachedAccess | undefined; + await redeemOnce(ctx, cached?.token); + }, + }; +} diff --git a/docs/scenarios/proofs/oauth2-refresh-token-rotation/rotating-refresh-strategy.ts b/docs/scenarios/proofs/oauth2-refresh-token-rotation/rotating-refresh-strategy.ts new file mode 100644 index 00000000..fded8bf2 --- /dev/null +++ b/docs/scenarios/proofs/oauth2-refresh-token-rotation/rotating-refresh-strategy.ts @@ -0,0 +1,108 @@ +// USER CODE — a custom `AuthStrategy` (the exported type) implementing the rotating +// `grant_type=refresh_token` grant that `oauth2()` cannot express. This file is the answer to +// "how much code does a user have to write?", so it contains NOTHING but the strategy. +// +// It provides, in order of importance: +// 1. rotation — the `refresh_token` from each response becomes the next request's input; +// 2. durability — the rotated token is written to `ctx.vault` BEFORE the access token is +// handed out, so a crash costs an access token, not the account; +// 3. single-flight — concurrent callers of one account await ONE redemption (in-process). +// +// It does NOT provide cross-process mutual exclusion — see locked-refresh-strategy.ts (proved by +// c6-cross-process-lock.ts). +import type { + Adapter, + AuthContext, + AuthStrategy, +} from '../../../../packages/core/src/types'; + +export interface RotatingRefreshOptions { + /** The provider's token endpoint. */ + tokenUrl: string; + clientId: string; + clientSecret: string; + /** Vault namespace — the CONNECTED ACCOUNT, which is the correct scope of mutual exclusion. */ + key: string; + /** Refresh token to start from; used only when the vault holds none yet. */ + seedRefreshToken: string; + /** Transport for the token request (the proof injects a fake provider here). */ + adapter: Adapter; + /** Treat the access token as stale this long before its stated expiry. Default 30s. */ + skewMs?: number; +} + +interface CachedAccess { + token: string; + expiresAt: number; // epoch ms; 0 = no known expiry +} + +interface TokenResponse { + access_token?: string; + refresh_token?: string; + expires_in?: number; +} + +export function rotatingRefresh(opts: RotatingRefreshOptions): AuthStrategy { + const skew = opts.skewMs ?? 30_000; + const accessKey = `rr:${opts.key}:access`; + const refreshKey = `rr:${opts.key}:refresh`; + // In-process single-flight, scoped to this strategy instance (one account per instance). + let inFlight: Promise | undefined; + + /** Redeem the stored refresh token ONCE: rotate, persist, cache. */ + const redeem = async (ctx: AuthContext): Promise => { + const stored = (await ctx.vault.get(refreshKey)) as string | undefined; + const res = await opts.adapter({ + url: opts.tokenUrl, + method: 'POST', + headers: { accept: 'application/json' }, + body: { + grant_type: 'refresh_token', + refresh_token: stored ?? opts.seedRefreshToken, + client_id: opts.clientId, + client_secret: opts.clientSecret, + }, + bodyType: 'form', + }); + const body = (res.body ?? {}) as TokenResponse; + if (res.status >= 400 || !body.access_token) + throw new Error( + `refresh_token grant failed: HTTP ${res.status}. The account may need re-authorization.`, + ); + // Persist the ROTATED token first: the old one is spent server-side the moment the + // provider answered, so the write must land before the access token is used. + if (body.refresh_token) + await ctx.vault.set(refreshKey, body.refresh_token); + const ttl = body.expires_in ? body.expires_in * 1000 : undefined; + await ctx.vault.set( + accessKey, + { token: body.access_token, expiresAt: ttl ? Date.now() + ttl : 0 }, + ttl, + ); + return body.access_token; + }; + + /** Coalesce concurrent redemptions into one; clear on settle so a failure never sticks. */ + const redeemOnce = (ctx: AuthContext): Promise => + (inFlight ??= redeem(ctx).finally(() => { + inFlight = undefined; + })); + + return { + name: 'rotatingRefresh', + async apply(req, ctx) { + const cached = (await ctx.vault.get(accessKey)) as + CachedAccess | undefined; + const fresh = + cached && + (cached.expiresAt === 0 || + Date.now() < cached.expiresAt - skew); + req.headers['authorization'] = + `Bearer ${fresh ? cached.token : await redeemOnce(ctx)}`; + }, + shouldRefresh: (res) => res.status === 401, + async refresh(ctx) { + await redeemOnce(ctx); + }, + }; +} diff --git a/docs/scenarios/proofs/pii-in-the-logs/README.md b/docs/scenarios/proofs/pii-in-the-logs/README.md new file mode 100644 index 00000000..cf36d09d --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/README.md @@ -0,0 +1,193 @@ +# Proofs — the customer data you didn't mean to log + +Runnable evidence for the claims in [`../../pii-in-the-logs.md`](../../pii-in-the-logs.md). + +**C1 and C2, the two deciding claims, are both confirmed — and the capture's clean split +("credentials are protected by default; customer PII is not") does not survive C4.** The PII half is +exactly as bad as predicted: a customer record reaches thirteen destinations with every one of seven +sentinels intact, and nothing in the default path removes a single one of them. The credential half +holds for credentials the library _places_ — a declarative `bearer`/`apiKey` never enters the event +stream at all — and fails for credentials that ride the payload: a vendor's `access_token` in a +response body, and a `client_secret` in a request body, are both written to the JSONL log in full. + +`sensitive: true` is settled. It is a cache opt-out and only a cache opt-out: 1 of 11 destinations +changed, and it was the cache. Across all 54 files of `packages/core/src` there is exactly **one** +read of the value. + +Two ADR claims were refuted by measurement: + +- **ADR 0018 §4** — "`findings` never leak a secret even when `redact` is off". True of soft drift, + false of hard validation, where the finding `detail` is the validator's own message. Zod's enum + message quotes the received value, and it reaches `consoleSink` and `loggerSink` — the two sinks + C1 measured as carrying nothing (C7(g)). +- **`DriftOptions.severity`** — "soft drift is always non-fatal". At the type level yes + (`DriftSeverity` excludes `error`); at runtime `severity: { undeclared: 'error' }` re-levels the + finding and fails the call (C7(f)). + +And a third thing that is not in the claims at all: **`console.error(err)` is safe and +`logger.error({ err })` is not.** `StitchError.body` is an own **enumerable** property while +`message` is not, so `err.stack` and `String(err)` carry nothing and `JSON.stringify(err)` carries +the entire customer record (C1(g)). + +Every script is standalone and offline. The transport is a fake in-memory `Adapter`; the only file +written is a JSONL trace under a `mkdtemp` directory that each script deletes. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c1-where-does-it-go.ts + +# all of them +for f in docs/scenarios/proofs/pii-in-the-logs/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/pii-in-the-logs/*.ts +``` + +## The method: sentinels, not reasoning + +Every claim here reduces to one question — _did this exact string reach that destination?_ — so the +primitive in [`harness.ts`](harness.ts) is not a value comparison but a **substring scan of a +destination's serialized bytes**, per sentinel, with the byte count reported alongside. A +destination is whatever can be reduced to bytes: the file a sink wrote, an intercepted +`process.stderr.write`, the messages handed to a `LoggerLike`, `JSON.stringify` of a wrapper, the +values a `StitchStore` was handed. The scan is deliberately `String.includes` — a cleverer matcher +would let the harness decide what counts as a leak. + +The seven sentinels in [`canary.ts`](canary.ts) are chosen to break each of the field's named +workarounds in turn: + +| code | where it lives | breaks | +| ----- | ------------------------- | ------------------------------------------- | +| `nm` | `customer.name` | no denylist has ever contained "name" | +| `em` | `customer.email` | the one field every denylist _does_ contain | +| `ssn` | `customer.ssn` | the one field every compliance doc names | +| `nst` | `profile.contact.mail` | a flat denylist misses it | +| `txt` | inside a free-text `note` | a key-based redactor cannot see it at all | +| `arr` | `contacts[1].email` | needs a walker, not a `delete` | +| `ren` | `primaryContactMail` | the "vendor added a field" case, today | + +An eighth (`LATE`) exists only for the truncation measurement in C1(b2): it sits past the JSONL +sink's default 2048-character cap, so its absence measures the cap and not a policy. Sharing a +literal with an early field would have made a truncation measurement read as a survival — which is +exactly what happened on the first run of this directory, and the fix is recorded in `canary.ts` +rather than quietly applied. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------ | ------------------------------------------ | ---------------------------------------------------------------------------------------------------------------- | +| `c1-where-does-it-go.ts` | where does the body go by default? | **13 destinations carry all 7; 11 carry none.** No partial redaction anywhere. `logger.error({err})` leaks | +| `c2-sensitive.ts` | does `sensitive: true` affect any logging? | **No. 1 of 11 destinations changed — the cache.** One read of the value in the whole of `packages/core/src` | +| `c3-inspect-redact.ts` | what does `.inspect({ redact })` redact? | **`redact: true` removes 0 of 7.** It is the credential denylist reused. Named lists work at depth and in arrays | +| `c4-credentials.ts` | is the credential half genuinely safe? | **PARTIAL.** Declarative auth 0/3 everywhere; a token in a RESPONSE body is written to the log 3/3 | +| `c5-boundary.ts` | can PII be stripped at the boundary? | **Yes — and only `hooks.onResponse` covers the failure path.** Order measured, not inferred | +| `c6-allowlist.ts` | is an allowlist expressible? | **Yes, 7 → 0 at every value-reading destination.** Residue: `.inspect().raw` and `StitchError.body` | +| `c7-drift-signal.ts` | does drift notice a new PII field? | **Yes, 3 new `undeclared` findings, 0 values.** REFUTES ADR 0018 §4 on the hard-validation path | +| `c8-assembled.ts` | the best available setup, and its cost | **42 lines, 2 seams, 0 of 9 destinations.** The boundary and the drift signal are mutually exclusive | + +## The C1 table + +The deciding measurement, printed verbatim by `c1-where-does-it-go.ts`. `●●` = the literal sentinel +is present in that destination's bytes. + +``` + destination bytes nm em ssn nst txt arr ren + ------------------------------ ------- --- --- --- --- --- --- --- + event:start 263 · · · · · · · + event:progress 68 · · · · · · · + event:result 560 ●● ●● ●● ●● ●● ●● ●● + event:done 69 · · · · · · · + └ result.data 490 ●● ●● ●● ●● ●● ●● ●● + fileSink (default) 970 ●● ●● ●● ●● ●● ●● ●● + fileSink body:{chars:false} 6622 ●● ●● ●● ●● ●● ●● ●● + fileSink body:false 523 · · · · · · · + fileSink (default, 2.9KB body) 2815 ●● ●● ●● ●● ●● ●● ●● + consoleSink (stderr) 188 · · · · · · · + loggerSink 161 · · · · · · · + .inspect().raw 490 ●● ●● ●● ●● ●● ●● ●● + .inspect().data 490 ●● ●● ●● ●● ●● ●● ●● + JSON.stringify(inspect()) 555 ●● ●● ●● ●● ●● ●● ●● + JSON.stringify(report()) 721 ●● ●● ●● ●● ●● ●● ●● + └ report.config 101 · · · · · · · + StitchError.body 490 ●● ●● ●● ●● ●● ●● ●● + StitchError.message 8 · · · · · · · + String(err) + err.stack 600 · · · · · · · + JSON.stringify(StitchError) 597 ●● ●● ●● ●● ●● ●● ●● + event:error (JSON) 103 · · · · · · · + cache entry (store.set) 627 ●● ●● ●● ●● ●● ●● ●● + otlpSink (exported spans) 455 · · · · · · · +``` + +Three things are worth reading off it directly. + +**There is no middle.** Every destination is 7 of 7 or 0 of 7. Nothing in the default path removes +`email` while keeping `plan`; either a destination gets the body or it gets metadata. That is a +_good_ property to have measured, because it means the exposure is a small set of well-defined +seams rather than a diffuse smear — but it also means there is nothing to tune. `redactHeaders` is +the only config-reachable body redaction in the whole library, and its documentation says it takes +header names. + +**Truncation is not redaction.** `fileSink (default, 2.9KB body)` still carries all seven. The cap +kept a 2048-character _prefix_; the eighth sentinel is missing purely because it sits at character +~2500. Reorder the vendor's JSON and the set that leaks changes. + +**Non-enumerability protects one field.** `.inspect().raw` is non-enumerable, exactly as ADR 0016 +says. `.inspect().data` — holding the same record — is not, so `JSON.stringify(wrapper)` leaks +everything anyway. The same shape repeats on `StitchError`: `body` is enumerable, `message` is not. + +## The seam order (C5) + +Measured by an execution log the seams write to in the order they actually fire, not read off the +source: + +``` +hooks.onRequest → hooks.onResponse → interpret → transform → output.validate +``` + +`hooks.onResponse` (engine.ts:705) is the earliest seam that sees a response body. On the **success** +path all three candidate seams are equivalent — each takes every destination from 7 to 0. On the +**failure** path they are not: a 500 leaves `StitchError.body` at 7/7 under `transform` and under a +stripping `interpret`, because the engine throws carrying the untouched `res` (engine.ts:824-831). +Only `hooks.onResponse` still holds, because it _mutated_ the object the error later carries. + +That mutation — `ctx.res.body = …` inside a hook typed `(ctx) => void` — is the single most +load-bearing construction in this directory, and it is documented nowhere as a privacy mechanism. + +## The tension C8 exists to name + +The boundary and the drift signal are mutually exclusive with the shipped seams: + +- `output: drift(SAFE)` filters every value-reading destination **and** emits a value-free inventory + of everything it stripped. It cannot touch `.inspect().raw` or the failure path. +- `hooks.onResponse` covers **everything**, including the failure path — and takes the drift signal + to **zero findings**, because drift diffs the response against the schema and the boundary removed + the response before the schema ever saw it. + +Having both means writing the key-diff walker yourself. `c8-assembled.ts` does, in 23 lines that +re-implement what `drift.ts` already contains and does not export, for a total of 42 executable +lines across two seams and 0 of 9 destinations leaking on both the success and the failure path. +The counts are read off the file at runtime, as this repository's Prettier formats it. + +None of it is upstream of the `Adapter`, which read the bytes first — measured in C8(f). + +## Why this directory imports Zod by path + +Same reason as [`../precision-loss/zod.ts`](../precision-loss/zod.ts): C6 asks whether a stripping +`output` schema keeps fields out of the log, and the stripping _is_ the behaviour under test. A +hand-rolled `{ validate }` stub would let this directory invent its own answer. In application code +the spelling is `import { z } from 'zod'`. diff --git a/docs/scenarios/proofs/pii-in-the-logs/c1-where-does-it-go.ts b/docs/scenarios/proofs/pii-in-the-logs/c1-where-does-it-go.ts new file mode 100644 index 00000000..c0903ca1 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c1-where-does-it-go.ts @@ -0,0 +1,544 @@ +// C1 (DECIDING) — where does a response body actually GO by default? +// +// The capture asks for a definitive table: every destination × every sentinel, present or absent. +// This script builds it by MEASUREMENT — each destination is reduced to the bytes it actually holds +// or wrote, and each of the seven sentinels is scanned for as a literal substring. +// +// The headline: the destinations split into two populations with nothing in between. Payload +// destinations carry ALL SEVEN sentinels; metadata destinations carry NONE. There is no partial +// redaction anywhere in the default path — no sink scrubs `email`, none scrubs `ssn`. The only +// thing that ever removes a sentinel from a payload destination is the character CAP, and a cap is +// not a filter: it keeps a prefix, which is the sentinels that happen to sort early. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c1-where-does-it-go.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { OtelSpan } from '../../../../packages/core/src/otlp'; +import { otlpSink } from '../../../../packages/core/src/otlp'; +import { consoleSink, loggerSink } from '../../../../packages/core/src/trace'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { + BASE, + LATE, + SENTINELS, + bulkyCanary, + bytesOf, + captureLogger, + captureStderr, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { + check, + checkSeq, + finish, + heading, + leakRow, + note, + printLeakTable, +} from './harness'; + +const ALL = SENTINELS.map((s) => s.code); + +async function main(): Promise { + heading('C1 (a) — the event spine: which events carry the body?'); + { + const sink = collectingSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/{id}', + adapter: fakeVendor(), + trace: sink, + }); + await call({ params: { id: 'cus_7Q2' }, body: { audit: 'lookup' } }); + + checkSeq('the spine', sink.types(), [ + 'start', + 'progress', + 'result', + 'done', + ]); + for (const ev of sink.events()) { + const row = leakRow( + `event:${ev.type}`, + bytesOf(ev), + SENTINELS, + 'JSON.stringify of the raw event a custom sink receives', + ); + void row; + } + const result = sink.of('result'); + check( + 'the `result` event carries all 7 sentinels', + leakRow( + ' └ result.data', + bytesOf((result as { data?: unknown })?.data), + SENTINELS, + ).hits.size, + 7, + ); + note( + '→ exactly ONE event carries the response body: `result`, on its `data` field. `start` carries the REQUEST input; `progress`/`done` carry timing only. A custom sink that does `JSON.stringify(event)` logs the whole customer record on the `result` event and nothing on the other three', + ); + } + + heading('C1 (b) — fileSink: the JSONL on disk, at three `body` settings'); + { + // Default cap. The canary's JSON is well under 2048 chars, so the default persists it whole + // — which is the point: the default is not "no body", it is "up to 2048 characters of body". + const t = tempFileSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: t.sink, + }); + await call(); + const jsonl = t.text(); + leakRow( + 'fileSink (default)', + jsonl, + SENTINELS, + 'the bytes read back off disk', + ); + check( + 'the default JSONL sink holds all 7 sentinels', + leakRow(' └ (same, asserted)', jsonl, SENTINELS).hits.size, + 7, + ); + note('JSONL bytes written', jsonl.length); + t.cleanup(); + } + { + const t = tempFileSink({ body: { chars: false } }); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: bulkyCanary() }), + trace: t.sink, + }); + await call(); + const full = t.text(); + const row = leakRow( + 'fileSink body:{chars:false}', + full, + SENTINELS, + 'full capture — the deliberate long spelling', + ); + check('full capture holds all 7', row.hits.size, 7); + check( + 'and the 8th, the one past character 2048', + full.includes(LATE), + true, + ); + note('bytes written at full capture', full.length); + t.cleanup(); + } + { + const t = tempFileSink({ body: false }); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: t.sink, + }); + await call(); + const marker = t.text(); + const row = leakRow( + 'fileSink body:false', + marker, + SENTINELS, + 'marker only — the cap is 0', + ); + check('body:false leaks nothing', row.hits.size, 0); + note('what it wrote instead', marker.trim().slice(0, 220)); + t.cleanup(); + } + + heading( + 'C1 (b2) — the default CAP is not a filter: it keeps a PREFIX (bulky body)', + ); + { + const t = tempFileSink(); // default 2048-char cap + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: bulkyCanary() }), + trace: t.sink, + }); + await call(); + const capped = t.text(); + const row = leakRow( + 'fileSink (default, 2.9KB body)', + capped, + SENTINELS, + 'truncated at 2048 chars — the `preview` prefix', + ); + check( + 'all 7 table sentinels sit BEFORE the cap, so all 7 land in the preview', + row.hits.size, + 7, + ); + check( + 'the record IS marked truncated', + capped.includes('"truncated":true'), + true, + ); + check( + 'and the 8th sentinel — the one placed past character 2048 — is gone', + capped.includes(LATE), + false, + ); + note( + '→ this is the sharpest edge in C1. The default cap looks like a privacy control and is a SIZE control: 7 of the 8 planted sentinels — name, email, SSN, the nested mail, the free-text mail, the array mail, the renamed key — all sit in the first 2048 characters, so all 7 persist. The 8th is absent for one reason only: it sits at character ~2500. Reorder the vendor JSON and the set that leaks changes', + ); + t.cleanup(); + } + + heading('C1 (c) — consoleSink: the bytes that reach stderr'); + { + const cap = captureStderr(); + try { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: consoleSink(), + }); + await call(); + } finally { + cap.restore(); + } + const err = cap.text(); + const row = leakRow( + 'consoleSink (stderr)', + err, + SENTINELS, + 'process.stderr.write intercepted', + ); + check('consoleSink leaks nothing', row.hits.size, 0); + note('what it printed', err.replace(/\x1b\[\d+m/g, '').trim()); + } + + heading('C1 (d) — loggerSink: the lines handed to pino/winston/console'); + { + const logger = captureLogger(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: loggerSink(logger), + }); + await call(); + const row = leakRow( + 'loggerSink', + logger.text(), + SENTINELS, + 'every message passed to the LoggerLike', + ); + check('loggerSink leaks nothing', row.hits.size, 0); + note('lines logged', logger.lines().length); + note( + 'the `result` line', + logger.lines().find((l) => l.level === 'info')?.message, + ); + } + + heading('C1 (e) — .inspect(): `raw`, `data`, and the wrapper itself'); + { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + }); + const w = await call.inspect(); + const rawRow = leakRow( + '.inspect().raw', + bytesOf(w.raw), + SENTINELS, + 'the pre-validation body, read deliberately', + ); + check('`raw` holds all 7', rawRow.hits.size, 7); + const dataRow = leakRow( + '.inspect().data', + bytesOf(w.data), + SENTINELS, + 'the validated value', + ); + check('`data` holds all 7 (no output schema)', dataRow.hits.size, 7); + const wrapRow = leakRow( + 'JSON.stringify(inspect())', + bytesOf(w), + SENTINELS, + 'the whole wrapper — `raw` is non-enumerable', + ); + check( + 'the WRAPPER still holds all 7 — via `data`, not `raw`', + wrapRow.hits.size, + 7, + ); + check( + '`raw` really is non-enumerable', + Object.keys(w).includes('raw'), + false, + ); + note( + "→ ADR 0016's non-enumerability protects `raw` and nothing else. `data` is a plain enumerable field holding the same customer record, so `JSON.stringify(wrapper)` — the thing the JSDoc warns against — leaks every sentinel anyway, through the field that was never hidden", + ); + } + + heading('C1 (f) — .report()'); + { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + }); + const r = await call.report(); + const row = leakRow( + 'JSON.stringify(report())', + bytesOf(r), + SENTINELS, + 'the full RunReport a consumer logs', + ); + check('the report holds all 7', row.hits.size, 7); + note( + 'report.config is the REDACTED config', + Object.keys(r.config).sort().join(','), + ); + leakRow( + ' └ report.config', + bytesOf(r.config), + SENTINELS, + 'the config echo only', + ); + } + + heading( + 'C1 (g) — the failure path: StitchError, its message, the error event', + ); + { + const sink = collectingSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ status: 500 }), + trace: sink, + }); + const out = await call.safe(); + check('the call failed', out.ok, false); + const err = out.error; + const bodyRow = leakRow( + 'StitchError.body', + bytesOf(err?.body), + SENTINELS, + 'the failing response body, lifted onto the error', + ); + check('`StitchError.body` holds all 7', bodyRow.hits.size, 7); + const msgRow = leakRow( + 'StitchError.message', + String(err?.message ?? ''), + SENTINELS, + 'the message string', + ); + check('the message leaks nothing', msgRow.hits.size, 0); + note('the message', err?.message); + leakRow( + 'String(err) + err.stack', + `${String(err)}\n${err?.stack ?? ''}`, + SENTINELS, + 'what a bare console.error(err) prints', + ); + const jsonRow = leakRow( + 'JSON.stringify(StitchError)', + bytesOf(err), + SENTINELS, + 'what a STRUCTURED logger serialises', + ); + check( + 'but JSON.stringify of the SAME error holds all 7', + jsonRow.hits.size, + 7, + ); + check( + 'because `body` is an own ENUMERABLE property', + Object.keys(err ?? {}).includes('body'), + true, + ); + note( + 'the enumerable keys of a StitchError', + Object.keys(err ?? {}).join(','), + ); + note( + '→ the sharpest single row in the table. `err.stack` and `String(err)` are clean, so a `console.error(err)` is safe and a `logger.error({ err })` is not: `StitchError` assigns `this.body` in its constructor (types.ts:1790), which makes it an own enumerable property, and every structured logger reaches for `JSON.stringify`. The `message` is NON-enumerable (the `Error` base sets it), so the JSON is `{"name","status","attempts","body"}` — the payload survives and the human-readable part does not', + ); + const evt = sink.of('error') as StitchEvent | undefined; + const evRow = leakRow( + 'event:error (JSON)', + bytesOf(evt), + SENTINELS, + 'the error event a custom sink receives', + ); + check( + 'the error EVENT leaks nothing — the body rides a non-enumerable symbol', + evRow.hits.size, + 0, + ); + note( + '→ the credential/PII split the capture predicted shows up HERE, in a shape it did not: the failing body is on `StitchError.body` (all 7 sentinels) but NOT on the error event (0 of 7). A trace sink sees `HTTP 500`; the `catch` block sees the whole customer record', + ); + } + + heading('C1 (h) — the cache entry'); + { + const store = recordingStore(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + store, + cache: { ttl: '60s' }, + }); + await call(); + const row = leakRow( + 'cache entry (store.set)', + store.text(), + SENTINELS, + 'every value handed to the StitchStore', + ); + check('the cache entry holds all 7', row.hits.size, 7); + note('store writes', store.writes().length); + note( + 'the value shape the cache persists', + Object.keys( + (store + .writes() + .find( + (w) => + w.value && + typeof w.value === 'object' && + 'v' in (w.value as object), + )?.value ?? {}) as object, + ).join(','), + ); + } + + heading('C1 (i) — otlpSink (not in the claims list, and worth the row)'); + { + const spans: OtelSpan[] = []; + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: otlpSink({ + exporter: { export: (s) => void spans.push(...s) }, + }), + }); + await call(); + const row = leakRow( + 'otlpSink (exported spans)', + bytesOf(spans), + SENTINELS, + 'the spans handed to the exporter', + ); + check('OTLP leaks nothing', row.hits.size, 0); + note('spans exported', spans.length); + } + + heading( + 'C1 (j) — an appendix nobody asked for: the header denylist hits BODY keys', + ); + { + // `trace.ts`'s `redact()` walks the WHOLE record replacing any key in the header denylist. + // It is documented as header redaction; it is actually key-name redaction at every depth, + // so a response body FIELD named `cookie`/`authorization`/`x-api-key` is scrubbed by pure + // coincidence of name — while `ssn` beside it is not. + const t = tempFileSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ + body: { + ssn: '078-05-1120', + cookie: '078-05-1120', + authorization: '078-05-1120', + }, + }), + trace: t.sink, + }); + await call(); + const text = t.text(); + check( + 'a body field named `cookie` IS redacted in the JSONL', + /"cookie":"\[REDACTED\]"/.test(text), + true, + ); + check( + 'a body field named `authorization` too', + /"authorization":"\[REDACTED\]"/.test(text), + true, + ); + check( + 'the identical value under `ssn` is not', + text.includes('"ssn":"078-05-1120"'), + true, + ); + note( + '→ the same string, three keys, two outcomes. The built-in sink already contains a working deep key-name redactor (`trace.ts:142`); it is simply pointed at a five-name credential list and is not reachable from config. `redactHeaders` widens it — and its type/JSDoc say "header names", so nothing tells you it also scrubs body keys', + ); + // Prove the escape hatch reaches the body too. + const t2 = tempFileSink({ redactHeaders: ['ssn', 'email', 'name'] }); + const call2 = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: t2.sink, + }); + await call2(); + const text2 = t2.text(); + check( + '`redactHeaders: ["ssn","email","name"]` scrubs those BODY fields', + /"ssn":"\[REDACTED\]"/.test(text2) && + /"email":"\[REDACTED\]"/.test(text2), + true, + ); + check( + 'and it reaches NESTED keys — profile.contact.mail if you name `mail`', + text2.includes('nested-canary@example.test'), + true, + ); + note( + "the nested mail survives because its KEY is `mail`, not `email` — so this hatch is a denylist with all a denylist's failure modes, but it does work at depth and it is the only body redaction that is reachable from config at all", + ); + t.cleanup(); + t2.cleanup(); + } + + const tally = printLeakTable(SENTINELS); + console.log( + `\n ${tally.leaking} destination(s) carry at least one sentinel; ${tally.clean} carry none.`, + ); + note('sentinel codes measured per destination', ALL.join(' ')); + + finish( + 'C1', + 'CONFIRMED, and the table is more binary than the capture drew it. The destinations split into two populations with NOTHING in between: payload destinations carry all 7 sentinels (the `result` event, fileSink at every non-zero cap, `.inspect().raw`, `.inspect().data`, `JSON.stringify(inspect())`, `.report()`, `StitchError.body`, `JSON.stringify(StitchError)`, the cache entry) and metadata destinations carry 0 of 7 (`start`/`progress`/`done`/`error` events, consoleSink, loggerSink, otlpSink, `StitchError.message`, `String(err)` + `err.stack`). No destination is partially redacted. Exactly ONE event carries the response body — `result`, on `data` — so "the event spine leaks" is really "one event leaks". Three measurements the capture does not contain: (1) the default fileSink cap is a SIZE control that keeps a PREFIX, so on a 2.9KB body all 7 table sentinels still persisted into the `preview` and only an 8th, planted deliberately past character 2048, was absent; (2) `JSON.stringify(.inspect())` leaks all 7 through the ENUMERABLE `data` field, so ADR 0016 non-enumerability protects `raw` and nothing else — and the same pattern repeats on the error: `String(err)`/`err.stack` are clean but `JSON.stringify(err)` carries all 7, because `StitchError.body` is an own enumerable property while `message` is not, so `console.error(err)` is safe and `logger.error({ err })` is not; (3) the JSONL sink already ships a working deep key-name redactor — a body field named `cookie` is replaced with [REDACTED] while the identical value under `ssn` is not, and `redactHeaders` (documented as "header names") is the one config-reachable way to point it at a body key', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c2-sensitive.ts b/docs/scenarios/proofs/pii-in-the-logs/c2-sensitive.ts new file mode 100644 index 00000000..99c3d9a5 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c2-sensitive.ts @@ -0,0 +1,306 @@ +// C2 (DECIDING, PRE-REGISTERED) — does `sensitive: true` affect ANY logging destination? +// +// The capture pre-registers a suspicion from scenario 18: `sensitive` is a CACHE opt-out and +// nothing more. It is also, by a distance, the nearest-looking key in the whole config to "do not +// log this" — `types.ts:1652-1658` calls itself "the honest 'do not persist this response' hatch +// for one-time tokens or compliance-bound data", which reads exactly like a logging control. +// +// This script settles it two ways, and they agree: +// +// 1. A GREP of the shipped source for every read of the identifier, performed at runtime over the +// real files rather than quoted from memory. +// 2. The whole C1 battery run TWICE — once without `sensitive`, once with — comparing the +// sentinel hit count at every destination. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c2-sensitive.ts +import { stitch } from '../../../../packages/core/src/index'; +import { consoleSink, loggerSink } from '../../../../packages/core/src/trace'; +import { + BASE, + SENTINELS, + bytesOf, + captureLogger, + captureStderr, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { check, checkSeq, finish, heading, note, scan } from './harness'; + +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; + +const SRC = join(process.cwd(), 'packages', 'core', 'src'); + +/** Every line in the shipped source mentioning the bare identifier `sensitive`. */ +function grepSensitive(): { file: string; line: number; text: string }[] { + const out: { file: string; line: number; text: string }[] = []; + for (const f of readdirSync(SRC).filter((n) => n.endsWith('.ts'))) { + const lines = readFileSync(join(SRC, f), 'utf8').split('\n'); + lines.forEach((text, i) => { + // The bare word only: `case-insensitive` / `latency-sensitive` are English, not reads. + if (!/(?> { + const hits = new Map(); + const record = (dest: string, text: string): void => { + hits.set(dest, scan(text, SENTINELS).size); + }; + // `sensitive` is documented as "only meaningful alongside a `cache` block", so every run here + // carries one — otherwise the comparison would be against a slot that was never consulted. + const base = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + ...(sensitive ? { sensitive: true } : {}), + } as const; + + { + const sink = collectingSink(); + const store = recordingStore(); + const call = stitch({ + ...base, + adapter: fakeVendor(), + trace: sink, + store, + }); + await call(); + record('event:result', bytesOf(sink.of('result'))); + record('event spine (all)', sink.text()); + record('cache entry (store.set)', store.text()); + hits.set(' └ store writes', store.writes().length); + } + { + const t = tempFileSink(); + const call = stitch({ + ...base, + adapter: fakeVendor(), + trace: t.sink, + store: recordingStore(), + }); + await call(); + record('fileSink (default)', t.text()); + t.cleanup(); + } + { + const cap = captureStderr(); + try { + const call = stitch({ + ...base, + adapter: fakeVendor(), + trace: consoleSink(), + store: recordingStore(), + }); + await call(); + } finally { + cap.restore(); + } + record('consoleSink (stderr)', cap.text()); + } + { + const logger = captureLogger(); + const call = stitch({ + ...base, + adapter: fakeVendor(), + trace: loggerSink(logger), + store: recordingStore(), + }); + await call(); + record('loggerSink', logger.text()); + } + { + const call = stitch({ + ...base, + adapter: fakeVendor(), + store: recordingStore(), + }); + const w = await call.inspect(); + record('.inspect().raw', bytesOf(w.raw)); + record('JSON.stringify(inspect())', bytesOf(w)); + const r = await call.report(); + record('JSON.stringify(report())', bytesOf(r)); + } + { + const call = stitch({ + ...base, + adapter: fakeVendor({ status: 500 }), + store: recordingStore(), + }); + const out = await call.safe(); + record('StitchError.body', bytesOf(out.error?.body)); + record('StitchError.message', String(out.error?.message ?? '')); + } + return hits; +} + +async function main(): Promise { + heading('C2 (a) — every mention of `sensitive` in the shipped source'); + { + const found = grepSensitive(); + const code = found.filter((f) => !isComment(f.text)); + for (const f of found) + console.log( + ` ${isComment(f.text) ? 'comment' : 'CODE '} ${f.file}:${f.line} ${f.text.slice(0, 96)}`, + ); + check('mentions in total', found.length, 6); + check('of which are CODE, not prose', code.length, 3); + checkSeq( + 'the three code sites, in full', + code.map((c) => `${c.file}:${c.line}`), + ['config-anatomy.ts:134', 'engine.ts:1020', 'types.ts:1658'], + ); + note( + 'two of the three are DECLARATIONS, not reads: `types.ts:1658` is the `StitchConfig` field, and `config-anatomy.ts:134` is the slot description — `sensitive: object`, i.e. no facts at all, which is why the slot rides onto the public `__config` untouched (no `dropped`, no `stage`, no `policy`). That leaves exactly one site that consults the VALUE', + ); + const engine = readFileSync(join(SRC, 'engine.ts'), 'utf8').split('\n'); + const read = engine + .map((t, i) => ({ line: i + 1, text: t.trim() })) + .filter((l) => l.text.includes('cfg.sensitive')); + checkSeq( + 'every read of the resolved value in the entire engine', + read.map((r) => `engine.ts:${r.line} ${r.text}`), + ['engine.ts:1020 if (!config || cfg.sensitive) return null;'], + ); + note( + '→ ONE read, in `ensureCache`. Nothing in `trace.ts`, `otlp.ts`, `stitch.ts` or any sink references the slot at all', + ); + } + + heading('C2 (b) — the battery, with and without `sensitive: true`'); + { + const without = await battery(false); + const withIt = await battery(true); + const keys = [...without.keys()]; + const w = Math.max(...keys.map((k) => k.length)); + console.log( + `\n ${'destination'.padEnd(w)} sentinels without sentinels with changed?\n` + + ` ${'-'.repeat(w)} ----------------- -------------- --------`, + ); + let changed = 0; + let destinations = 0; + for (const k of keys) { + const a = without.get(k) ?? -1; + const b = withIt.get(k) ?? -1; + // Rows indented with `└` are counters (store writes), not destinations — excluded from + // the tally so "1 of 11 destinations changed" counts destinations. + const isDest = !k.startsWith(' └'); + if (isDest) destinations++; + if (isDest && a !== b) changed++; + console.log( + ` ${k.padEnd(w)} ${String(a).padStart(17)} ${String(b).padStart(14)} ${a === b ? 'no' : 'YES'}`, + ); + } + check('destinations measured', destinations, 11); + check( + 'destinations whose leak changed when `sensitive: true` was set', + changed, + 1, + ); + check( + 'and the one that changed is the cache: writes without', + without.get(' └ store writes'), + 1, + ); + check('writes with', withIt.get(' └ store writes'), 0); + check( + 'the cache entry bytes went from 7 sentinels …', + without.get('cache entry (store.set)'), + 7, + ); + check('… to 0', withIt.get('cache entry (store.set)'), 0); + check( + 'the JSONL sink is UNCHANGED — still all 7', + withIt.get('fileSink (default)'), + 7, + ); + check( + 'the `result` event is UNCHANGED — still all 7', + withIt.get('event:result'), + 7, + ); + check( + '`.inspect().raw` is UNCHANGED — still all 7', + withIt.get('.inspect().raw'), + 7, + ); + check( + '`StitchError.body` is UNCHANGED — still all 7', + withIt.get('StitchError.body'), + 7, + ); + } + + heading('C2 (c) — and it announces itself on the redacted config'); + { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + cache: { ttl: '60s' }, + sensitive: true, + }); + const r = await call.report(); + check( + '`sensitive: true` survives onto the PUBLIC `__config`', + (r.config as { sensitive?: unknown }).sensitive, + true, + ); + note( + 'the redacted config keys', + Object.keys(r.config).sort().join(','), + ); + note( + '→ a small compounding hazard: the slot that does NOT stop logging is itself logged, so a `.report()` line reads `"sensitive":true` beside the customer record it did not protect', + ); + } + + heading('C2 (d) — the thing it is not: a `sensitive` stitch with no cache'); + { + const t = tempFileSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: t.sink, + sensitive: true, + }); + await call(); + const text = t.text(); + check( + '`sensitive: true` with no `cache` block: JSONL still holds all 7', + scan(text, SENTINELS).size, + 7, + ); + note( + 'so the slot is accepted, changes nothing, and writes the whole record to disk', + ); + t.cleanup(); + } + + finish( + 'C2', + 'CONFIRMED — the pre-registered suspicion holds exactly, and the measurement is unusually clean. `sensitive: true` changed the leak at 1 of 11 destinations, and that destination is the cache: store writes 1 → 0, cache-entry sentinels 7 → 0. Every other destination is byte-for-byte unaffected: the `result` event 7/7, the JSONL sink 7/7, `.inspect().raw` 7/7, `.report()` 7/7, `StitchError.body` 7/7, consoleSink/loggerSink 0/0 either way. The source agrees: across all 54 files of `packages/core/src` there are 6 mentions of the identifier, 3 of them code and 2 of THOSE declarations (`types.ts:1658` the config field, `config-anatomy.ts:134` the slot description) — exactly ONE site reads the value: `engine.ts:1020`, `if (!config || cfg.sensitive) return null`, inside `ensureCache`. No sink, no trace module, and no event builder references it. Stated plainly: `sensitive: true` means DO NOT PERSIST THIS TO THE CACHE. It does not mean do not log, do not trace, do not put on an error, or do not write to disk — and the JSONL sink will still write the full body to a file while the slot is set. It also survives onto the public `__config`, so `.report()` prints `"sensitive":true` next to the unredacted record', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c3-inspect-redact.ts b/docs/scenarios/proofs/pii-in-the-logs/c3-inspect-redact.ts new file mode 100644 index 00000000..9e4be0af --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c3-inspect-redact.ts @@ -0,0 +1,381 @@ +// C3 — what does `.inspect({ redact })` actually redact (ADR 0018)? +// +// The ADR is unusually explicit about its own limits ("name-based only — a sharing convenience, +// _not_ a leak guarantee", §3), so the interesting question is not whether the doc is honest — it +// is — but what the honest tool covers when pointed at a real customer record. Measured here: +// nested fields, array elements, renamed keys, free text, the two path grammars, the default, and +// the one thing the ADR does not say out loud. +// +// The headline: with `redact: true` and nothing else, ZERO of the seven sentinels are removed. The +// shared denylist is a CREDENTIAL denylist — `token`/`secret`/`password`/`apikey`/`signature` — and +// no spelling of a customer field is in it. `redact` only does PII work when you hand it the field +// names yourself, which is the denylist-you-must-enumerate the capture says the field cannot write. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c3-inspect-redact.ts +import { + isSecretKey, + redactSecretsDeep, + stitch, +} from '../../../../packages/core/src/index'; +import { + ARRAY, + BASE, + EMAIL, + FREETEXT, + NAME, + NESTED, + RENAMED, + SENTINELS, + SSN, + bytesOf, + fakeVendor, +} from './canary'; +import { check, checkSeq, finish, heading, note, scan } from './harness'; + +const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), +}); + +/** + * Which sentinel codes survive in `raw` under a given `redact` setting. + * + * Note the call shape: `redact` is the SECOND argument, after the (possibly empty) input. Writing + * `call.inspect({ redact: true })` puts the options object in the INPUT slot, where the engine reads + * it as a `StitchInput` and redaction silently never happens. TypeScript rejects that spelling — + * measured in (h) — but a JS consumer, or a `// @ts-expect-error`, gets the silent version. + */ +async function survivors(redact?: boolean | string[]): Promise { + const w = + redact === undefined + ? await call.inspect() + : await call.inspect({}, { redact }); + return [...scan(bytesOf(w.raw), SENTINELS)]; +} + +async function main(): Promise { + heading('C3 (a) — the default is OFF, exactly as the ADR titles itself'); + { + const bare = await survivors(); + checkSeq( + '`.inspect()` with no options — every sentinel survives', + bare, + ['nm', 'em', 'ssn', 'nst', 'txt', 'arr', 'ren'], + ); + const w = await call.inspect(); + check( + '`raw` is the SAME object the engine retained, not a clone', + typeof w.raw === 'object', + true, + ); + note( + 'ADR 0018 §3: "The default protection is non-enumerability; `redact` is the deliberate-sharing escape hatch layered on top." Measured: true, and non-enumerability is a weak protection — see C1(e), where `JSON.stringify(wrapper)` leaks everything through the enumerable `data`', + ); + } + + heading( + 'C3 (b) — `redact: true`: the shared denylist, against a customer record', + ); + { + const kept = await survivors(true); + checkSeq( + 'sentinels REMOVED by the built-in denylist', + SENTINELS.map((s) => s.code).filter((c) => !kept.includes(c)), + [], + ); + check('sentinels that survive `redact: true`', kept.length, 7); + // The reason, stated as a measurement rather than an assertion about the list. + const keys = [ + 'name', + 'email', + 'ssn', + 'mail', + 'note', + 'primaryContactMail', + 'contacts', + 'profile', + 'contact', + ]; + checkSeq( + "which of the canary's key names the denylist considers secret", + keys.filter((k) => isSecretKey(k)), + [], + ); + checkSeq( + 'and, for contrast, the names it DOES catch', + [ + 'access_token', + 'client_secret', + 'password', + 'apikey', + 'x-amz-signature', + 'sig', + 'pwd', + ].filter((k) => isSecretKey(k)), + [ + 'access_token', + 'client_secret', + 'password', + 'apikey', + 'x-amz-signature', + 'sig', + 'pwd', + ], + ); + note( + '→ REFUTATION-ADJACENT, and the single most load-bearing measurement in C3: `redact: true` is not a PII control at all. It is the credential denylist, reused. Against this record it removes nothing', + ); + } + + heading('C3 (c) — `redact: [names]`: NESTED, ARRAY, RENAMED, FREE TEXT'); + { + const kept = await survivors(['email']); + check( + "`redact: ['email']` removes the TOP-LEVEL email", + kept.includes('em'), + false, + ); + check( + 'and the ARRAY-element email at contacts[1].email — a bare name matches at any depth', + kept.includes('arr'), + false, + ); + check( + 'but NOT the nested `profile.contact.mail` — different key name', + kept.includes('nst'), + true, + ); + check( + 'nor `primaryContactMail` — the renamed key', + kept.includes('ren'), + true, + ); + check( + 'nor the address inside the free-text `note`', + kept.includes('txt'), + true, + ); + note('survivors', kept.join(',')); + } + { + const kept = await survivors(['email', 'mail', 'name', 'ssn', 'note']); + checkSeq('the full hand-written list catches six of seven', kept, [ + 'ren', + ]); + check( + '`mail` (bare) DOES reach `profile.contact.mail` two levels down', + kept.includes('nst'), + false, + ); + check( + 'and `note` blanks the whole free-text field, taking the address with it', + kept.includes('txt'), + false, + ); + note( + "→ the survivor is `primaryContactMail`. That is not an oversight in the list — it is the capture's thesis in one value: the field you did not know to name is the field that leaks", + ); + } + + heading( + 'C3 (d) — the path grammar: prefix, wildcard, and the `[]` mismatch', + ); + { + const body = { + profile: { contact: { mail: NESTED } }, + contacts: [{ email: 'a@example.test' }, { email: ARRAY }], + }; + const at = (v: unknown, p: string[]): unknown => + p.reduce( + (acc, k) => (acc as Record | undefined)?.[k], + v, + ); + // Exact dotted path. + const exact = redactSecretsDeep(body, ['profile.contact.mail']); + check( + 'exact path `profile.contact.mail`', + at(exact, ['profile', 'contact', 'mail']), + 'REDACTED', + ); + // Prefix: a pattern that is a proper prefix nukes the whole subtree. + const prefix = redactSecretsDeep(body, ['profile']); + check( + 'a prefix pattern `profile` replaces the ENTIRE subtree with the sentinel', + at(prefix, ['profile']), + 'REDACTED', + ); + // Wildcard on an object level. + const wild = redactSecretsDeep(body, ['profile.*.mail']); + check( + 'wildcard `profile.*.mail` matches one object level', + at(wild, ['profile', 'contact', 'mail']), + 'REDACTED', + ); + // The array-index grammar mismatch. + const bracket = redactSecretsDeep(body, ['contacts[].email']); + check( + '`contacts[].email` — the DRIFT grammar — matches nothing', + bytesOf(bracket).includes(ARRAY), + true, + ); + const star = redactSecretsDeep(body, ['contacts.*.email']); + check( + '`contacts.*.email` matches nothing either — the path is `contacts[1].email`', + bytesOf(star).includes(ARRAY), + true, + ); + const indexed = redactSecretsDeep(body, ['contacts[1].email']); + check( + 'only the concrete index `contacts[1].email` matches', + bytesOf(indexed).includes(ARRAY), + false, + ); + const bare = redactSecretsDeep(body, ['email']); + check( + 'so for arrays the usable spelling is the BARE key name, which matches at every index', + bytesOf(bare).includes(ARRAY), + false, + ); + note( + '→ NOT IN THE CLAIMS: the library carries two path grammars that disagree on arrays. A drift finding reports `contacts[].email`; pasting that exact string into `redact` matches nothing. `matchPath` handles `*` per dot-segment and `[` only as a prefix boundary, so the collapsed `[]` form — the one the library PRINTS at you — is the one form that cannot be used here', + ); + } + + heading( + 'C3 (e) — what redaction does NOT touch: `data`, `findings`, `status`', + ); + { + const w = await call.inspect( + {}, + { redact: ['email', 'mail', 'name', 'ssn', 'note'] }, + ); + check( + '`raw` is scrubbed down to one survivor', + scan(bytesOf(w.raw), SENTINELS).size, + 1, + ); + check( + 'but `data` still holds all 7 — redaction never touches the value', + scan(bytesOf(w.data), SENTINELS).size, + 7, + ); + check( + 'so `JSON.stringify(wrapper)` after a full redact still leaks all 7', + scan(bytesOf(w), SENTINELS).size, + 7, + ); + note( + '→ this is the trap the ADR does not spell out. `redact` protects the field that was ALREADY non-enumerable and leaves the enumerable one alone. A consumer who reads "pass `{ redact: true }` when you want to pipe this into a log" and then logs the wrapper is no safer than before', + ); + } + + heading( + 'C3 (f) — `.report({ redact })` shares the path, and `redact` returns a CLONE', + ); + { + const r = await call.report( + {}, + { redact: ['email', 'mail', 'name', 'ssn', 'note'] }, + ); + check( + '`.report()` honours `redact` on `raw` too', + scan(bytesOf(r.raw), SENTINELS).size, + 1, + ); + check( + 'and its `data` is untouched, like `.inspect()`', + scan(bytesOf(r.data), SENTINELS).size, + 7, + ); + // Non-mutation: the engine's retained body must survive redaction intact. + const original = { email: EMAIL, ssn: SSN, name: NAME, extra: RENAMED }; + const clone = redactSecretsDeep(original, ['email']); + check( + 'redactSecretsDeep does not mutate its input', + original.email, + EMAIL, + ); + check( + 'it returns a scrubbed clone', + (clone as { email: unknown }).email, + 'REDACTED', + ); + check( + 'the sentinel it writes is `REDACTED` (no brackets — the trace sink uses `[REDACTED]`)', + (clone as { email: unknown }).email, + 'REDACTED', + ); + note( + 'two different redaction sentinels ship in one library: `REDACTED` from `util.ts` (inspect/url/query) and `[REDACTED]` from `trace.ts` (headers). Cosmetic, but a log-aggregator rule written for one will not match the other', + ); + } + + heading('C3 (g) — is there a stitch-level default? No.'); + { + // ADR 0018 §1 permits one ("a `drift`-config or a `defaultInspect` block") as future work. + // Measured against the shipped surface: the option exists ONLY per call. + const cfg = (call as unknown as { __config: Record }) + .__config; + check( + 'no `defaultInspect` slot on the resolved config', + 'defaultInspect' in cfg, + false, + ); + check('no `redact` slot either', 'redact' in cfg, false); + note( + 'so redaction cannot be made the default for a stitch, a seam, or a process — deliberately (ADR 0018 §1 calls a global toggle "the silent-blinding footgun"). The consequence for this scenario: there is no configuration you can write ONCE that makes `.inspect()` safe everywhere it is called', + ); + } + + heading('C3 (h) — the calling convention, and the silent-no-op spelling'); + { + // `redact` is the SECOND parameter. Passing the options object alone puts it in the INPUT + // slot: the engine reads it as a `StitchInput` (no `params`/`query`/`body`/`headers`, so the + // request is unchanged) and `opts` stays undefined — redaction never runs, silently. + const wrong = await ( + call.inspect as unknown as (o: unknown) => Promise<{ raw: unknown }> + )({ redact: ['email', 'mail', 'name', 'ssn', 'note'] }); + check( + '`call.inspect({ redact: [...] })` — one argument — redacts NOTHING', + scan(bytesOf(wrong.raw), SENTINELS).size, + 7, + ); + const right = await call.inspect( + {}, + { redact: ['email', 'mail', 'name', 'ssn', 'note'] }, + ); + check( + '`call.inspect({}, { redact: [...] })` — two arguments — redacts six of seven', + scan(bytesOf(right.raw), SENTINELS).size, + 1, + ); + note( + "the cast above is doing real work: TypeScript REJECTS the one-argument spelling outright — `error TS2353: Object literal may only specify known properties, and 'redact' does not exist in type 'StitchInput'`, and `TS2559` for a non-literal. So the type system closes this, and it is only reachable from JavaScript or through a deliberate cast. Recorded because a security option whose no-op spelling is the shorter one is worth knowing about", + ); + } + + checkSeq( + 'sanity: the free-text address is unreachable by ANY key-based rule', + [ + ...scan( + bytesOf( + redactSecretsDeep( + { note: `mail ${FREETEXT} please`, other: 1 }, + ['email', 'mail', 'ssn', 'name', 'contact', 'address'], + ), + ), + SENTINELS, + ), + ], + ['txt'], + ); + + finish( + 'C3', + "CONFIRMED as an opt-in, name-based, per-call DENYLIST over `raw` only — and measured to be far narrower against PII than its name suggests. Four numbers: (1) `redact: true`, the shared denylist, removes 0 of 7 sentinels from a customer record — none of `name`/`email`/`ssn`/`mail`/`note`/`contacts`/`primaryContactMail` is a secret key, because the list is the credential list (`token`/`secret`/`password`/`apikey`/`signature`/`sig`/`pwd`), reused; (2) `redact: [names]` DOES work at depth and across array elements — a bare `mail` reaches `profile.contact.mail`, a bare `email` reaches `contacts[1].email` at every index — so nested and array coverage is genuinely there, but only for names you enumerate; (3) a renamed key (`primaryContactMail`) and an address inside free text are unreachable by construction, which is the capture's thesis reproduced exactly; (4) the default is off and there is NO stitch-level or process-level default — `defaultInspect` from ADR 0018 §1 was never implemented, so no single configuration can make every `.inspect()` call safe. Two findings not in the claims: the path grammar disagrees with the DRIFT path grammar on arrays (a finding printed as `contacts[].email` matches nothing when pasted into `redact`; only the bare key or a concrete `contacts[1].email` works), and `redact` scrubs `raw` while leaving the ENUMERABLE `data` untouched — so after a full redact, `JSON.stringify(wrapper)` still leaks all 7", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c4-credentials.ts b/docs/scenarios/proofs/pii-in-the-logs/c4-credentials.ts new file mode 100644 index 00000000..e61b8fe0 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c4-credentials.ts @@ -0,0 +1,493 @@ +// C4 — is the CREDENTIAL half genuinely safe? +// +// The capture's hypothesis is a clean split: credentials are protected by default, customer PII is +// not. Scenario 18 found the credential boundary held for MCP; this is the tracing equivalent. A +// bearer token, an `apiKey` in a query string, and a cookie go through every C1 destination and +// each destination's bytes are scanned for the literal value. +// +// The split is real, and it is narrower than "credentials are protected". Measured, the protection +// is a property of THREE specific things — the declarative auth seam (which runs after the `start` +// event is built), the built-in sinks' header/URL/query scrubbers, and the payload-free default +// formatters — and it does NOT extend to (a) a custom sink, which receives the raw event, or +// (b) a credential arriving in a RESPONSE body, which the JSONL sink writes out in full. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c4-credentials.ts +import { apiKey, bearer } from '../../../../packages/core/src/auth'; +import { stitch } from '../../../../packages/core/src/index'; +import type { OtelSpan } from '../../../../packages/core/src/otlp'; +import { otlpSink } from '../../../../packages/core/src/otlp'; +import { consoleSink, loggerSink } from '../../../../packages/core/src/trace'; +import { + BASE, + BEARER_TOKEN, + COOKIE_VALUE, + CREDENTIALS, + QUERY_KEY, + bytesOf, + captureLogger, + captureStderr, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { + check, + finish, + heading, + leakRow, + note, + printLeakTable, + scan, +} from './harness'; + +async function main(): Promise { + heading( + 'C4 (a) — DECLARATIVE auth: the credential is applied after the `start` event is built', + ); + { + const sink = collectingSink(); + const vendor = fakeVendor(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: vendor, + trace: sink, + auth: bearer(() => BEARER_TOKEN), + }); + await call(); + // Sanity: the token really did reach the wire — otherwise "absent everywhere" is vacuous. + check( + 'the token DID reach the transport (so the measurement is not vacuous)', + bytesOf(vendor.seen()[0]?.headers).includes(BEARER_TOKEN), + true, + ); + const row = leakRow( + 'raw event spine — auth: bearer()', + sink.text(), + CREDENTIALS, + 'JSON of every event, unredacted, as a custom sink sees it', + ); + check('the raw spine holds no credential', row.hits.size, 0); + note( + '→ `auth.apply` runs on a CLONE of the request inside the attempt loop (engine.ts:646-649), and the `start` event was built from the pre-auth `baseReq`. So declarative auth never enters the event stream at all — not even for a custom sink', + ); + } + { + const sink = collectingSink(); + const vendor = fakeVendor(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: vendor, + trace: sink, + auth: apiKey({ + in: 'query', + name: 'api_key', + secret: () => QUERY_KEY, + }), + }); + await call(); + check( + 'the query key DID reach the wire', + (vendor.seen()[0]?.url ?? '').includes(QUERY_KEY), + true, + ); + const row = leakRow( + 'raw event spine — apiKey in: query', + sink.text(), + CREDENTIALS, + 'same, with the key appended to the URL post-start', + ); + check('and still nothing on the spine', row.hits.size, 0); + } + { + const sink = collectingSink(); + const vendor = fakeVendor(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: vendor, + trace: sink, + auth: apiKey({ + in: 'cookie', + name: 'sid', + secret: () => COOKIE_VALUE, + }), + }); + await call(); + check( + 'the cookie DID reach the wire', + (vendor.seen()[0]?.headers['cookie'] ?? '').includes(COOKIE_VALUE), + true, + ); + const row = leakRow( + 'raw event spine — apiKey in: cookie', + sink.text(), + CREDENTIALS, + 'same, with the key on the Cookie header post-start', + ); + check('nothing on the spine', row.hits.size, 0); + } + + heading( + 'C4 (b) — HAND-ROLLED credentials, passed as per-call input: the raw event DOES carry them', + ); + { + const sink = collectingSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: sink, + }); + await call({ + headers: { + authorization: `Bearer ${BEARER_TOKEN}`, + cookie: `sid=${COOKIE_VALUE}`, + }, + query: { api_key: QUERY_KEY }, + }); + const row = leakRow( + 'raw event spine — hand-rolled headers', + sink.text(), + CREDENTIALS, + 'what a CUSTOM sink receives', + ); + check( + 'all three credentials are on the raw `start` event', + row.hits.size, + 3, + ); + note( + '→ this is what `trace.ts` warns about in prose ("a custom sink receives the RAW event, so a `start` event\'s `input.headers` still holds `authorization`/`cookie`"), measured. Core redacts INSIDE its own sinks, not on the event', + ); + } + + heading('C4 (c) — the same call through each BUILT-IN sink'); + { + const t = tempFileSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: t.sink, + }); + await call({ + headers: { + authorization: `Bearer ${BEARER_TOKEN}`, + cookie: `sid=${COOKIE_VALUE}`, + }, + query: { api_key: QUERY_KEY }, + }); + const text = t.text(); + const row = leakRow( + 'fileSink (default) — hand-rolled', + text, + CREDENTIALS, + 'the JSONL on disk', + ); + check('the JSONL sink redacts all three', row.hits.size, 0); + check( + 'header → [REDACTED]', + /"authorization":"\[REDACTED\]"/.test(text), + true, + ); + check( + 'cookie → [REDACTED]', + /"cookie":"\[REDACTED\]"/.test(text), + true, + ); + check( + 'the structured query value → [REDACTED]', + /"api_key":"\[REDACTED\]"/.test(text), + true, + ); + check( + 'and the URL string is scrubbed too — as REDACTED, no brackets', + /api_key=REDACTED/.test(text), + true, + ); + note( + 'one record, one credential, TWO sentinels: `input.query.api_key` is `"[REDACTED]"` (trace.ts\'s constant) while `url` carries `api_key=REDACTED` (util.ts\'s `URL_REDACTED`). Harmless until someone greps their aggregator for one spelling', + ); + t.cleanup(); + } + { + const cap = captureStderr(); + try { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: consoleSink(), + }); + await call({ + headers: { authorization: `Bearer ${BEARER_TOKEN}` }, + query: { api_key: QUERY_KEY }, + }); + } finally { + cap.restore(); + } + const row = leakRow( + 'consoleSink — hand-rolled', + cap.text(), + CREDENTIALS, + 'stderr', + ); + check('consoleSink: nothing', row.hits.size, 0); + } + { + const logger = captureLogger(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: loggerSink(logger), + }); + await call({ + headers: { authorization: `Bearer ${BEARER_TOKEN}` }, + query: { api_key: QUERY_KEY }, + }); + const row = leakRow( + 'loggerSink — hand-rolled', + logger.text(), + CREDENTIALS, + 'the messages handed to the logger', + ); + check('loggerSink: nothing', row.hits.size, 0); + note('the `start` line it logged', logger.lines()[0]?.message); + } + { + const spans: OtelSpan[] = []; + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + trace: otlpSink({ + exporter: { export: (s) => void spans.push(...s) }, + }), + }); + await call({ + headers: { authorization: `Bearer ${BEARER_TOKEN}` }, + query: { api_key: QUERY_KEY }, + }); + const row = leakRow( + 'otlpSink — hand-rolled', + bytesOf(spans), + CREDENTIALS, + 'the exported spans', + ); + check('otlpSink: nothing', row.hits.size, 0); + note( + '`url.full` after scrubbing', + (spans[0]?.attributes as Record | undefined)?.[ + 'url.full' + ], + ); + } + + heading('C4 (d) — the probes and the error path'); + { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + store: recordingStore(), + }); + const w = await call.inspect({ + headers: { authorization: `Bearer ${BEARER_TOKEN}` }, + query: { api_key: QUERY_KEY }, + }); + const row = leakRow( + 'JSON.stringify(inspect())', + bytesOf(w), + CREDENTIALS, + 'the whole wrapper', + ); + check('the inspection carries no credential', row.hits.size, 0); + const r = await call.report({ + headers: { authorization: `Bearer ${BEARER_TOKEN}` }, + }); + const rrow = leakRow( + 'JSON.stringify(report())', + bytesOf(r), + CREDENTIALS, + 'the whole report, config echo included', + ); + check('the report carries none either', rrow.hits.size, 0); + note( + 'the report echoes the REDACTED `__config`, and `auth` is a `dropped: redact` slot (config-anatomy.ts:120) — so even a configured strategy is absent, projected down to a non-secret `authScheme`', + ); + } + { + const store = recordingStore(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + store, + cache: { ttl: '60s' }, + auth: bearer(() => BEARER_TOKEN), + }); + await call({ headers: { authorization: `Bearer ${BEARER_TOKEN}` } }); + const row = leakRow( + 'cache (keys + values)', + store.text() + JSON.stringify(store.writes().map((wr) => wr.key)), + CREDENTIALS, + 'both what was stored and the key it was stored under', + ); + check('no credential in the cache', row.hits.size, 0); + note( + 'the cache key is a hash of the pre-auth request descriptor, so a header credential neither lands in the key nor in the value', + ); + } + { + const sink = collectingSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ status: 401 }), + trace: sink, + auth: bearer(() => BEARER_TOKEN), + }); + const out = await call.safe(); + check('the call failed', out.ok, false); + const row = leakRow( + 'StitchError (message + body + stack)', + `${out.error?.message ?? ''}${bytesOf(out.error?.body)}${out.error?.stack ?? ''}`, + CREDENTIALS, + 'everything a catch block can reach', + ); + check('a 401 error carries no credential', row.hits.size, 0); + note('the message', out.error?.message); + } + + heading( + 'C4 (e) — where the boundary does NOT hold: a credential in a RESPONSE body', + ); + { + // The vendor is a token endpoint (or a debug endpoint echoing the session). The response + // body is response data, and the JSONL sink's redactor is the five-name HEADER denylist — + // `isSecretKey` (which knows `access_token`) is never applied to a `result`. + const t = tempFileSink(); + const call = stitch({ + name: 'refreshSession', + baseUrl: BASE, + path: '/v1/oauth/token', + method: 'POST', + adapter: fakeVendor({ + body: { + access_token: BEARER_TOKEN, + refresh_token: QUERY_KEY, + session_cookie: COOKIE_VALUE, + }, + }), + trace: t.sink, + }); + await call(); + const text = t.text(); + const row = leakRow( + 'fileSink — credential in the RESPONSE body', + text, + CREDENTIALS, + 'the JSONL on disk', + ); + check( + 'all three credentials are written to the log file in full', + row.hits.size, + 3, + ); + note( + '→ REFUTATION of the clean reading. "Credentials are protected by default" holds for credentials the LIBRARY places (auth strategies) and for credentials on the REQUEST (headers/url/query, all scrubbed). It does not hold for a credential the vendor SENDS BACK. `util.ts` ships `isSecretKey`, which knows `access_token`/`refresh_token` by stem, and `redactEventForTransport` already applies `redactSecretsDeep` to a start `input.body` for the `serve` SSE stream — the JSONL/console sinks simply never call it on a `result`', + ); + t.cleanup(); + } + { + // And the contrast that makes it precise: the SAME event, delivered over `stitch serve`'s + // SSE transport, is deep-scrubbed on the request side. Two redaction policies, one library. + const cap = captureStderr(); + try { + const call = stitch({ + name: 'login', + baseUrl: BASE, + path: '/v1/login', + method: 'POST', + adapter: fakeVendor(), + trace: consoleSink(), + }); + await call({ body: { user: 'w', client_secret: QUERY_KEY } }); + } finally { + cap.restore(); + } + const row = leakRow( + 'consoleSink — secret in the REQUEST body', + cap.text(), + CREDENTIALS, + 'stderr (the console formatter prints no body at all)', + ); + check( + 'console prints no body, so nothing leaks there', + row.hits.size, + 0, + ); + const t = tempFileSink(); + const call2 = stitch({ + name: 'login', + baseUrl: BASE, + path: '/v1/login', + method: 'POST', + adapter: fakeVendor(), + trace: t.sink, + }); + await call2({ body: { user: 'w', client_secret: QUERY_KEY } }); + const jrow = leakRow( + 'fileSink — secret in the REQUEST body', + t.text(), + CREDENTIALS, + 'the JSONL on disk', + ); + check( + 'but the JSONL sink writes a `client_secret` request body verbatim', + jrow.hits.size, + 1, + ); + note( + "→ a second, sharper version of the same gap: `redactEventForTransport` (trace.ts:85) deep-scrubs exactly this — a `start` frame's `input.body` — before it rides the unauthenticated `stitch serve` SSE stream. The JSONL file sink, writing to your disk, does not. The mechanism exists in the same file; it is wired to one transport only", + ); + t.cleanup(); + } + + const tally = printLeakTable(CREDENTIALS); + console.log( + `\n ${tally.leaking} destination(s) carry a credential; ${tally.clean} carry none.`, + ); + check( + 'sanity: the three credential literals are distinct', + new Set([BEARER_TOKEN, QUERY_KEY, COOKIE_VALUE]).size, + 3, + ); + check( + 'sanity: no credential literal is itself a secret-looking KEY name', + scan(`${BEARER_TOKEN}${QUERY_KEY}${COOKIE_VALUE}`, CREDENTIALS).size, + 3, + ); + + finish( + 'C4', + 'PARTIAL — the credential half is genuinely safer than the PII half, and "protected by default" is too strong. What holds: a DECLARATIVE strategy (`bearer`, `apiKey` in query/cookie) never enters the event stream at all, because `auth.apply` runs on a request clone inside the attempt loop while the `start` event was built from the pre-auth `baseReq` — 0 of 3 credentials on the raw spine, even for a naive custom sink. Hand-rolled request credentials are scrubbed by every built-in sink: the JSONL file gets `"authorization":"[REDACTED]"`, `"cookie":"[REDACTED]"`, `"api_key":"REDACTED"` and a scrubbed `url`; console, logger and OTLP print no headers at all; `.inspect()`, `.report()`, the cache (key AND value) and a 401 `StitchError` carry none. What does NOT hold, two ways, both measured: (1) a CUSTOM sink receives the raw event, so hand-rolled `input.headers`/`input.query` reach it in the clear — 3 of 3 — which `trace.ts` documents in prose and this confirms; (2) a credential in a RESPONSE body is written to the JSONL log in full — `access_token`, `refresh_token`, `session_cookie`, 3 of 3 — because the file sink\'s redactor is the five-name HEADER denylist, not `isSecretKey`. The same file already ships the deep secret-key scrubber and applies it to a request body for the `serve` SSE transport (`redactEventForTransport`); the disk sink simply never calls it. A `client_secret` in a REQUEST body is written verbatim for the same reason. So the honest split is: credentials the library PLACES are protected; credentials that ride the payload are treated exactly like customer PII, which is to say not at all', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c5-boundary.ts b/docs/scenarios/proofs/pii-in-the-logs/c5-boundary.ts new file mode 100644 index 00000000..4064343b --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c5-boundary.ts @@ -0,0 +1,354 @@ +// C5 — can PII be stripped AT THE BOUNDARY, before anything copies it? +// +// Three candidate seams: `hooks.onResponse`, a surface's `interpret`, and `transform`. The capture +// asks which runs earliest and whether a value stripped there stays out of the trace sink AND the +// cache. Both questions are answered by measurement — the ordering by an execution log the three +// seams write to in the order they actually fire, the coverage by re-running the C1 battery with +// each seam installed. +// +// The result has a shape the capture does not anticipate: the three seams are NOT interchangeable +// with different ergonomics. They cover DIFFERENT SETS of destinations, and only one of them — +// `hooks.onResponse` — covers the failure path, because on a non-2xx neither `interpret`'s return +// value nor `transform` is ever consulted: the engine throws carrying the untouched `res`. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c5-boundary.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + AdapterResponse, + ResolvedStitchConfig, + StitchConfig, +} from '../../../../packages/core/src/types'; +import { + BASE, + SENTINELS, + bytesOf, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { + check, + checkSeq, + finish, + heading, + leakRow, + note, + printLeakTable, + scan, +} from './harness'; + +/** The strip every seam applies: keep the two operational fields, drop everything else. */ +function safeShape(body: unknown): unknown { + const b = (body ?? {}) as Record; + return { id: b['id'], plan: b['plan'] }; +} + +/** A surface whose `interpret` returns the stripped body as the value. */ +const strippingSurface: Surface = { + id: 'stripping', + interpret: (res: AdapterResponse, cfg: ResolvedStitchConfig) => + verdictOf(res, cfg) ?? { ok: true, data: safeShape(res.body) }, +}; + +/** Run the destination battery for one config and report sentinel counts per destination. */ +async function battery( + extra: Partial, + opts: { status?: number } = {}, +): Promise> { + const hits = new Map(); + const t = tempFileSink(); + const sink = collectingSink(); + const store = recordingStore(); + const base = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + ...extra, + } as StitchConfig; + + const traced = stitch({ + ...base, + adapter: fakeVendor({ status: opts.status ?? 200 }), + trace: t.sink, + store, + }); + await traced.safe(); + hits.set('fileSink (JSONL)', scan(t.text(), SENTINELS).size); + hits.set('cache entry', scan(store.text(), SENTINELS).size); + t.cleanup(); + + const spined = stitch({ + ...base, + adapter: fakeVendor({ status: opts.status ?? 200 }), + trace: sink, + store: recordingStore(), + }); + await spined.safe(); + hits.set('raw event spine', scan(sink.text(), SENTINELS).size); + + const probed = stitch({ + ...base, + adapter: fakeVendor({ status: opts.status ?? 200 }), + store: recordingStore(), + }); + const w = await probed.inspect(); + hits.set('.inspect().raw', scan(bytesOf(w.raw), SENTINELS).size); + hits.set('.inspect().data', scan(bytesOf(w.data), SENTINELS).size); + + const failing = stitch({ + ...base, + adapter: fakeVendor({ status: opts.status ?? 200 }), + store: recordingStore(), + }); + const out = await failing.safe(); + hits.set( + 'StitchError.body', + scan(bytesOf(out.error?.body), SENTINELS).size, + ); + return hits; +} + +function printBattery( + label: string, + m: Map, + baseline?: Map, +): void { + const w = Math.max(...[...m.keys()].map((k) => k.length)); + console.log(`\n ${label}`); + for (const [k, v] of m) { + const b = baseline?.get(k); + const delta = + b === undefined ? '' : b === v ? ' (unchanged)' : ` was ${b}`; + console.log(` ${k.padEnd(w)} ${String(v).padStart(2)}${delta}`); + } +} + +async function main(): Promise { + heading('C5 (a) — the ORDER the three seams actually fire in'); + { + const order: string[] = []; + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + kind: { + id: 'ordered', + interpret: (res, cfg) => { + order.push('interpret'); + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }, + hooks: { + onRequest: () => void order.push('hooks.onRequest'), + onResponse: () => void order.push('hooks.onResponse'), + }, + transform: (b) => { + order.push('transform'); + return b; + }, + output: { + '~standard': { + version: 1, + vendor: 'proof', + validate: (v: unknown) => { + order.push('output.validate'); + return { value: v }; + }, + }, + } as never, + }); + await call(); + checkSeq('measured seam order', order, [ + 'hooks.onRequest', + 'hooks.onResponse', + 'interpret', + 'transform', + 'output.validate', + ]); + note( + '→ `hooks.onResponse` is the EARLIEST seam that sees a response body: engine.ts:705, immediately after the transport returns and before the surface is asked what the response means (engine.ts:775). `transform` is third, output validation last', + ); + } + + heading('C5 (b) — the baseline: no seam installed'); + const baseline = await battery({}); + printBattery('no stripping', baseline); + check( + 'baseline: the JSONL holds all 7', + baseline.get('fileSink (JSONL)'), + 7, + ); + + heading('C5 (c) — `hooks.onResponse` mutating `res.body`'); + { + const withHook = await battery({ + hooks: { + onResponse: ({ res }) => { + if (res) res.body = safeShape(res.body); + }, + }, + }); + printBattery('hooks.onResponse', withHook, baseline); + check('JSONL: clean', withHook.get('fileSink (JSONL)'), 0); + check('cache: clean', withHook.get('cache entry'), 0); + check('raw event spine: clean', withHook.get('raw event spine'), 0); + check('.inspect().raw: clean', withHook.get('.inspect().raw'), 0); + check('.inspect().data: clean', withHook.get('.inspect().data'), 0); + note( + 'the hook is typed `(ctx) => void | Promise`, so this works by MUTATING `ctx.res.body` in place — the engine passes the live `AdapterResponse` and keeps using it. There is no return-a-new-body form', + ); + } + + heading('C5 (d) — a surface `interpret` that returns the stripped value'); + { + const withSurface = await battery({ kind: strippingSurface }); + printBattery('kind: strippingSurface', withSurface, baseline); + check('JSONL: clean', withSurface.get('fileSink (JSONL)'), 0); + check('cache: clean', withSurface.get('cache entry'), 0); + check('.inspect().raw: clean', withSurface.get('.inspect().raw'), 0); + } + + heading('C5 (e) — `transform`'); + { + const withTransform = await battery({ transform: safeShape }); + printBattery('transform', withTransform, baseline); + check('JSONL: clean', withTransform.get('fileSink (JSONL)'), 0); + check('cache: clean', withTransform.get('cache entry'), 0); + check( + '.inspect().raw: ALSO clean — `raw` is captured AFTER transform', + withTransform.get('.inspect().raw'), + 0, + ); + note( + '→ worth pinning: `.inspect().raw` is documented as "the pre-validation body", and pre-validation is exactly what it is — engine.ts:1202 takes `rawBody = value` after `transform` and `pick` have already run. So `transform` covers `raw`, but a drift diff computed against it can no longer see what the vendor really sent', + ); + } + + heading( + 'C5 (f) — the failure path, where the three seams STOP being equivalent', + ); + { + const failBaseline = await battery({}, { status: 500 }); + check( + 'baseline 500: `StitchError.body` holds all 7', + failBaseline.get('StitchError.body'), + 7, + ); + const failTransform = await battery( + { transform: safeShape }, + { status: 500 }, + ); + check( + '`transform` does NOT protect it — the transform never runs on a 500', + failTransform.get('StitchError.body'), + 7, + ); + const failSurface = await battery( + { kind: strippingSurface }, + { status: 500 }, + ); + check( + 'a stripping `interpret` does not either — its value is discarded, the raw `res` is thrown', + failSurface.get('StitchError.body'), + 7, + ); + const failHook = await battery( + { + hooks: { + onResponse: ({ res }) => { + if (res) res.body = safeShape(res.body); + }, + }, + }, + { status: 500 }, + ); + check( + '`hooks.onResponse` DOES — it mutated the object the engine later attaches', + failHook.get('StitchError.body'), + 0, + ); + note( + "→ the decisive C5 result. On a non-2xx the engine builds `e.response = res` from the untouched adapter response (engine.ts:824-831); `transform`/`pick`/output validation are never reached and `interpret`'s success value is discarded. Only a seam that MUTATED `res` in place is still in effect. That makes `hooks.onResponse` the only one of the three that is a boundary in the sense the scenario means", + ); + } + + heading('C5 (g) — the leak table for the winning seam'); + { + const t = tempFileSink(); + const store = recordingStore(); + const sink = collectingSink(); + const cfg = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + hooks: { + onResponse: ({ res }: { res?: AdapterResponse }) => { + if (res) res.body = safeShape(res.body); + }, + }, + } as StitchConfig; + const a = stitch({ + ...cfg, + adapter: fakeVendor(), + trace: t.sink, + store, + }); + await a(); + leakRow('fileSink', t.text(), SENTINELS, 'with onResponse installed'); + leakRow('cache entry', store.text(), SENTINELS, ''); + const b = stitch({ + ...cfg, + adapter: fakeVendor(), + trace: sink, + store: recordingStore(), + }); + await b(); + leakRow('raw event spine', sink.text(), SENTINELS, ''); + const c = stitch({ + ...cfg, + adapter: fakeVendor(), + store: recordingStore(), + }); + const w = await c.inspect(); + leakRow('.inspect() wrapper', bytesOf(w), SENTINELS, ''); + leakRow('.inspect().raw', bytesOf(w.raw), SENTINELS, ''); + const r = await c.report(); + leakRow('.report()', bytesOf(r), SENTINELS, ''); + const d = stitch({ + ...cfg, + adapter: fakeVendor({ status: 500 }), + store: recordingStore(), + }); + const out = await d.safe(); + leakRow('StitchError.body', bytesOf(out.error?.body), SENTINELS, ''); + // The one place it CANNOT reach: the transport itself already saw the bytes. + const vendor = fakeVendor(); + const e = stitch({ ...cfg, adapter: vendor, store: recordingStore() }); + await e(); + leakRow( + 'the Adapter (upstream of every seam)', + bytesOf(vendor.seen()), + SENTINELS, + 'the request only — the response never passes back through it', + ); + const tally = printLeakTable(SENTINELS); + check( + 'every destination is clean with one 3-line hook', + tally.leaking, + 0, + ); + t.cleanup(); + } + + finish( + 'C5', + 'CONFIRMED, with an ordering result that changes the answer. Measured seam order on a live call: hooks.onRequest → hooks.onResponse → interpret → transform → output.validate. `hooks.onResponse` (engine.ts:705) is the EARLIEST seam that can see a response body — it fires immediately after the transport returns and before the surface is asked to interpret. On the SUCCESS path all three candidate seams work and are equivalent: each one takes the JSONL sink, the cache entry, the raw event spine, `.inspect().raw` and `.inspect().data` from 7 sentinels to 0. On the FAILURE path they are not: a 500 leaves `StitchError.body` at 7/7 under `transform` and at 7/7 under a stripping `interpret`, because the engine throws carrying the untouched `res` (engine.ts:824-831) — `transform` is never reached and `interpret`\'s success value is discarded. Only `hooks.onResponse` still holds, at 0/7, because it MUTATED the response object the error later carries. So there is exactly one seam that is a boundary in the sense this scenario means, it is three lines, and its type is `(ctx) => void` — you strip by mutating `ctx.res.body` in place, which is nowhere described as a privacy mechanism. Two side measurements: `.inspect().raw` is captured AFTER transform/pick (engine.ts:1202), so "pre-validation body" is literal and a transform-based strip also blinds the drift diff; and no seam reaches the Adapter, which necessarily saw the bytes first', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c6-allowlist.ts b/docs/scenarios/proofs/pii-in-the-logs/c6-allowlist.ts new file mode 100644 index 00000000..0667a973 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c6-allowlist.ts @@ -0,0 +1,409 @@ +// C6 — is an ALLOWLIST expressible? Does an `output` schema that strips unknown keys keep them out +// of the trace, the log and the cache? +// +// Scenario 20 measured that `output` DOES use its parsed value (unlike `input`, which discards it — +// issue #648). So a stripping schema should genuinely filter, and it does: with a four-field Zod +// object declared as `output`, the JSONL sink, the console line, the `result` event, the cache +// entry and `.inspect().data` all go from 7 sentinels to 0, at every depth and inside array +// elements, with no field names enumerated anywhere. +// +// And then there is the residue, which is the part worth writing down. An allowlist that filters +// five destinations leaves two carrying the full record — `.inspect().raw`, by design, and +// `StitchError.body` on any non-2xx, because validation never runs on a failure. The second one is +// not a design decision anybody made; it is where the two mechanisms simply do not meet. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c6-allowlist.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { + BASE, + SENTINELS, + bytesOf, + canary, + captureStderr, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { + check, + checkSeq, + finish, + heading, + leakRow, + note, + printLeakTable, + scan, +} from './harness'; +import { z } from './zod'; + +// The allowlist. Four fields, declared positively — nothing about `email`/`ssn`/`mail` appears +// anywhere in it, which is the whole difference from C3's denylist. +const SAFE = z.object({ + id: z.string(), + plan: z.string(), + profile: z.object({ locale: z.string() }), + contacts: z.array(z.object({ label: z.string() })), +}); + +async function battery(extra: Partial, status = 200) { + const hits = new Map(); + const base = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + ...extra, + } as StitchConfig; + + const t = tempFileSink(); + const store = recordingStore(); + const a = stitch({ + ...base, + adapter: fakeVendor({ status }), + trace: t.sink, + store, + }); + await a.safe(); + hits.set('fileSink (JSONL)', scan(t.text(), SENTINELS).size); + hits.set('cache entry', scan(store.text(), SENTINELS).size); + t.cleanup(); + + const sink = collectingSink(); + const b = stitch({ + ...base, + adapter: fakeVendor({ status }), + trace: sink, + store: recordingStore(), + }); + await b.safe(); + hits.set('raw event spine', scan(sink.text(), SENTINELS).size); + + const cap = captureStderr(); + try { + const c = stitch({ + ...base, + adapter: fakeVendor({ status }), + trace: 'console', + store: recordingStore(), + }); + await c.safe(); + } finally { + cap.restore(); + } + hits.set('consoleSink', scan(cap.text(), SENTINELS).size); + + const d = stitch({ + ...base, + adapter: fakeVendor({ status }), + store: recordingStore(), + }); + const w = await d.inspect(); + hits.set('.inspect().data', scan(bytesOf(w.data), SENTINELS).size); + hits.set('.inspect().raw', scan(bytesOf(w.raw), SENTINELS).size); + hits.set('.inspect() wrapper', scan(bytesOf(w), SENTINELS).size); + const out = await d.safe(); + hits.set( + 'StitchError.body', + scan(bytesOf(out.error?.body), SENTINELS).size, + ); + return hits; +} + +function printBattery( + label: string, + m: Map, + baseline?: Map, +): void { + const w = Math.max(...[...m.keys()].map((k) => k.length)); + console.log(`\n ${label}`); + for (const [k, v] of m) { + const b = baseline?.get(k); + const delta = + b === undefined ? '' : b === v ? ' (unchanged)' : ` was ${b}`; + console.log(` ${k.padEnd(w)} ${String(v).padStart(2)}${delta}`); + } +} + +async function main(): Promise { + heading( + 'C6 (a) — the schema really does strip (the premise, checked directly)', + ); + { + const parsed = SAFE.parse(canary()); + checkSeq( + 'the parsed value keeps only the declared keys', + Object.keys(parsed).sort(), + ['contacts', 'id', 'plan', 'profile'], + ); + check( + 'nested: `profile.contact` is gone', + 'contact' in (parsed.profile as object), + false, + ); + check( + 'array elements: `contacts[1].email` is gone', + 'email' in ((parsed.contacts as object[])[1] ?? {}), + false, + ); + check( + 'and nothing in the parsed value matches any sentinel', + scan(bytesOf(parsed), SENTINELS).size, + 0, + ); + } + + heading('C6 (b) — the battery, without and with the `output` allowlist'); + const baseline = await battery({}); + printBattery('no output schema', baseline); + const allow = await battery({ output: SAFE }); + printBattery('output: SAFE (strips unknown keys)', allow, baseline); + { + check( + 'fileSink: 7 → 0 — the log is clean', + allow.get('fileSink (JSONL)'), + 0, + ); + check('cache entry: 7 → 0', allow.get('cache entry'), 0); + check('raw event spine: 7 → 0', allow.get('raw event spine'), 0); + check('.inspect().data: 7 → 0', allow.get('.inspect().data'), 0); + check( + '.inspect() wrapper: 7 → 0 (`data` was the enumerable leak in C1(e))', + allow.get('.inspect() wrapper'), + 0, + ); + check( + 'but `.inspect().raw` is UNCHANGED at 7 — validation runs after `raw` is captured', + allow.get('.inspect().raw'), + 7, + ); + note( + '→ scenario 20 confirmed on the output side: the engine serves `value = validated` (engine.ts:1223), so a stripping schema is a real filter and not merely a check. The `input` side discards its parsed value (issue #648); the `output` side does not', + ); + } + + heading('C6 (c) — the residue: the failure path'); + { + const failing = await battery({ output: SAFE }, 500); + printBattery('output: SAFE, vendor returns 500', failing); + check( + '`StitchError.body` still holds all 7 on a 500', + failing.get('StitchError.body'), + 7, + ); + check( + 'and so does the JSONL? No — the sink never sees a body it was not given', + failing.get('fileSink (JSONL)'), + 0, + ); + check( + 'and — NOT IN THE CLAIMS — `JSON.stringify(inspection)` is back to 7 on a failure', + failing.get('.inspect() wrapper'), + 7, + ); + note( + '→ the route is the enumerable `error` field: `StitchError` assigns `this.body` in its constructor, so it is an OWN ENUMERABLE property and `JSON.stringify(err)` emits `{"name":"StitchError","status":500,"attempts":1,"body":{…}}`. `err.stack` and `String(err)` are clean (C1(g) measured 0), so the leak is specific to JSON-serialising the error — which is exactly what a structured logger does', + ); + note( + '→ the allowlist is airtight on every destination that reads the VALIDATED value and absent on every destination that reads the RESPONSE. Output validation is stage 7; a non-2xx never reaches it. So an `output` allowlist plus an unguarded `catch (e) { log.error(e.body) }` is a complete filter with a hole exactly where an incident actually gets logged', + ); + } + + heading('C6 (d) — the three unknown-key modes, measured'); + { + const loose = SAFE.passthrough(); + const strict = SAFE.strict(); + const l = await battery({ output: loose }); + check( + '`.passthrough()` returns the JSONL to 5 of 7 — not 7', + l.get('fileSink (JSONL)'), + 5, + ); + note( + "→ measured, not assumed: `.passthrough()` is SHALLOW. The two sentinels it does NOT restore are exactly the two that sit inside a nested `z.object` — `profile.contact.mail` and `contacts[1].email` — because the inner objects are still stripping. So Zod's unknown-key mode is per-object, and an allowlist leaks at whatever depth you relaxed it", + ); + const s = await battery({ output: strict }); + check( + '`.strict()` FAILS the call instead of filtering — JSONL clean, but…', + s.get('fileSink (JSONL)'), + 0, + ); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + output: strict, + }); + const out = await call.safe(); + check('… the call fails', out.ok, false); + check( + "the ERROR message is the engine's, not the schema's", + out.error?.message, + 'contract violation (drift)', + ); + const strictProbe = await call.inspect(); + check( + "the schema's complaint rides the FINDINGS instead", + strictProbe.findings.some((f) => /ssn/.test(f.detail ?? '')), + true, + ); + note( + 'the finding detail', + strictProbe.findings.find((f) => f.change === 'invalid')?.detail, + ); + check( + 'and no finding carries a VALUE', + scan(bytesOf(strictProbe.findings), SENTINELS).size, + 0, + ); + note( + "→ NOT IN THE CLAIMS: `.strict()`'s complaint enumerates the undeclared KEY NAMES into a `DriftFinding.detail` that every sink logs — `Unrecognized key(s) in object: 'name', 'email', 'ssn', …`. Names, never values (0 sentinels), but the strictest allowlist is also the one that writes a field inventory of the vendor's response into your log", + ); + const strictSink = tempFileSink(); + const traced = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + output: strict, + trace: strictSink.sink, + }); + await traced.safe(); + check( + 'and the JSONL does log that message, sentinel-free', + scan(strictSink.text(), SENTINELS).size, + 0, + ); + check('with the key names in it', /ssn/.test(strictSink.text()), true); + strictSink.cleanup(); + } + + heading('C6 (e) — the allowlist under `drift()`: filtering AND a signal'); + { + const spec = drift(SAFE, { severity: 'info' }); + const sink = collectingSink(); + const t = tempFileSink(); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + output: spec, + trace: t.sink, + }); + const w = await call.inspect(); + check('`data` is filtered', scan(bytesOf(w.data), SENTINELS).size, 0); + check( + 'and every stripped field is reported as an `undeclared` finding', + w.findings.filter((f) => f.change === 'undeclared').length >= 5, + true, + ); + checkSeq( + 'the finding PATHS name the stripped fields', + w.findings + .filter((f) => f.change === 'undeclared') + .map((f) => f.path) + .sort(), + [ + 'contacts[].email', + 'email', + 'name', + 'note', + 'primaryContactMail', + 'profile.contact', + 'ssn', + ], + ); + check( + 'and no finding carries a VALUE — 0 sentinels across every finding', + scan(bytesOf(w.findings), SENTINELS).size, + 0, + ); + const traced = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + output: spec, + trace: sink, + }); + await traced(); + check( + 'the drift events on the spine are sentinel-free too', + scan( + bytesOf(sink.events().filter((e) => e.type === 'drift')), + SENTINELS, + ).size, + 0, + ); + check( + 'and the JSONL that logged all of them is clean', + scan(t.text(), SENTINELS).size, + 0, + ); + note( + '→ this is the combination the capture calls "correct by construction": an allowlist that filters the value AND emits a named, value-free inventory of everything it filtered. `detailFor` (drift.ts:77) emits KINDS only — `undeclared field (string)` — so the diagnostic that tells you a PII field appeared does not itself contain the PII', + ); + t.cleanup(); + } + + heading('C6 (f) — the leak table under the assembled allowlist'); + { + const t = tempFileSink(); + const store = recordingStore(); + const cfg = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + output: drift(SAFE, { severity: 'info' }), + } as StitchConfig; + const a = stitch({ + ...cfg, + adapter: fakeVendor(), + trace: t.sink, + store, + }); + await a(); + leakRow('fileSink', t.text(), SENTINELS, 'output allowlist only'); + leakRow('cache entry', store.text(), SENTINELS, ''); + const b = stitch({ + ...cfg, + adapter: fakeVendor(), + store: recordingStore(), + }); + const w = await b.inspect(); + leakRow('.inspect() wrapper', bytesOf(w), SENTINELS, ''); + leakRow('.inspect().raw', bytesOf(w.raw), SENTINELS, 'THE RESIDUE'); + leakRow('.inspect().findings', bytesOf(w.findings), SENTINELS, ''); + const c = stitch({ + ...cfg, + adapter: fakeVendor({ status: 500 }), + store: recordingStore(), + }); + const out = await c.safe(); + leakRow( + 'StitchError.body (500)', + bytesOf(out.error?.body), + SENTINELS, + 'THE RESIDUE', + ); + const tally = printLeakTable(SENTINELS); + check( + 'destinations still leaking under the allowlist', + tally.leaking, + 2, + ); + check('destinations the allowlist covers', tally.clean, 4); + t.cleanup(); + } + + finish( + 'C6', + "CONFIRMED — an allowlist is expressible, it genuinely filters, and it is the only mechanism in this directory that survives a vendor adding a field. A four-field Zod `output` schema takes the JSONL sink, the `result` event / raw spine, the console line, the cache entry, `.inspect().data` and the whole `.inspect()` wrapper from 7 sentinels to 0, at depth (`profile.contact` gone) and inside array elements (`contacts[].email` gone), with no PII field name written anywhere. This confirms scenario 20 on the output side: the engine serves `value = validated` (engine.ts:1223), so `output` filters where `input` merely checks (issue #648). Wrapped in `drift()` it also emits a value-free inventory of everything it stripped — 7 `undeclared` findings whose paths name the fields and whose details are KINDS only (`undeclared field (string)`), 0 sentinels across every finding and every drift event. The residue is exactly two destinations and both are structural: `.inspect().raw` stays at 7/7 by design (it is captured before validation — it exists to show what the vendor really sent), and `StitchError.body` stays at 7/7 on any non-2xx because output validation is stage 7 and a failure never reaches it. Mode notes: `.passthrough()` returns the JSONL to 5/7 rather than 7/7, because it is SHALLOW — the two sentinels behind a nested `z.object` stay stripped; and `.strict()` fails the call rather than filtering, its message being the engine's `contract violation (drift)` while the schema's complaint rides a `DriftFinding.detail` that enumerates the undeclared KEY NAMES into every sink — names, never values. One finding outside the claims: on the failure path `JSON.stringify(inspection)` goes back to 7/7 through the enumerable `error` field, because `StitchError` assigns `this.body` in its constructor and `JSON.stringify(err)` therefore emits the whole response body (`err.stack` and `String(err)` stay clean)", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c7-drift-signal.ts b/docs/scenarios/proofs/pii-in-the-logs/c7-drift-signal.ts new file mode 100644 index 00000000..70392ba0 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c7-drift-signal.ts @@ -0,0 +1,497 @@ +// C7 — the drift angle. When a vendor ADDS a PII field, does `drift()`'s `undeclared` finding give +// a usable "new field appeared" signal? And — the question that matters more — does the FINDING +// itself contain the PII value? +// +// The answer to the first is yes, with one hard precondition and one hard limit. The answer to the +// second is yes for SOFT drift (paths and kinds only, never values, exactly as ADR 0018 §4 claims) +// and NO for HARD validation, where the finding `detail` is the validator's own message and Zod's +// enum/union messages quote the received value verbatim — into `consoleSink` and `loggerSink`, +// the two destinations C1 measured as carrying nothing. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c7-drift-signal.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { OtelSpan } from '../../../../packages/core/src/otlp'; +import { otlpSink } from '../../../../packages/core/src/otlp'; +import { consoleSink, loggerSink } from '../../../../packages/core/src/trace'; +import type { + DriftFinding, + DriftOptions, + StitchConfig, +} from '../../../../packages/core/src/types'; +import { + BASE, + SENTINELS, + SSN, + bytesOf, + canary, + captureLogger, + captureStderr, + collectingSink, + fakeVendor, + tempFileSink, +} from './canary'; +import type { Sentinel } from './harness'; +import { + check, + checkSeq, + finish, + heading, + leakRow, + note, + printLeakTable, + resetLeakTable, + scan, +} from './harness'; +import { z } from './zod'; + +// The consumer contract: four fields, positively declared (C6's allowlist). +const SAFE = z.object({ + id: z.string(), + plan: z.string(), + profile: z.object({ locale: z.string() }), + contacts: z.array(z.object({ label: z.string() })), +}); + +// The new field the vendor ships in a minor release. A distinct sentinel so its appearance in any +// destination is unambiguous. +const TAX_ID = 'taxid-canary-GB-4471'; +const NEW_FIELD: readonly Sentinel[] = [ + { code: 'tax', value: TAX_ID, at: 'taxId (added by the vendor)' }, +]; + +/** The vendor's response, one release later: the same record plus a tax identifier. */ +function afterTheRelease(): Record { + const body = canary(); + return { + ...body, + taxId: TAX_ID, + profile: { ...(body['profile'] as object), taxId: TAX_ID }, + contacts: (body['contacts'] as Record[]).map((c) => ({ + ...c, + taxId: TAX_ID, + })), + }; +} + +async function findingsFor( + body: unknown, + opts?: DriftOptions, +): Promise { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body }), + output: drift(SAFE, opts ?? {}), + }); + return (await call.inspect()).findings; +} + +async function main(): Promise { + heading('C7 (a) — the baseline, before the vendor changed anything'); + const before = await findingsFor(canary()); + { + checkSeq( + 'undeclared paths in the original response', + before + .filter((f) => f.change === 'undeclared') + .map((f) => f.path) + .sort(), + [ + 'contacts[].email', + 'email', + 'name', + 'note', + 'primaryContactMail', + 'profile.contact', + 'ssn', + ], + ); + note( + 'so the signal is NOISY at rest: seven findings on a response nobody changed. The "new field appeared" event has to be read as a DIFF against this, not as an alert', + ); + } + + heading('C7 (b) — the vendor ships `taxId`: does anything notice?'); + const after = await findingsFor(afterTheRelease()); + { + const beforePaths = new Set(before.map((f) => f.path)); + const added = after + .filter((f) => !beforePaths.has(f.path)) + .map((f) => f.path) + .sort(); + checkSeq('paths that are NEW this release', added, [ + 'contacts[].taxId', + 'profile.taxId', + 'taxId', + ]); + check( + 'all three are `undeclared`', + after + .filter((f) => added.includes(f.path)) + .every((f) => f.change === 'undeclared'), + true, + ); + check( + 'at level `info` by default', + after + .filter((f) => added.includes(f.path)) + .every((f) => f.level === 'info'), + true, + ); + note( + 'the top-level finding', + after.find((f) => f.path === 'taxId'), + ); + note( + 'the array finding — summarised across elements (ADR 0017)', + after.find((f) => f.path === 'contacts[].taxId'), + ); + note( + '→ YES, and it is precise: a top-level addition, an addition nested one level down, and an addition inside every array element are all reported separately, each with a path you can act on', + ); + } + + heading('C7 (c) — does the FINDING carry the value? (the leak question)'); + { + const row = leakRow( + 'the findings array', + bytesOf(after), + NEW_FIELD, + 'JSON of every DriftFinding', + ); + check( + '0 occurrences of the new value in any finding', + row.hits.size, + 0, + ); + checkSeq( + 'every `undeclared` detail is a KIND, never a value', + [ + ...new Set( + after + .filter((f) => f.change === 'undeclared') + .map((f) => f.detail), + ), + ].sort(), + [ + 'all 2 elements: undeclared field (string)', + 'undeclared field (object)', + 'undeclared field (string)', + ], + ); + check( + 'and no PII sentinel either', + scan(bytesOf(after), SENTINELS).size, + 0, + ); + } + + heading('C7 (d) — and through every sink the finding reaches'); + { + const cfg = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + output: drift(SAFE, {}), + } as StitchConfig; + const sink = collectingSink(); + const a = stitch({ + ...cfg, + adapter: fakeVendor({ body: afterTheRelease() }), + trace: sink, + }); + await a(); + const drifts = sink.events().filter((e) => e.type === 'drift'); + check('drift events on the spine', drifts.length, 10); + leakRow('drift events (raw)', bytesOf(drifts), NEW_FIELD, ''); + + const t = tempFileSink(); + const b = stitch({ + ...cfg, + adapter: fakeVendor({ body: afterTheRelease() }), + trace: t.sink, + }); + await b(); + leakRow('fileSink', t.text(), NEW_FIELD, 'JSONL on disk'); + check( + 'the JSONL names the new path', + t.text().includes('"path":"taxId"'), + true, + ); + t.cleanup(); + + const cap = captureStderr(); + try { + const c = stitch({ + ...cfg, + adapter: fakeVendor({ body: afterTheRelease() }), + trace: consoleSink(), + }); + await c(); + } finally { + cap.restore(); + } + leakRow('consoleSink', cap.text(), NEW_FIELD, 'stderr'); + note( + 'the console line for the new field', + cap + .text() + .replace(/\x1b\[\d+m/g, '') + .split('\n') + .find((l) => l.includes('taxId')), + ); + + const logger = captureLogger(); + const d = stitch({ + ...cfg, + adapter: fakeVendor({ body: afterTheRelease() }), + trace: loggerSink(logger), + }); + await d(); + leakRow('loggerSink', logger.text(), NEW_FIELD, ''); + check( + 'the drift lines log at the finding level (`info`)', + logger + .lines() + .filter( + (l) => l.level === 'info' && l.message.includes('drift'), + ).length, + 10, + ); + + const spans: OtelSpan[] = []; + const e = stitch({ + ...cfg, + adapter: fakeVendor({ body: afterTheRelease() }), + trace: otlpSink({ + exporter: { export: (s) => void spans.push(...s) }, + }), + }); + await e(); + leakRow('otlpSink', bytesOf(spans), NEW_FIELD, ''); + check( + 'OTLP carries level/path/change but NOT the detail', + bytesOf(spans).includes('stitch.drift.path'), + true, + ); + check( + 'the exported spans contain the string "drift.detail"', + bytesOf(spans).includes('drift.detail'), + false, + ); + + const tally = printLeakTable(NEW_FIELD); + console.log( + `\n the new field's VALUE reaches ${tally.leaking} of ${tally.leaking + tally.clean} destinations. Its NAME reaches all of them.`, + ); + check("no destination carries the new field's value", tally.leaking, 0); + resetLeakTable(); + } + + heading('C7 (e) — the precondition: no `output`, no signal'); + { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: afterTheRelease() }), + }); + const w = await call.inspect(); + check( + 'with no `output` schema, findings on the SAME changed response', + w.findings.length, + 0, + ); + const bare = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: afterTheRelease() }), + output: SAFE, + }); + const w2 = await bare.inspect(); + check( + 'with a plain `output` schema but no `drift()` wrapper', + w2.findings.length, + 0, + ); + note( + '→ drift is schema-ANCHORED (ADR 0015): the diff is `raw` vs the VALIDATED value, so with nothing declared there is nothing to diff and the addition is invisible. The "vendor added a PII field" alarm is available only to a project that already wrote the allowlist — which means C7 does not stand alone, it is a property of C6', + ); + } + + heading('C7 (f) — the limit: `undeclared` cannot be made fatal'); + { + // `DriftSeverity` is 'warn' | 'info' | 'verbose' — 'error' is deliberately not in it, and + // the JSDoc says so: "Soft drift is always non-fatal; to fail on a change, make the field + // required/strict in the schema". Measured through a cast, so the runtime behaviour is on + // the record rather than inferred from the type. + const findings = await findingsFor(afterTheRelease(), { + severity: { undeclared: 'error' }, + } as unknown as DriftOptions); + check( + 'a re-level to `error` IS honoured at runtime …', + findings.filter((f) => f.path === 'taxId' && f.level === 'error') + .length, + 1, + ); + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: afterTheRelease() }), + output: drift(SAFE, { + severity: { undeclared: 'error' }, + } as unknown as DriftOptions), + }); + const out = await call.safe(); + check('… and it DOES fail the call', out.ok, false); + check( + 'with the contract-violation message', + out.error?.message, + 'contract violation (drift)', + ); + note( + '→ NOT IN THE CLAIMS, and it cuts against the documentation. `DriftSeverity` excludes `error` and the JSDoc states "soft drift is always non-fatal", but the runtime path (`resolveSeverity` → `levelOf` → `if (finding.level === \'error\') fatal = true`) has no guard: a severity map that names `error` is a TYPE error and a working kill-switch. Either the type should admit it as a documented "fail on any new field" hatch, or the runtime should reject it — right now it is a fail-closed behaviour reachable only by a cast', + ); + } + + heading( + 'C7 (g) — REFUTATION: a HARD finding CAN carry the value, into the payload-free sinks', + ); + { + // ADR 0018 §4: "`detailFor` emits kinds only, never values … so `findings` never leak a + // secret even when `redact` is off." That holds for the three SOFT kinds, which is all + // `detailFor` produces. Hard failures do not go through `detailFor`: `validationErrors` + // (drift.ts:50) copies the VALIDATOR's message into `detail`, and Zod's enum/union + // messages quote the received value. + const ENUMED = z.object({ + id: z.string(), + plan: z.enum(['enterprise', 'free']), + }); + const vendorSentBadPlan = { id: 'cus_7Q2', plan: SSN }; + const probe = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: vendorSentBadPlan }), + output: ENUMED, + }); + const w = await probe.inspect(); + const invalid = w.findings.find((f) => f.change === 'invalid'); + check('there is a hard `invalid` finding', invalid !== undefined, true); + check( + 'and its `detail` contains the RECEIVED VALUE', + (invalid?.detail ?? '').includes(SSN), + true, + ); + note('the finding detail', invalid?.detail); + + const t = tempFileSink(); + const a = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: vendorSentBadPlan }), + output: ENUMED, + trace: t.sink, + }); + await a.safe(); + const fileRow = leakRow( + 'fileSink — hard finding detail', + t.text(), + SENTINELS, + 'JSONL on disk', + ); + check('the JSONL carries the value', fileRow.hits.has('ssn'), true); + t.cleanup(); + + const cap = captureStderr(); + try { + const b = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: vendorSentBadPlan }), + output: ENUMED, + trace: consoleSink(), + }); + await b.safe(); + } finally { + cap.restore(); + } + const consoleRow = leakRow( + 'consoleSink — hard finding detail', + cap.text(), + SENTINELS, + 'stderr — the sink C1 measured at 0/7', + ); + check('consoleSink carries it too', consoleRow.hits.has('ssn'), true); + note( + 'the stderr line', + cap + .text() + .replace(/\x1b\[\d+m/g, '') + .split('\n') + .find((l) => l.includes('drift')), + ); + + const logger = captureLogger(); + const c = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: vendorSentBadPlan }), + output: ENUMED, + trace: loggerSink(logger), + }); + await c.safe(); + const loggerRow = leakRow( + 'loggerSink — hard finding detail', + logger.text(), + SENTINELS, + 'the messages handed to pino/winston', + ); + check('and loggerSink', loggerRow.hits.has('ssn'), true); + + const spans: OtelSpan[] = []; + const d = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor({ body: vendorSentBadPlan }), + output: ENUMED, + trace: otlpSink({ + exporter: { export: (s) => void spans.push(...s) }, + }), + }); + await d.safe(); + const otlpRow = leakRow( + 'otlpSink — hard finding detail', + bytesOf(spans), + SENTINELS, + 'OTLP exports level/path/change only', + ); + check( + 'OTLP alone stays clean — it drops `detail`', + otlpRow.hits.size, + 0, + ); + note( + "→ the REFUTATION, stated plainly: ADR 0018 §4 says `findings` never leak a value. It is true of the three SOFT kinds (`detailFor`, drift.ts:77) and false of the HARD kind (`validationErrors`, drift.ts:50, which copies `iss.message` verbatim). Whether a value escapes therefore depends on the SCHEMA LIBRARY's message wording, not on anything in this repo: Zod says \"Expected number, received string\" for a type error (safe) and \"Invalid enum value. Expected 'enterprise' | 'free', received '078-05-1120'\" for an enum (not safe). The two sinks documented as payload-free — console and logger — print it, because a finding is metadata by classification", + ); + } + + const hard = printLeakTable(SENTINELS); + console.log( + `\n hard-validation finding detail: ${hard.leaking} of ${hard.leaking + hard.clean} destinations carry the received VALUE.`, + ); + + finish( + 'C7', + "CONFIRMED for the soft signal, with two qualifications and one REFUTATION. The signal works and is precise: when the vendor adds `taxId` at the top level, one level down, and inside every array element, `drift()` emits exactly three NEW `undeclared` findings — `taxId`, `profile.taxId`, `contacts[].taxId` — at level `info`, and the finding contains NO value (0 of 1 new-field sentinel and 0 of 7 PII sentinels across the findings array, the raw drift events, the JSONL, stderr, the logger and OTLP). Qualification one: the signal is noisy at rest — the same schema produces 7 `undeclared` findings on the UNCHANGED response, so \"a new field appeared\" is a diff against a baseline, not an alert. Qualification two: it is schema-anchored, so with no `output` (or with `output` but no `drift()` wrapper) the same changed response yields 0 findings — C7 is a property of C6, not an independent safety net. A limit worth recording: `severity: { undeclared: 'error' }` is a TYPE error (`DriftSeverity` excludes `error`, and the JSDoc says soft drift is always non-fatal) but a WORKING kill-switch at runtime — through a cast it re-levels the finding and fails the call. And the REFUTATION: ADR 0018 §4 claims `findings` never leak a secret because `detailFor` emits kinds only. That holds for the three soft kinds and NOT for hard validation — `validationErrors` (drift.ts:50) copies the validator's own message into `detail`, and Zod's enum message quotes the received value (\"Invalid enum value. Expected 'enterprise' | 'free', received '078-05-1120'\"). Measured end to end: that value reaches the JSONL file, `consoleSink` and `loggerSink` — the two sinks C1 measured at 0 of 7 — while OTLP alone stays clean because it exports level/path/change and drops `detail`", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/c8-assembled.ts b/docs/scenarios/proofs/pii-in-the-logs/c8-assembled.ts new file mode 100644 index 00000000..634df8a1 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/c8-assembled.ts @@ -0,0 +1,465 @@ +// C8 — assemble the best available "no customer data reaches a log" setup, count the lines, name +// the seams, and state what it costs. +// +// Three variants are built and measured, because the measurements in C5 and C6 turn out to be in +// tension and the tension is the finding: +// +// A the ALLOWLIST alone — `output: drift(SAFE)`. Every destination that reads the validated +// value goes clean, and the "vendor added a field" signal survives. +// Two destinations do not: `.inspect().raw`, and `StitchError.body` +// on any non-2xx. +// B the BOUNDARY alone — `hooks.onResponse` mutating `res.body`. EVERY destination goes +// clean, including the two A leaves. And the drift signal goes to +// zero, because drift diffs the body against the schema and the +// boundary removed the body before the schema ever saw it. +// C BOTH, plus a hand-rolled key inventory — clean everywhere AND a names-only signal, at the +// cost of writing the walker `drift.ts` already contains. +// +// The line counts are read off THIS FILE at runtime, between the `>>> BEGIN USER CODE` markers, so +// they are the real number rather than an estimate. +// +// pnpm exec tsx docs/scenarios/proofs/pii-in-the-logs/c8-assembled.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + AdapterResponse, + DriftFinding, + StitchConfig, +} from '../../../../packages/core/src/types'; +import { + BASE, + SENTINELS, + bytesOf, + canary, + captureLogger, + captureStderr, + collectingSink, + fakeVendor, + recordingStore, + tempFileSink, +} from './canary'; +import { + check, + checkSeq, + finish, + heading, + leakRow, + note, + printLeakTable, + resetLeakTable, + scan, +} from './harness'; +import { z } from './zod'; + +import { readFileSync } from 'node:fs'; + +// =========================================================================== +// >>> BEGIN USER CODE — variant A: the allowlist +// =========================================================================== +const SAFE = z.object({ + id: z.string(), + plan: z.string(), + profile: z.object({ locale: z.string() }), + contacts: z.array(z.object({ label: z.string() })), +}); + +const variantA = { + output: drift(SAFE, { severity: ['info', 'warn'] }), +} satisfies Partial; +// =========================================================================== +// <<< END USER CODE — variant A +// =========================================================================== + +// =========================================================================== +// >>> BEGIN USER CODE — variant B: the boundary +// =========================================================================== +const variantB = { + hooks: { + onResponse: ({ res }: { res?: AdapterResponse }) => { + if (res) res.body = SAFE.safeParse(res.body).data ?? null; + }, + }, +} satisfies Partial; +// =========================================================================== +// <<< END USER CODE — variant B +// =========================================================================== + +// =========================================================================== +// >>> BEGIN USER CODE — variant C: the boundary that reports before it strips +// =========================================================================== +/** Every key path in `body` that the allowlist does not declare. Names only — never a value. */ +function undeclared(body: unknown, allow: unknown, at = ''): string[] { + if (Array.isArray(body)) + return Array.isArray(allow) && allow.length > 0 + ? [ + ...new Set( + body.flatMap((v) => undeclared(v, allow[0], `${at}[]`)), + ), + ] + : []; + if ( + !body || + typeof body !== 'object' || + !allow || + typeof allow !== 'object' + ) + return []; + const shape = allow as Record; + return Object.entries(body as Record).flatMap(([k, v]) => { + const path = at ? `${at}.${k}` : k; + if (!(k in shape)) return [path]; + return undeclared(v, shape[k], path); + }); +} + +/** The allowlist as plain data, so it can drive both the strip and the inventory. */ +const SHAPE = { + id: 1, + plan: 1, + profile: { locale: 1 }, + contacts: [{ label: 1 }], +}; + +const seen = new Set(); + +const variantC = { + output: drift(SAFE, { severity: ['info', 'warn'] }), + hooks: { + onResponse: ({ res }: { res?: AdapterResponse }) => { + if (!res) return; + for (const p of undeclared(res.body, SHAPE)) seen.add(p); + res.body = SAFE.safeParse(res.body).data ?? null; + }, + }, +} satisfies Partial; +// =========================================================================== +// <<< END USER CODE — variant C +// =========================================================================== + +/** + * Executable (non-blank, non-comment) lines between a BEGIN/END marker pair in this file. + * + * Counted at runtime off the file on disk, so the number in the verdict is the real one — and the + * one this repository's Prettier config produces, which is the honest unit for "what would this + * cost me in my codebase" rather than a hand-minified best case. + */ +function userLines(variant: string): number { + const src = readFileSync(process.argv[1] ?? '', 'utf8').split('\n'); + const start = src.findIndex((l) => + l.includes(`>>> BEGIN USER CODE — variant ${variant}`), + ); + const end = src.findIndex((l) => + l.includes(`<<< END USER CODE — variant ${variant}`), + ); + if (start < 0 || end < 0) return -1; + return src + .slice(start + 2, end - 1) + .map((l) => l.trim()) + .filter((l) => l !== '' && !l.startsWith('//') && !l.startsWith('*')) + .length; +} + +/** Run every C1 destination against one config and return the sentinel count per destination. */ +async function battery( + extra: Partial, + status = 200, +): Promise> { + const hits = new Map(); + const base = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + ...extra, + } as StitchConfig; + const mk = (over: Partial) => + stitch({ + ...base, + adapter: fakeVendor({ status }), + store: recordingStore(), + ...over, + } as StitchConfig); + + const t = tempFileSink(); + const store = recordingStore(); + await mk({ trace: t.sink, store }).safe(); + hits.set('fileSink (JSONL)', scan(t.text(), SENTINELS).size); + hits.set('cache entry', scan(store.text(), SENTINELS).size); + t.cleanup(); + + const spine = collectingSink(); + await mk({ trace: spine }).safe(); + hits.set('raw event spine', scan(spine.text(), SENTINELS).size); + + const cap = captureStderr(); + try { + await mk({ trace: 'console' }).safe(); + } finally { + cap.restore(); + } + hits.set('consoleSink', scan(cap.text(), SENTINELS).size); + + const logger = captureLogger(); + const { loggerSink } = await import('../../../../packages/core/src/trace'); + await mk({ trace: loggerSink(logger) }).safe(); + hits.set('loggerSink', scan(logger.text(), SENTINELS).size); + + const probe = mk({}); + const w = await probe.inspect(); + hits.set('.inspect().raw', scan(bytesOf(w.raw), SENTINELS).size); + hits.set('.inspect() wrapper', scan(bytesOf(w), SENTINELS).size); + hits.set('.report()', scan(bytesOf(await probe.report()), SENTINELS).size); + const out = await probe.safe(); + hits.set( + 'StitchError.body', + scan(bytesOf(out.error?.body), SENTINELS).size, + ); + hits.set( + 'JSON.stringify(error)', + scan(bytesOf(out.error ?? null), SENTINELS).size, + ); + return hits; +} + +async function findingsFor( + extra: Partial, +): Promise { + const call = stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: fakeVendor(), + ...extra, + } as StitchConfig); + return (await call.inspect()).findings; +} + +function printBattery(label: string, m: Map): number { + const w = Math.max(...[...m.keys()].map((k) => k.length)); + console.log(`\n ${label}`); + let leaking = 0; + for (const [k, v] of m) { + if (v > 0) leaking++; + console.log( + ` ${k.padEnd(w)} ${String(v).padStart(2)}${v > 0 ? ' LEAKS' : ''}`, + ); + } + return leaking; +} + +async function main(): Promise { + heading( + 'C8 (a) — the baseline: a well-instrumented stitch with no protection', + ); + const bare200 = await battery({}); + const bare500 = await battery({}, 500); + { + const leak200 = printBattery('nothing configured, 200', bare200); + const leak500 = printBattery('nothing configured, 500', bare500); + check('destinations measured per run', bare200.size, 10); + check('destinations leaking on a 200', leak200, 6); + check('destinations leaking on a 500', leak500, 4); + } + + heading('C8 (b) — variant A: the allowlist alone'); + { + const a200 = await battery(variantA); + const a500 = await battery(variantA, 500); + const leak200 = printBattery('output: drift(SAFE), 200', a200); + const leak500 = printBattery('output: drift(SAFE), 500', a500); + check('user-code lines', userLines('A'), 9); + check('destinations still leaking on a 200', leak200, 1); + check('and it is `.inspect().raw`', a200.get('.inspect().raw'), 7); + check('destinations still leaking on a 500', leak500, 4); + check( + 'the drift signal SURVIVES — undeclared findings', + (await findingsFor(variantA)).filter( + (f) => f.change === 'undeclared', + ).length, + 7, + ); + } + + heading('C8 (c) — variant B: the boundary alone'); + { + const b200 = await battery(variantB); + const b500 = await battery(variantB, 500); + const leak200 = printBattery('hooks.onResponse, 200', b200); + const leak500 = printBattery('hooks.onResponse, 500', b500); + check('user-code lines', userLines('B'), 7); + check('destinations leaking on a 200', leak200, 0); + check('destinations leaking on a 500', leak500, 0); + check( + 'but the drift signal is GONE — 0 findings', + (await findingsFor(variantB)).length, + 0, + ); + note( + '→ the tension, measured. Drift diffs the response body against the validated value; the boundary removed the body before the schema ever saw it, so there is nothing left to diff. You cannot have the earliest boundary AND the built-in "a new field appeared" signal, because the signal is computed from exactly the bytes the boundary removes', + ); + } + + heading( + 'C8 (d) — variant C: strip at the boundary, report the names first', + ); + { + seen.clear(); + const c200 = await battery(variantC); + const c500 = await battery(variantC, 500); + const leak200 = printBattery( + 'onResponse + output allowlist, 200', + c200, + ); + const leak500 = printBattery( + 'onResponse + output allowlist, 500', + c500, + ); + check('user-code lines', userLines('C'), 42); + check('destinations leaking on a 200', leak200, 0); + check('destinations leaking on a 500', leak500, 0); + checkSeq( + 'and the undeclared inventory the hook recorded, names only', + [...seen].sort(), + [ + 'contacts[].email', + 'email', + 'name', + 'note', + 'primaryContactMail', + 'profile.contact', + 'ssn', + ], + ); + check( + 'the inventory carries no value', + scan(bytesOf([...seen]), SENTINELS).size, + 0, + ); + check( + 'and it catches a NEW field the day it appears', + (() => { + seen.clear(); + return undeclared({ ...canary(), taxId: 'x' }, SHAPE).includes( + 'taxId', + ); + })(), + true, + ); + } + + heading('C8 (e) — the final table: variant C, every destination'); + { + resetLeakTable(); + const cfg = { + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + cache: { ttl: '60s' }, + ...variantC, + } as StitchConfig; + const t = tempFileSink(); + const store = recordingStore(); + await stitch({ + ...cfg, + adapter: fakeVendor(), + trace: t.sink, + store, + })(); + leakRow( + 'fileSink (JSONL)', + t.text(), + SENTINELS, + 'default 2048-char cap', + ); + leakRow('cache entry', store.text(), SENTINELS, ''); + t.cleanup(); + + const spine = collectingSink(); + await stitch({ + ...cfg, + adapter: fakeVendor(), + trace: spine, + store: recordingStore(), + })(); + leakRow( + 'raw event spine', + spine.text(), + SENTINELS, + 'a naive custom sink', + ); + + const cap = captureStderr(); + try { + await stitch({ + ...cfg, + adapter: fakeVendor(), + trace: 'console', + store: recordingStore(), + })(); + } finally { + cap.restore(); + } + leakRow('consoleSink', cap.text(), SENTINELS, ''); + + const probe = stitch({ + ...cfg, + adapter: fakeVendor(), + store: recordingStore(), + }); + const w = await probe.inspect(); + leakRow('.inspect().raw', bytesOf(w.raw), SENTINELS, ''); + leakRow('.inspect() wrapper', bytesOf(w), SENTINELS, ''); + leakRow('.report()', bytesOf(await probe.report()), SENTINELS, ''); + + const failing = stitch({ + ...cfg, + adapter: fakeVendor({ status: 500 }), + store: recordingStore(), + }); + const out = await failing.safe(); + leakRow('StitchError.body', bytesOf(out.error?.body), SENTINELS, ''); + leakRow( + 'JSON.stringify(error)', + bytesOf(out.error ?? null), + SENTINELS, + 'what a structured logger writes', + ); + const tally = printLeakTable(SENTINELS); + check('destinations still leaking under variant C', tally.leaking, 0); + check('destinations measured', tally.leaking + tally.clean, 9); + } + + heading('C8 (f) — what it does NOT buy you'); + { + // The one thing no seam can fix: the transport already read the bytes. Anything that + // logged inside the adapter, or a proxy in front of it, is upstream of every seam here. + const vendor = fakeVendor(); + let adapterSaw = ''; + const spy = (async (req) => { + const res = await vendor(req); + adapterSaw = bytesOf(res.body); + return res; + }) as typeof vendor; + await stitch({ + name: 'getCustomer', + baseUrl: BASE, + path: '/v1/customers/1', + adapter: spy, + ...variantC, + } as StitchConfig)(); + check( + 'a wrapper INSIDE the adapter still sees all 7', + scan(adapterSaw, SENTINELS).size, + 7, + ); + note( + "→ `hooks.onResponse` is the earliest seam the LIBRARY offers, not the earliest seam that exists. A custom adapter, an HTTP proxy, a service mesh access log, or the vendor's own logs are all upstream of it. This directory measures the library's boundary; the network's is elsewhere", + ); + } + + finish( + 'C8', + 'ASSEMBLED and measured. Baseline: a stitch with a trace sink, a cache and a `.report()` leaks the customer record at 6 of 10 destinations on a 200 and 4 of 10 on a 500. Variant A — the allowlist alone, `output: drift(SAFE, …)`, 9 executable lines, one seam (`output`) — takes that to 1 of 10 on a 200 (only `.inspect().raw`, by design) and leaves the 500 path untouched at 4 of 10. Variant B — the boundary alone, `hooks.onResponse` mutating `res.body`, 7 executable lines, one seam — takes BOTH to 0 of 10, including `StitchError.body` and `JSON.stringify(error)`, and costs the entire drift signal: 0 findings, because drift diffs the response against the schema and the boundary removed the response first. Variant C — both plus a 23-line hand-rolled key walker, 42 executable lines, two seams (`hooks.onResponse` + `output`) — is 0 of 9 destinations on both the success and the failure path AND recovers a names-only inventory of every undeclared field (7 paths, 0 sentinels), which is the "log the detection, not the data" shape the scenario\'s own sources recommend. The costs, stated: (1) 42 lines (as this repo\'s Prettier formats them), of which 23 re-implement a walker `drift.ts` already contains and does not export; (2) the allowlist must be written and maintained — the whole response shape, which is the thing that drifts; (3) `res.body = …` inside a `(ctx) => void` hook is nowhere documented as a privacy mechanism, so the correct construction is discoverable only by reading the engine; (4) `.inspect().raw` is deliberately unreachable by any of this on variant A and is only covered in B/C because the body was destroyed before capture — which also means `.inspect()` can no longer answer the question it exists for; and (5) none of it is upstream of the Adapter, which read the bytes first — measured, a spy inside the transport still sees all 7', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/pii-in-the-logs/canary.ts b/docs/scenarios/proofs/pii-in-the-logs/canary.ts new file mode 100644 index 00000000..f8d99782 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/canary.ts @@ -0,0 +1,295 @@ +// The canary payload, the fake vendor, and the capture rigs every claim in this directory shares. +// +// THE METHOD. The response body carries seven sentinels — literal strings that appear nowhere else +// in the process. Each destination (an event, a sink's file, a captured stderr buffer, a store +// write, an `.inspect()` wrapper) is reduced to BYTES and scanned for each sentinel. A row of the +// C1 table is therefore a measurement of what a destination actually holds, never a summary of what +// the source code appears to promise. +// +// The seven are chosen to break each of the field's four named workarounds in turn: +// +// nm a name — no denylist has ever contained "name" +// em an email — the one field every denylist DOES contain +// ssn an SSN — the one field every compliance doc names +// nst a NESTED mail — `profile.contact.mail`: a flat denylist misses it +// txt a FREE-TEXT mail — inside prose: a key-based redactor cannot see it at all +// arr an ARRAY-element mail — `contacts[1].email`: needs a walker, not a `delete` +// ren a RENAMED key — `primaryContactMail`: the "vendor added a field" case, today +// +// Everything here is offline: the transport is a fake in-memory `Adapter`, and the only file +// written is a JSONL trace under a `mkdtemp` directory that each script deletes. +import { memoryStore } from '../../../../packages/core/src/index'; +import { fileSink } from '../../../../packages/core/src/trace'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, + StitchEvent, + StitchStore, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; +import type { Sentinel } from './harness'; + +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +// ---- the sentinels --------------------------------------------------------- + +export const NAME = 'Wilhelmina Ashcombe'; +export const EMAIL = 'w.ashcombe@example.test'; +export const SSN = '078-05-1120'; +export const NESTED = 'nested-canary@example.test'; +export const FREETEXT = 'freetext-canary@example.test'; +export const ARRAY = 'array-canary@example.test'; +export const RENAMED = 'renamed-canary@example.test'; +/** + * An EIGHTH sentinel, used only by {@link bulkyCanary} and never by the main table: it sits past + * the trace sink's default 2048-character cap. Kept out of {@link SENTINELS} so its absence measures + * the cap and nothing else — sharing a literal with an early field would have made a truncation + * measurement read as a survival. + */ +export const LATE = 'late-canary@example.test'; + +export const SENTINELS: readonly Sentinel[] = [ + { code: 'nm', value: NAME, at: 'customer.name' }, + { code: 'em', value: EMAIL, at: 'customer.email' }, + { code: 'ssn', value: SSN, at: 'customer.ssn' }, + { code: 'nst', value: NESTED, at: 'profile.contact.mail' }, + { code: 'txt', value: FREETEXT, at: 'note (free text)' }, + { code: 'arr', value: ARRAY, at: 'contacts[1].email' }, + { code: 'ren', value: RENAMED, at: 'primaryContactMail' }, +]; + +// ---- the credential sentinels (C4) ----------------------------------------- +// Distinct from the PII set so one scan can tell a credential leak from a customer-data leak. None +// of these literals contains a secret-looking KEY name — the denylist matches keys, never values, +// so a value that happened to spell `token` would make the measurement meaningless. + +export const BEARER_TOKEN = 'canary-bearer-9RKQ4W'; +export const QUERY_KEY = 'canary-apikey-4WZT1M'; +export const COOKIE_VALUE = 'canary-cookie-1MPD7X'; + +export const CREDENTIALS: readonly Sentinel[] = [ + { code: 'bea', value: BEARER_TOKEN, at: 'Authorization: Bearer …' }, + { code: 'qry', value: QUERY_KEY, at: '?api_key= (url + query)' }, + { code: 'ckl', value: COOKIE_VALUE, at: 'Cookie: sid=…' }, +]; + +// ---- the payload ----------------------------------------------------------- + +/** The vendor's customer record. A fresh object each call — nothing is shared between runs. */ +export function canary(): Record { + return { + id: 'cus_7Q2', + plan: 'enterprise', + name: NAME, + email: EMAIL, + ssn: SSN, + primaryContactMail: RENAMED, + note: `Customer asked us to reach them at ${FREETEXT} instead of the address on file.`, + profile: { + locale: 'en-GB', + contact: { mail: NESTED, phone: '+44 20 7946 0958' }, + }, + contacts: [ + { label: 'work', email: 'desk@example.test' }, + { label: 'home', email: ARRAY }, + ], + }; +} + +/** + * The same record padded past the trace sink's default 2048-character body cap, with a sentinel + * DELIBERATELY placed after the padding. Truncation is not redaction, and this is what proves it: + * the sentinels before the cap survive into the `preview`, the one after it does not. + */ +export function bulkyCanary(): Record { + const base = canary(); + return { + ...base, + // ~2.4 KB of filler between the early sentinels and the last one. + history: Array.from({ length: 40 }, (_, i) => ({ + at: `2026-0${(i % 9) + 1}-01T00:00:00Z`, + event: 'plan.renewed', + actor: 'billing-service', + detail: `renewal #${i} processed against the enterprise plan`, + })), + // Placed last on purpose: past the default cap, so its absence measures the cap, not a policy. + lateContact: { mail: LATE }, + }; +} + +export const BASE = 'https://vendor.example.test'; + +// ---- the fake vendor ------------------------------------------------------- + +export interface FakeVendor extends Adapter { + /** Every request the transport saw, headers cloned at call time. */ + seen(): readonly AdapterRequest[]; + count(): number; +} + +/** An in-memory `Adapter` that answers with a fixed body/status. No network, no `fetch`. */ +export function fakeVendor( + opts: { + body?: unknown; + status?: number; + headers?: Record; + } = {}, +): FakeVendor { + const seen: AdapterRequest[] = []; + const fn = (async (req: AdapterRequest): Promise => { + seen.push({ ...req, headers: { ...req.headers } }); + return { + status: opts.status ?? 200, + headers: opts.headers ?? { 'content-type': 'application/json' }, + body: + opts.body === undefined ? canary() : structuredClone(opts.body), + url: req.url, + }; + }) as FakeVendor; + fn.seen = () => seen; + fn.count = () => seen.length; + return fn; +} + +// ---- capture rigs ---------------------------------------------------------- + +/** A `TraceSink` that keeps every RAW event — the event spine, exactly as the engine emitted it. */ +export function collectingSink(): TraceSink & { + events(): readonly StitchEvent[]; + types(): string[]; + of(type: string): StitchEvent | undefined; + /** `JSON.stringify` of the whole spine — what a naive custom sink would log. */ + text(): string; +} { + const events: StitchEvent[] = []; + return { + handle(event: StitchEvent, _ctx: TraceContext): void { + events.push(event); + }, + events: () => events, + types: () => events.map((e) => e.type), + of: (type: string) => events.find((e) => e.type === type), + text: () => JSON.stringify(events), + }; +} + +/** + * Swap `process.stderr.write` for a collector — `consoleSink` writes there, so this captures the + * exact bytes a terminal (and therefore a container log driver) would receive. + */ +export function captureStderr(): { text(): string; restore(): void } { + const chunks: string[] = []; + const proc = process as unknown as { + stderr: { write: (s: string) => boolean }; + }; + const original = proc.stderr.write.bind(proc.stderr); + proc.stderr.write = (s: string): boolean => { + chunks.push(String(s)); + return true; + }; + return { + text: () => chunks.join(''), + restore: () => { + proc.stderr.write = original; + }, + }; +} + +/** A `LoggerLike` that keeps every line — what `loggerSink` handed to pino/winston/console. */ +export function captureLogger(): { + error(m: string): void; + warn(m: string): void; + info(m: string): void; + debug(m: string): void; + lines(): readonly { level: string; message: string }[]; + text(): string; +} { + const lines: { level: string; message: string }[] = []; + const push = + (level: string) => + (message: string): void => { + lines.push({ level, message }); + }; + return { + error: push('error'), + warn: push('warn'), + info: push('info'), + debug: push('debug'), + lines: () => lines, + text: () => lines.map((l) => `${l.level} ${l.message}`).join('\n'), + }; +} + +/** + * A real `fileSink` writing into a fresh `mkdtemp` directory. `text()` reads the JSONL back — the + * bytes that landed on disk, not a reconstruction — and `cleanup()` removes the directory. + */ +export function tempFileSink(opts?: Parameters[1]): { + sink: TraceSink; + path: string; + text(): string; + cleanup(): void; +} { + const dir = mkdtempSync(join(tmpdir(), 'stitch-pii-')); + const path = join(dir, 'trace.jsonl'); + const sink = fileSink(path, opts); + return { + sink, + path, + text: () => { + try { + return readFileSync(path, 'utf8'); + } catch { + return ''; + } + }, + cleanup: () => rmSync(dir, { recursive: true, force: true }), + }; +} + +/** + * `memoryStore` with a tap on `set`. The default store holds values by reference, so "what is in + * the cache" is not otherwise inspectable; this records the value the cache engine handed down, + * which is the thing a Redis/file store would have serialised and persisted. + */ +export function recordingStore(): StitchStore & { + writes(): readonly { key: string; value: unknown }[]; + /** JSON of every recorded write — the bytes a serialising store would have persisted. */ + text(): string; +} { + const inner = memoryStore(); + const writes: { key: string; value: unknown }[] = []; + const store = { + ...inner, + async set(key: string, value: unknown, ttl?: number): Promise { + writes.push({ key, value }); + await inner.set(key, value, ttl); + }, + writes: () => writes, + text: () => { + try { + return JSON.stringify(writes); + } catch { + return String(writes); + } + }, + }; + return store as StitchStore & { + writes(): readonly { key: string; value: unknown }[]; + text(): string; + }; +} + +/** Serialize anything to bytes for the scanner, surviving cycles and non-JSON values. */ +export function bytesOf(value: unknown): string { + if (typeof value === 'string') return value; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} diff --git a/docs/scenarios/proofs/pii-in-the-logs/harness.ts b/docs/scenarios/proofs/pii-in-the-logs/harness.ts new file mode 100644 index 00000000..2a3da912 --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/harness.ts @@ -0,0 +1,196 @@ +// Assertion + measurement harness for the `pii-in-the-logs` proofs. Every check prints a line and +// the script exits non-zero if any check failed. No test framework — these are standalone `tsx` +// scripts, exactly like the other proof directories. +// +// What this directory needs that the others did not: a SENTINEL SCANNER. Every claim here reduces +// to one question — "did this exact string reach that destination?" — so the primitive is not a +// value comparison but a substring scan of a destination's SERIALIZED BYTES, per sentinel, with the +// byte count reported alongside. `leakRow` records one destination; `printLeakTable` prints the +// destination × sentinel matrix that C1 exists to produce. +// +// The scan is deliberately dumb (`String.includes`). A cleverer matcher would let the harness +// decide what counts as a leak; a substring scan of the bytes a sink actually wrote cannot. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides several rows. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v.toString()}n`; + if (typeof v === 'string') return JSON.stringify(v); + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** Assert an exact string match, printing the measured string. For error messages, mostly. */ +export function checkStr( + label: string, + actual: string, + expected: string, +): void { + checks++; + const ok = actual === expected; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${JSON.stringify(actual)}${ok ? '' : ` (expected ${JSON.stringify(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence prints in full whether it passes or fails — an event spine IS the evidence for several + * rows here. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +// ---- the sentinel scanner -------------------------------------------------- +// A sentinel is a literal string planted in the canary payload. A destination is anything that can +// be reduced to bytes: a JSONL line, a captured stderr write, a logger message, `JSON.stringify` of +// a wrapper, the bytes a store was handed. `scan` answers, for one destination, which sentinels are +// present in it. + +/** One sentinel: a short column code, the literal planted value, and where it sits in the body. */ +export interface Sentinel { + /** Fixed-width column code for the leak table (kept to 3 chars so the matrix fits 100 cols). */ + code: string; + /** The literal string planted in the canary payload — what `scan` looks for. */ + value: string; + /** Where it lives in the response body, for the legend. */ + at: string; +} + +/** Which of `sentinels` appear literally in `text`. */ +export function scan( + text: string, + sentinels: readonly Sentinel[], +): Set { + const hits = new Set(); + for (const s of sentinels) if (text.includes(s.value)) hits.add(s.code); + return hits; +} + +export interface LeakRow { + dest: string; + bytes: number; + hits: Set; + /** Printed under the table — how this destination's bytes were obtained. */ + how: string; +} + +const leakRows: LeakRow[] = []; + +/** + * Record one destination's serialized bytes against the sentinel set. Returns the row so a caller + * can assert on `hits.size` / `hits.has(code)` immediately. + * + * `text` must be the bytes the destination ACTUALLY holds or wrote — a file's contents, a captured + * stderr buffer, `JSON.stringify` of the object a consumer would log. Passing a hand-built summary + * would make the table a restatement of the author's belief instead of a measurement. + */ +export function leakRow( + dest: string, + text: string, + sentinels: readonly Sentinel[], + how = '', +): LeakRow { + const row: LeakRow = { + dest, + bytes: Buffer.byteLength(text, 'utf8'), + hits: scan(text, sentinels), + how, + }; + leakRows.push(row); + return row; +} + +/** Print the accumulated destination × sentinel matrix plus a per-sentinel tally. */ +export function printLeakTable(sentinels: readonly Sentinel[]): { + leaking: number; + clean: number; +} { + const w = Math.max(...leakRows.map((r) => r.dest.length), 11); + const codes = sentinels.map((s) => s.code); + const head = codes.map((c) => c.padStart(3)).join(' '); + const rule = codes.map(() => '---').join(' '); + console.log( + `\n ${'destination'.padEnd(w)} ${'bytes'.padStart(7)} ${head}\n` + + ` ${'-'.repeat(w)} ${'-'.repeat(7)} ${rule}`, + ); + for (const r of leakRows) { + const cells = codes + .map((c) => (r.hits.has(c) ? ' ●●' : ' ·').padStart(3)) + .join(' '); + console.log( + ` ${r.dest.padEnd(w)} ${String(r.bytes).padStart(7)} ${cells}`, + ); + } + console.log(`\n ●● = the literal sentinel is present in this destination's bytes + · = absent + + legend:`); + for (const s of sentinels) + console.log( + ` ${s.code.padStart(3)} ${s.at.padEnd(28)} ${JSON.stringify(s.value)}`, + ); + const leaking = leakRows.filter((r) => r.hits.size > 0).length; + return { leaking, clean: leakRows.length - leaking }; +} + +/** Reset the accumulated table (a script that builds more than one matrix). */ +export function resetLeakTable(): void { + leakRows.length = 0; +} + +/** The rows collected so far — for a script that wants to assert over the whole matrix. */ +export function rows(): readonly LeakRow[] { + return leakRows; +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a LEAK. The verdict statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/pii-in-the-logs/zod.ts b/docs/scenarios/proofs/pii-in-the-logs/zod.ts new file mode 100644 index 00000000..8e599b2a --- /dev/null +++ b/docs/scenarios/proofs/pii-in-the-logs/zod.ts @@ -0,0 +1,12 @@ +// Real Zod, imported by path — the same convenience `precision-loss/zod.ts` and +// `stale-fixture/zod.ts` document. +// +// C6 asks whether an `output` schema that strips unknown keys keeps them out of the log, and the +// answer depends entirely on what a REAL schema library does with an undeclared key. A hand-rolled +// `{ validate }` stub would let this directory invent the very behaviour under test — and the +// stripping is the behaviour under test. +// +// `packages/core` already depends on Zod v4 in devDependencies; pnpm does not hoist it to the +// workspace root and `docs/` has no manifest, so the import goes by relative path. It resolves +// under `tsx`. In application code the spelling is `import { z } from 'zod'`. +export { z } from '../../../../packages/core/node_modules/zod'; diff --git a/docs/scenarios/proofs/precision-loss/README.md b/docs/scenarios/proofs/precision-loss/README.md new file mode 100644 index 00000000..8e07102c --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/README.md @@ -0,0 +1,180 @@ +# Proofs — the ID that changed on the way in + +Runnable evidence for the claims in [`../../precision-loss.md`](../../precision-loss.md). + +**C1 is confirmed. C2 — the claim the scenario turns on — is half refuted, and the half that fell +is the capture's conclusion.** The corruption is real and completely silent: `1234567890123456789` +arrives as `1234567890123456768`, on a four-event spine with zero findings, and the sent digits +appear nowhere in it. But the capture's two structural conclusions do not survive measurement. +Validation _can_ separate a corrupted id from an intact one, and the `Adapter` is _not_ the only +seam that can see the raw bytes — `wire: { response: 'text' }` is published config that hands the +verbatim string to `transform`, on the stock `fetchAdapter`. + +Along the way the survey found a library bug that is not in the claims list at all: **a `bigint` in +`params` silently vanishes**, producing a request to the wrong URL with no error and no event +(C5(a)). + +Every script is standalone and offline. The transport under test is the library's **real** +`fetchAdapter`, fed a fake `fetch` that returns a real `Response` — so `http-adapter.ts:135` +(`parsed = text === '' ? undefined : JSON.parse(text)`) runs verbatim rather than being imitated. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/precision-loss/c2-detection.ts + +# all of them +for f in docs/scenarios/proofs/precision-loss/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/precision-loss/*.ts +``` + +### Why the fixtures are strings + +Every other proof directory in this repo starts from a fixture **object**. This one cannot. The +whole scenario is the gap between the bytes a vendor sent and the value JavaScript ended up holding, +and an object literal has already closed that gap the wrong way: `1234567890123456789` typed into a +`.ts` file **is** `1234567890123456768`, because the TypeScript source is parsed by the same lexer. +So the fixtures in [`wire.ts`](wire.ts) are string constants, and digits are only ever asserted +against strings. + +The same trap bit this directory twice while it was being written, and both are recorded in the +source rather than quietly fixed: + +- C7(e)'s first sampler built random ids with `Math.floor(rand() * 3e17)`. That product is itself a + double above 2^53, so every id it could produce was already exactly representable — the measured + false-positive rate came out **17× too high** (9.15% instead of 0.535%). The generator now + assembles ids entirely in `BigInt`. +- C3's first sentinel was a control character (`\u0000`), the collision-free choice — and + `JSON.parse` rejects a raw control character inside a string literal. The printable sentinel that + replaced it has one residual false positive, and C3(a2) measures it instead of hiding it. + +### Why this directory imports Zod by path + +Same reason as [`../stale-fixture/zod.ts`](../stale-fixture/zod.ts): C2 asks what `z.number().int()`, +`z.bigint()` and `z.string()` actually do when handed a corrupted double, and a hand-rolled +`{ validate }` stub would let this directory invent the answer. In application code the spelling is +`import { z } from 'zod'`. + +## What each script establishes + +| Script | Question | Measured | +| ---------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `c1-corruption.ts` | does the default path corrupt, silently? | **Yes, and totally silent.** 3 of 8 values corrupted, 4-event spine, 0 findings. Money does **not** corrupt on the wire | +| `c2-detection.ts` | can anything downstream detect it? | **9 seams blind, 2 see raw text.** `isSafeInteger` works; `wire.response:'text'` refutes "adapter is the only seam" | +| `c3-custom-adapter.ts` | can a custom adapter fix it, and cost? | **Yes, 84 lines.** `JSON.stringify` throws; trace sinks **survive** — `trace.ts` already ships `bigintSafe` | +| `c4-cache.ts` | does `cache` survive a BigInt body? | **`memoryStore` yes, JSON store throws.** With a replacer, `typeof` differs between a cache hit and a miss | +| `c5-request-side.ts` | does a large ID survive going out? | **4 positions, 3 outcomes.** query/form fine, JSON body throws, and **`params` silently drops a bigint** | +| `c6-streams.ts` | are the streaming surfaces safer? | **Split by decoder, not surface.** bytes/lines/download lossless; ndjson/json/sse corrupt | +| `c7-detector.ts` | is there a spelling that is loud? | **Yes — a `warn` drift finding, non-fatal.** 15 lines. 0 false negatives; 0.535% false positives on real snowflakes | +| `c8-assembled.ts` | the safest setup, and what it costs? | **Two setups: 16 lines to repair, 18 to detect.** 5 seams prevent, 1 reports, 6 cannot see it at all | + +## Files + +- `wire.ts` — the digits, as string constants (`SNOWFLAKE`, `TWO53_PLUS_1`, `BIGINT_PK`, `MAX_SAFE`, + `SMALL_ID`, `MONEY`), the wire payloads assembled from them by concatenation, and the transports. + `wireAdapter` is the library's **real** `fetchAdapter` with a fake `fetch` underneath, so the parse + under test is `http-adapter.ts:135` itself. Also `quoteBigInts` — the single-pass, string-aware + scanner C3 puts under test — and the two revivers built on it. +- `harness.ts` — `check` / `checkDigits` / `checkStr` / `checkSeq` / `note` / `heading` / `finish`, + plus the one thing this scenario needed that the others did not: `digits()`, which routes an + integral double through `BigInt` so the printed value is the exact integer the double **is**, + never the shortest round-tripping form. Three digit strings exist for one id and the harness has + to be able to tell them apart. Also the two table printers (`printWireTable`, `printSeamTable`). +- `zod.ts` — real Zod, imported by relative path. + +## The three digit strings + +Worth stating once, because several rows below only make sense with it. For the id +`1234567890123456789`: + +| What | Digits | +| ------------------------------------------------- | --------------------- | +| the vendor sent | `1234567890123456789` | +| the double actually holds (`BigInt(n)`) | `1234567890123456768` | +| `JSON.stringify(n)` / `String(n)` — shortest form | `1234567890123456800` | + +The third is what appears in a log, in a `TraceSink` record, and in the URL when you hand the id +back (C5). So an id read and re-sent goes out as the **third** of these — not the second, which is what you +would see if you inspected the value in a debugger. + +## What each downstream seam sees, for one corrupted ID + +Sent on the wire: `{"id":1234567890123456789}`. Measured in C2. + +| Seam | Holds | Observed | +| -------------------------------- | ------------ | -------------------------------------------------------------------- | +| `output: z.number().int()` | parsed body | accepts `1234567890123456768` — zero findings | +| `output: z.bigint()` | parsed body | rejects — and rejects the intact id too (100% false positive) | +| `output: z.string()` | parsed body | same | +| `output: .refine(isSafeInteger)` | parsed body | **rejects corrupted, accepts intact** — a working detector | +| `drift()` | parsed body | `warn\|coerced\|id\|number -> string`, value `"1234567890123456800"` | +| `.inspect().raw` | parsed body | an object; `.id` is the **number** `1234567890123456768` | +| `.report()` | parsed body | 9 enumerable keys, `findings: []`, `source: 'live'` | +| `hooks.onResponse` | parsed body | `ctx = {attempt,name,res}`; `ctx.res = {body,headers,status,url}` | +| `Surface.interpret` | parsed body | `res.body.id = 1234567890123456768` | +| `TraceSink` | parsed body | 4 events, no wire text, no symbol channels | +| **`wire:{response:'text'}`** | **raw text** | `ctx.res.body === '{"id":1234567890123456789}'` | +| **`transform` (under text)** | **raw text** | repaired to `"1234567890123456789"`, no custom adapter | + +`raw` means _pre-validation_, not _pre-parse_. That one word is the whole of C2. + +## What was refuted + +The capture made four structural predictions that measurement contradicted. + +1. **"Validation cannot help."** `z.number().refine(Number.isSafeInteger)` rejects the corrupted id + and accepts the intact one. Walking the boundary (2^53−1 / 2^53 / 2^53+1 / 2^53+2) shows + `lossless=false` never co-occurs with `flagged=false`: **false negatives are impossible**, because + any integer a double cannot represent is above 2^53 and so is its parse. The cost is false + positives on large-but-representable integers — 0.535% over 20 000 real-shaped snowflakes. +2. **"That makes the `Adapter` the only candidate seam."** `wire: { response: 'text' }` is published + `StitchConfig`, honoured by the stock `fetchAdapter` at line 123 — _before_ the JSON branch at + 133–135. The body handed to the engine is the unparsed string, and `transform` then runs on it. + C2(j) recovers the exact sent digits with no custom adapter. +3. **"Trace sinks break on a BigInt body."** They do not. `trace.ts` ships a `bigintSafe` replacer, + commented "Tracing must never break the call it observes". `fileSink` wrote + `{"id":"1234567890123456789n"}` — the one diagnostic surface in this whole scenario that ends up + holding the vendor's actual digits. The cache **key** encoder handles bigint deliberately too + (`cache.ts:42`). +4. **"Money has the same shape."** It does not, on the wire. `19.99` round-trips to the token + `"19.99"`, because the nearest double's shortest form _is_ `"19.99"`. The value is still inexact + (`19.98999999999999843681`, and `19.99 * 100 === 1998.9999999999998`) — so decimals fail in + **arithmetic**, integers above 2^53 fail in **transport**. Only the second is a wire-fidelity bug, + and only the second is detectable by comparing digits. + +## Found outside the claims list + +- **`bigint` in `params` silently vanishes** (C5(a)). `expandTemplateVar` (`util.ts:392`) branches on + `string | number | boolean`; a bigint matches none, falls to the object arm, and + `Object.entries()` is `[]` — so nothing is emitted. The measured URL is + `https://api.snowflake.test/v1/things/`, with **no error and no event**. A pipeline repaired per C3 + that hands its BigInt id back to a path parameter requests the collection instead of the item. The + sibling position, `query`, handles bigint correctly (`stringifyLeaf`, `util.ts:332`) — so the two + URL positions disagree with each other. +- **`z.coerce.number()` silently undoes the C3 repair**, back to `1234567890123456768` (C3(c)). +- **A cache hit and a cache miss can return different TYPES** (C4(d)): with a bigint-aware store + replacer, `typeof data.id` is `bigint` on a miss and `string` on a hit — invisible to any test + suite that starts cold. +- **`JSON.stringify(report)` throws** on a BigInt body (C3(f)), and `.report()` is documented as safe + to log. It is safe to _read_ and throws when logged. +- **`transform` and `wire.response` are independent at the type level** (C8(c)): `transform` is + `(body: unknown) => unknown`, so a parser written `(text: string)` does not typecheck in the slot + even though `wire.response: 'text'` guarantees a string at runtime. +- **`transform` is redacted out of `__config`** (it is a function), so "is this stitch repaired?" is + only half auditable — `wire` shows, `transform` does not. diff --git a/docs/scenarios/proofs/precision-loss/c1-corruption.ts b/docs/scenarios/proofs/precision-loss/c1-corruption.ts new file mode 100644 index 00000000..0eeb0a1a --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c1-corruption.ts @@ -0,0 +1,326 @@ +// C1 (DECIDING) — does the default path corrupt, and is it silent? +// +// The measurement is deliberately blunt: put a known digit string on the wire, run the library's +// real default transport over it, and print the digits that came out the other side. No schema, no +// drift, no options — the plainest `stitch()` anyone would write. +// +// The scenario claims two things and this script separates them, because they are not the same +// claim and they do not have to both be true: +// +// (1) the value CHANGES — measured as sent-digits vs received-digits, field by field. +// (2) NOTHING SAYS SO — measured by enumerating the entire event spine, every drift finding, +// and every field of `.report()`, and finding no mention of it. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c1-corruption.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + digits, + finish, + heading, + note, + printWireTable, + wireRow, +} from './harness'; +import { + BASE, + BIGINT_PK, + FIFTH, + MAX_SAFE, + MONEY, + SMALL_ID, + SNOWFLAKE, + TENTH, + TWO53_PLUS_1, + WIRE_TEXT, + fmt, + wireAdapter, +} from './wire'; + +/** Everything one run of the plainest possible stitch produced. */ +interface Run { + data: Record; + events: StitchEvent[]; + findings: string[]; +} + +/** The plainest stitch anyone would write: a base, a path, a transport. Nothing else. */ +async function plainCall(text: string): Promise { + const events: StitchEvent[] = []; + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + events.push(e); + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: wireAdapter(text), + trace: sink, + }); + const data = (await call()) as Record; + return { data, events, findings }; +} + +async function main(): Promise { + heading('C1 (a) — the wire text, verbatim'); + // Printed in full first, so every digit asserted below is visibly the digit that was sent. + note('the exact bytes the fake vendor returns', WIRE_TEXT); + note('WIRE_TEXT length in bytes', WIRE_TEXT.length); + + const run = await plainCall(WIRE_TEXT); + + heading('C1 (b) — sent digits vs received digits, field by field'); + { + wireRow('snowflake (Discord/X)', SNOWFLAKE, run.data['snowflake']); + wireRow('2^53 + 1', TWO53_PLUS_1, run.data['two53_plus_1']); + wireRow('bigint PK (int64 max)', BIGINT_PK, run.data['bigint_pk']); + wireRow('money 19.99', MONEY, run.data['money']); + wireRow('0.1', TENTH, run.data['tenth']); + wireRow('0.2', FIFTH, run.data['fifth']); + wireRow('CONTROL 2^53 - 1', MAX_SAFE, run.data['max_safe']); + wireRow('CONTROL small id', SMALL_ID, run.data['small_id']); + const tally = printWireTable(); + console.log(''); + + // The three that must break, asserted as EXACT digit strings — the received digits are + // hard-coded here so a change in V8's parse would fail this script rather than silently + // rewrite the evidence. + checkDigits( + 'snowflake received', + run.data['snowflake'], + '1234567890123456768', + ); + checkDigits( + '2^53+1 received', + run.data['two53_plus_1'], + '9007199254740992', + ); + checkDigits( + 'bigint PK received', + run.data['bigint_pk'], + '9223372036854775808', + ); + + // The two that must survive, asserted the same way. Without these the claim would be + // "JavaScript numbers are approximate", which is not the scenario. + checkDigits('CONTROL 2^53-1 received', run.data['max_safe'], MAX_SAFE); + checkDigits( + 'CONTROL small id received', + run.data['small_id'], + SMALL_ID, + ); + + check('values corrupted', tally.corrupted, 3); + check('values intact', tally.intact, 5); + + note( + 'the snowflake moved by exactly this many units', + String(BigInt(SNOWFLAKE) - BigInt(digits(run.data['snowflake']))), + ); + } + + heading( + 'C1 (b2) — the money rows came back "intact", and that is a REAL result, not a bug in the table', + ); + { + // The capture files "a decimal money amount" alongside the snowflakes, as the same failure. + // Measured, it is NOT the same failure, and the distinction decides whether a round-trip + // detector can ever see it. + // + // `19.99` is not exactly representable either — but the nearest double's SHORTEST + // round-tripping decimal form is the string `"19.99"`. So it goes out as `19.99` and comes + // back as `19.99`: the wire round-trip is LOSSLESS even though the value is inexact. A + // 64-bit integer has no such luck, because two distinct integers share one double and the + // shortest form of that double is a third string again. + check( + 'the received money value re-serialises to the same token that was sent', + JSON.stringify(run.data['money']), + MONEY, + ); + note( + 'yet the value is not 19.99 — at 20 decimal places it is', + (run.data['money'] as number).toFixed(20), + ); + check( + 'so `money * 100` is not an integer number of cents', + Number.isInteger((run.data['money'] as number) * 100), + false, + ); + note('19.99 * 100 =', String((run.data['money'] as number) * 100)); + note( + '0.1 + 0.2 as received', + (run.data['tenth'] as number) + (run.data['fifth'] as number), + ); + note( + '→ decimals fail in ARITHMETIC, integers above 2^53 fail in TRANSPORT. Only the second is a wire-fidelity bug, and only the second is in principle detectable by comparing digits', + '', + ); + } + + heading( + 'C1 (c) — the vendor also sent the SAME id as a string. It survived', + ); + { + // The `id_str` convention, measured. It is the one row in the table that is both large and + // intact, and it is intact because it never went through a number. + checkDigits( + 'snowflake_str', + run.data['snowflake_str'], + `"${SNOWFLAKE}"`, + ); + check( + 'and it does NOT equal the number field', + String(run.data['snowflake_str']) === digits(run.data['snowflake']), + false, + ); + note( + 'so the corruption is not "large integers are impossible in JS" — it is "the JSON number type is lossy". The string field crossed the same wire, the same adapter, the same engine', + '', + ); + } + + heading('C1 (d) — SILENCE: the complete event spine'); + { + const spine = run.events.map((e) => e.type); + checkSeq('every event type emitted, in order', spine, [ + 'start', + 'progress', + 'result', + 'done', + ]); + const phases = run.events + .filter((e) => e.type === 'progress') + .map((e) => (e as { phase?: string }).phase); + checkSeq('progress phases', phases, ['request']); + checkSeq('drift findings', run.findings, []); + check( + 'events of type "drift"', + run.events.filter((e) => e.type === 'drift').length, + 0, + ); + check( + 'events of type "error"', + run.events.filter((e) => e.type === 'error').length, + 0, + ); + // The engine's teaching channel — `info` events are how it says "you asked for upload + // progress on a transport that cannot do it". Nothing here reaches for it. + check( + 'events of type "info"', + run.events.filter((e) => e.type === 'info').length, + 0, + ); + } + + heading('C1 (e) — SILENCE: does any event carry the sent digits at all?'); + { + // A weaker question than "was there a warning": is the information even PRESENT anywhere on + // the spine, for a user who went looking? Run a payload with ONE field — no `snowflake_str` + // to confound the search — and grep every serialised event for the digits that were sent. + const solo = await plainCall(`{"id":${SNOWFLAKE}}`); + const dump = solo.events + .map((e) => { + try { + return JSON.stringify(e); + } catch { + return String(e); + } + }) + .join('\n'); + check( + 'the SENT digits 1234567890123456789 appear anywhere on the event spine', + dump.includes(SNOWFLAKE), + false, + ); + note( + 'the `result` event, serialised', + dump.split('\n').find((l) => l.includes('"result"')) ?? '', + ); + note( + 'so the spine is not merely quiet about the change — it does not contain the information needed to notice one', + '', + ); + } + + heading( + 'C1 (e2) — a third digit string: what the corrupted value RE-SERIALISES to', + ); + { + // Worth its own row because it is the shape a user will actually see in a log. There are + // THREE distinct digit strings in play, and only the first is the vendor's. + const got = run.data['snowflake']; + note('1. the digits the vendor sent', SNOWFLAKE); + note('2. the exact integer the double holds (via BigInt)', digits(got)); + note( + '3. what JSON.stringify prints for that double', + JSON.stringify(got), + ); + check( + 'how many DISTINCT digit strings for one id', + new Set([SNOWFLAKE, digits(got), JSON.stringify(got)]).size, + 3, + ); + note( + '→ echoing the id back to the vendor sends the THIRD of these, not the second. `JSON.stringify` picks the SHORTEST decimal that round-trips to the same double, which is neither the sent value nor the stored one', + '', + ); + } + + heading('C1 (f) — SILENCE: what `.report()` shows'); + { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: wireAdapter(WIRE_TEXT), + }); + const rep = await call.report(); + checkSeq('report.findings', rep.findings, []); + check('report.error', rep.error, null); + check('report.status', rep.status, 200); + check('report.attempts', rep.attempts, 1); + check('report.source', rep.source, 'live'); + check('report.cache', rep.cache, 'disabled'); + checkSeq( + 'every enumerable key on the report', + Object.keys(rep).sort(), + [ + 'attempts', + 'cache', + 'config', + 'data', + 'error', + 'findings', + 'source', + 'status', + 'timing', + ], + ); + checkDigits( + 'and report.raw — the pre-validation body — holds the CORRUPTED digits', + (rep.raw as Record)['snowflake'], + '1234567890123456768', + ); + note( + "`.report()` is the library's widest diagnostic surface: nine keys, zero findings, and its `raw` is the parsed object", + '', + ); + } + + finish( + 'C1', + 'CONFIRMED, AND SILENT — with one REFINEMENT to the capture. The library\'s real default transport (`fetchAdapter`, fed a fake `fetch` so `http-adapter.ts:135` runs verbatim) turned 1234567890123456789 into 1234567890123456768 (a drift of 21), 9007199254740993 into 9007199254740992, and 9223372036854775807 into 9223372036854775808. Both controls survived exactly — 9007199254740991 and 4242 — and the SAME snowflake sent as a string in the same body arrived byte-perfect, which localises the fault to the JSON number type rather than to JavaScript. THE REFINEMENT: the money rows did NOT corrupt on the wire. 19.99 round-trips to the token "19.99" because the nearest double\'s shortest form IS "19.99"; the value is still inexact (19.98999999999999843681, and 19.99*100 is not an integer) and 0.1+0.2 is still 0.30000000000000004, so decimals fail in ARITHMETIC, not in TRANSPORT. Only integers above 2^53 are a wire-fidelity bug. Silence is confirmed in the strong sense: the whole spine is start / progress(request) / result / done — FOUR events, zero drift, zero error, zero info — and the digits 1234567890123456789 appear nowhere in it, so nothing downstream is withholding a warning it could have given. `.report()` adds nine enumerable keys and no findings. And there are three distinct digit strings for one id: sent 1234567890123456789, stored 1234567890123456768, re-serialised 1234567890123456800', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c2-detection.ts b/docs/scenarios/proofs/precision-loss/c2-detection.ts new file mode 100644 index 00000000..19390adc --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c2-detection.ts @@ -0,0 +1,499 @@ +// C2 (DECIDING) — can ANYTHING downstream detect the corruption? +// +// The capture predicts every seam fails, "because `raw` is already the parsed body". This script +// walks each seam in turn and prints what it actually held. Two of the rows come out differently +// from the prediction, and the second one is the most important measurement in this directory: +// +// REFUTED (i) — `Number.isSafeInteger` inside an `output` schema IS a working detector. It has +// zero false negatives (a corrupted integer is necessarily > 2^53, so it is +// necessarily unsafe) and a bounded, characterisable false-positive set (a large +// integer that happened to be exactly representable). The capture's "a schema that +// says z.number().int() passes it" is true; "validation cannot help" is not. +// REFUTED (ii) — the Adapter is NOT the only seam that can see raw text. `wire: { response: +// 'text' }` is a published config option that makes `AdapterResponse.body` the +// UNPARSED STRING, on the stock `fetchAdapter`. `transform` then runs on text. +// +// Everything else the capture predicted is confirmed, and confirmed by measurement rather than by +// reading: `.inspect().raw`, `hooks.onResponse`, `Surface.interpret`, `drift()` and a `TraceSink` +// all hold the already-parsed body, and none of them can reach the bytes. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c2-detection.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import { httpSurface } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + AdapterResponse, + HookContext, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + checkStr, + digits, + finish, + heading, + note, + printSeamTable, + seamRow, +} from './harness'; +import { + BASE, + ONE_ID_TEXT, + ONE_SAFE_ID_TEXT, + SNOWFLAKE, + fmt, + wireAdapter, +} from './wire'; +import { z } from './zod'; + +/** Run one stitch over `text` with an arbitrary extra config, never throwing. */ +async function run( + text: string, + extra: Record = {}, +): Promise<{ + ok: boolean; + data: unknown; + message: string; + findings: string[]; + events: StitchEvent[]; +}> { + const findings: string[] = []; + const events: StitchEvent[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + events.push(e); + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: wireAdapter(text), + trace: sink, + ...extra, + } as never); + const r = await call.safe(); + return { + ok: r.ok, + data: r.data, + message: r.error?.message ?? '', + findings, + events, + }; +} + +async function main(): Promise { + heading("C2 (a) — `output: z.number().int()`, the capture's example"); + { + const corrupt = await run(ONE_ID_TEXT, { + output: z.object({ id: z.number().int() }), + }); + check('the call SUCCEEDED on a corrupted id', corrupt.ok, true); + checkSeq('drift findings', corrupt.findings, []); + checkDigits( + 'and the validated value it handed back', + (corrupt.data as { id: number }).id, + '1234567890123456768', + ); + seamRow( + 'output: z.number().int()', + 'SEES_PARSED', + 'accepted 1234567890123456768 as a valid int', + ); + note( + 'the corrupted value is a perfectly valid integer, so an integer schema has nothing to object to', + '', + ); + } + + heading( + 'C2 (b) — `z.bigint()` and `z.string()`: they fail, but on EVERYTHING', + ); + { + // The interesting number is not "does it reject the corrupted id" — it is "does it accept + // the intact one". A check that rejects both is a type mismatch, not a detector. + for (const [name, schema] of [ + ['z.bigint()', z.object({ id: z.bigint() })], + ['z.string()', z.object({ id: z.string() })], + ] as const) { + const bad = await run(ONE_ID_TEXT, { output: schema }); + const good = await run(ONE_SAFE_ID_TEXT, { output: schema }); + check(`${name} rejects the CORRUPTED id`, bad.ok, false); + check( + `${name} also rejects the INTACT id (false positive)`, + good.ok, + false, + ); + note(`${name} finding on the intact id`, good.findings[0] ?? ''); + seamRow( + `output: ${name}`, + 'SEES_PARSED', + 'rejects every JSON number — 100% false-positive rate', + ); + } + note( + '→ these do not detect corruption. They detect "the wire type is number", which is always true, so they are unusable as a guard rather than a partial one', + '', + ); + } + + heading( + 'C2 (c) — REFUTATION: `Number.isSafeInteger` in a schema DOES separate them', + ); + { + // The capture says "validation cannot help: the corrupted value is a perfectly valid number". + // The first half is wrong. Validation cannot recover the SENT value — nothing can — but it + // can reliably answer "did this field pass through the lossy zone", which is the question + // that turns a silent bug into a loud one. + const Safe = z.object({ + id: z.number().refine(Number.isSafeInteger, { + message: 'integer exceeds 2^53 — precision was lost in transit', + }), + }); + const bad = await run(ONE_ID_TEXT, { output: Safe }); + const good = await run(ONE_SAFE_ID_TEXT, { output: Safe }); + check('rejects the CORRUPTED id', bad.ok, false); + check('ACCEPTS the intact id — no false positive', good.ok, true); + checkSeq('the finding it produced', bad.findings, [ + 'error|invalid|id|integer exceeds 2^53 — precision was lost in transit', + ]); + seamRow( + 'output: .refine(isSafeInteger)', + 'SEES_PARSED', + 'REJECTS corrupted, ACCEPTS intact — a working detector', + ); + + // The exact boundary, since the claim is about reliability. Every integer a double cannot + // represent is above 2^53, so its parsed value is above 2^53 too: there is no corrupted + // value that `isSafeInteger` calls safe. FALSE NEGATIVES ARE IMPOSSIBLE, and this walks the + // boundary to show it. + const boundary = [ + ['2^53 - 1', '9007199254740991'], + ['2^53', '9007199254740992'], + ['2^53 + 1', '9007199254740993'], + ['2^53 + 2', '9007199254740994'], + ] as const; + const rows: string[] = []; + for (const [label, sent] of boundary) { + const r = await run(`{"id":${sent}}`, { + output: z.object({ id: z.number() }), + }); + const got = digits((r.data as { id: number }).id); + const lossless = got === sent; + const flagged = !Number.isSafeInteger( + (r.data as { id: number }).id, + ); + rows.push( + `${label}: sent ${sent} got ${got} lossless=${String(lossless)} flagged=${String(flagged)}`, + ); + } + for (const r of rows) note(r); + checkSeq('the boundary, walked', rows, [ + '2^53 - 1: sent 9007199254740991 got 9007199254740991 lossless=true flagged=false', + '2^53: sent 9007199254740992 got 9007199254740992 lossless=true flagged=true', + '2^53 + 1: sent 9007199254740993 got 9007199254740992 lossless=false flagged=true', + '2^53 + 2: sent 9007199254740994 got 9007199254740994 lossless=true flagged=true', + ]); + note( + '→ read the two right-hand columns. `lossless=false` NEVER co-occurs with `flagged=false`: no false negatives, ever. The reverse does occur — 2^53 and 2^53+2 round-tripped exactly and were still flagged — so the cost is FALSE POSITIVES on large-but-representable integers (the even ones, above the boundary)', + '', + ); + note( + 'which is the correct trade for an ID: you cannot tell 9007199254740992-because-that-is-what-they-sent from 9007199254740992-because-they-sent-...93. The value is untrustworthy either way', + '', + ); + } + + heading('C2 (d) — `drift()`: what is on the LEFT side of the diff?'); + { + // The capture asks the critical question directly: does drift compare against raw TEXT or + // against the parsed body? `engine.ts:1201` says `const rawBody = value` — the post- + // transform, pre-validation PARSED value. This measures it rather than reading it. + // + // The probe: a schema that COERCES number -> string. Drift reports coercions, and the + // finding's detail names the before/after kinds — but more usefully, the coerced VALUE lands + // in `data`. If drift's left side were the wire text, the coerced string would be the sent + // digits. If it is the parsed double, the coerced string is `String(double)`. + const Coerce = z.object({ id: z.coerce.string() }); + const r = await run(ONE_ID_TEXT, { output: drift(Coerce) }); + check('the call succeeded', r.ok, true); + checkStr( + 'the coerced id — sent digits, or String(double)?', + (r.data as { id: string }).id, + '1234567890123456800', + ); + check( + 'does the coerced string equal the sent digits?', + (r.data as { id: string }).id === SNOWFLAKE, + false, + ); + checkSeq('the drift finding', r.findings, [ + 'warn|coerced|id|number -> string', + ]); + note( + '→ `number -> string` is drift SEEING a number on its left side. If the left side were the wire text, the kind would have been `string -> string` and the value would have been 1234567890123456789. It is neither', + '', + ); + seamRow( + 'drift()', + 'SEES_PARSED', + 'diff(parsed, validated) — coerced to "1234567890123456800"', + ); + + // And the plain case: with a matching schema there is nothing to diff, so drift is silent. + const plain = await run(ONE_ID_TEXT, { + output: drift(z.object({ id: z.number() })), + }); + checkSeq( + 'drift() findings on a plain matching schema', + plain.findings, + [], + ); + note( + "the library's flagship feature, pointed at its most basic failure — a value that is not the value the vendor sent — reports nothing, because both sides of its diff are downstream of the loss", + '', + ); + } + + heading('C2 (e) — `.inspect().raw`'); + { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: wireAdapter(ONE_ID_TEXT), + }); + const ins = await call.inspect(); + check('typeof inspection.raw', typeof ins.raw, 'object'); + check('inspection.source', ins.source, 'live'); + checkDigits( + 'inspection.raw.id', + (ins.raw as { id: unknown }).id, + '1234567890123456768', + ); + check( + 'typeof inspection.raw.id', + typeof (ins.raw as { id: unknown }).id, + 'number', + ); + seamRow( + '.inspect().raw', + 'SEES_PARSED', + 'an object whose .id is the number 1234567890123456768', + ); + note( + '`raw` means "pre-VALIDATION", not "pre-parse". It is the left side of drift\'s diff, and it is the same object `output` sees', + '', + ); + } + + heading('C2 (f) — `hooks.onResponse`: what exactly is `ctx.res.body`?'); + { + let ctxKeys: string[] = []; + let resKeys: string[] = []; + let bodyType = ''; + let idDigits = ''; + let textReachable = false; + await run(ONE_ID_TEXT, { + hooks: { + onResponse: (ctx: HookContext) => { + ctxKeys = Object.keys(ctx).sort(); + const res = (ctx as { res: AdapterResponse }).res; + resKeys = Object.keys(res).sort(); + bodyType = typeof res.body; + idDigits = digits((res.body as { id: unknown }).id); + // The whole question: is the wire text reachable from here by ANY route? + const dump = JSON.stringify({ ctxKeys, resKeys }); + textReachable = + dump.includes(SNOWFLAKE) || + resKeys.some((k) => + /text|raw|bytes|body_?text|source/i.test(k), + ); + }, + }, + }); + checkSeq('Object.keys(ctx)', ctxKeys, ['attempt', 'name', 'res']); + checkSeq('Object.keys(ctx.res)', resKeys, [ + 'body', + 'headers', + 'status', + 'url', + ]); + checkStr('typeof ctx.res.body', bodyType, 'object'); + checkStr('ctx.res.body.id', idDigits, '1234567890123456768'); + check('any key on ctx.res that could hold text', textReachable, false); + seamRow( + 'hooks.onResponse', + 'SEES_PARSED', + 'ctx.res = {body,headers,status,url}; body.id = 1234567890123456768', + ); + note( + 'four keys. The `AdapterResponse` the hook receives is the SAME object `fetchAdapter` returned, and `fetchAdapter` discarded `text` at line 135', + '', + ); + } + + heading('C2 (g) — `Surface.interpret`'); + { + // The surface hook runs on the AdapterResponse directly — the earliest engine-level seam + // there is. It is still after the adapter. + let seenType = ''; + let seenId = ''; + const spySurface: Surface = { + ...httpSurface, + id: 'http', + interpret: (res: AdapterResponse) => { + seenType = typeof res.body; + seenId = digits((res.body as { id: unknown }).id); + return { ok: true, data: res.body }; + }, + }; + const r = await run(ONE_ID_TEXT, { kind: spySurface }); + check('the surface ran', r.ok, true); + checkStr('typeof res.body inside interpret', seenType, 'object'); + checkStr('res.body.id inside interpret', seenId, '1234567890123456768'); + seamRow( + 'Surface.interpret', + 'SEES_PARSED', + 'res.body.id = 1234567890123456768', + ); + } + + heading('C2 (h) — a `TraceSink`'); + { + const r = await run(ONE_ID_TEXT); + const dump = r.events + .map((e) => { + try { + return JSON.stringify(e); + } catch { + return String(e); + } + }) + .join('\n'); + check( + 'sent digits anywhere in the trace', + dump.includes(SNOWFLAKE), + false, + ); + // Symbols are the library's channel for out-of-band payloads (RAW_BODY). Check whether a + // sink could reach one. + const syms = r.events.flatMap((e) => Object.getOwnPropertySymbols(e)); + checkSeq( + 'own symbols on any traced event', + syms.map((s) => s.toString()), + [], + ); + seamRow( + 'TraceSink', + 'SEES_PARSED', + 'four events, no wire text, no symbol channels', + ); + } + + heading( + 'C2 (i) — REFUTATION: `wire: { response: "text" }` DOES deliver the raw bytes', + ); + { + // The capture concludes "that makes the `Adapter` the only candidate seam". It is not. + // `wire.response` is a published `StitchConfig` option that sets `AdapterRequest.responseType`, + // and `fetchAdapter` honours `'text'` at line 123-124 — BEFORE the json branch at 133-135. + // The body handed to the engine is then the unparsed string, on the STOCK transport. + let hookBodyType = ''; + let hookBody = ''; + const r = await run(ONE_ID_TEXT, { + wire: { response: 'text' }, + hooks: { + onResponse: (ctx: HookContext) => { + const res = (ctx as { res: AdapterResponse }).res; + hookBodyType = typeof res.body; + hookBody = String(res.body); + }, + }, + }); + check('the call succeeded', r.ok, true); + checkStr('typeof ctx.res.body', hookBodyType, 'string'); + checkStr( + 'ctx.res.body — the VERBATIM WIRE TEXT', + hookBody, + ONE_ID_TEXT, + ); + checkStr( + 'and the resolved data is that same string', + String(r.data), + ONE_ID_TEXT, + ); + check( + 'the SENT digits are present and intact in user space', + hookBody.includes(SNOWFLAKE), + true, + ); + seamRow( + 'wire:{response:"text"}', + 'SEES_TEXT', + `ctx.res.body === ${JSON.stringify(ONE_ID_TEXT)}`, + ); + note( + '→ this is a stock `stitch()` with one extra config key and NO custom adapter. The capture\'s "the Adapter is the only candidate seam" is REFUTED', + '', + ); + } + + heading( + 'C2 (j) — …and `transform` runs on that text, so the repair is in config', + ); + { + // Chaining the refutation: with `wire.response: 'text'`, `transform` is a pre-parse seam. + // It receives the bytes and returns whatever it likes — including a body with the id kept + // as a string. No adapter written, no dependency added. + const r = await run(ONE_ID_TEXT, { + wire: { response: 'text' }, + transform: (body: unknown) => + JSON.parse( + String(body).replace( + /:\s*(-?\d{16,})/g, + (_m: string, d: string) => `:"${d}"`, + ), + ) as unknown, + output: z.object({ id: z.string() }), + }); + check('the call succeeded', r.ok, true); + checkStr( + 'and `data.id` is the SENT digits, exactly', + (r.data as { id: string }).id, + SNOWFLAKE, + ); + seamRow( + 'transform (under text)', + 'SEES_TEXT', + `repaired to "${SNOWFLAKE}" with no custom adapter`, + ); + note( + 'the regex here is the capture\'s own "JSON parser written in regex", and it carries that criticism honestly — C3 does it properly with a scanner. The measured point is only WHERE it can run: `transform`, in config, not in a transport', + '', + ); + } + + heading('C2 — the seam table'); + { + const tally = printSeamTable(); + console.log(''); + check('seams that see the raw text', tally.text, 2); + check('seams that see only the parsed body', tally.parsed, 9); + note( + 'and BOTH text-seeing rows are the same seam pair — `wire.response: "text"` moves the parse into user space, and `transform` is where it lands', + '', + ); + } + + finish( + 'C2', + 'PARTIALLY REFUTED — the capture is right about the seams it named and wrong about its conclusion. CONFIRMED: `output: z.number().int()` accepts 1234567890123456768 with zero findings; `.inspect().raw` is an OBJECT whose `.id` is the number 1234567890123456768 (`raw` means pre-VALIDATION, not pre-parse); `hooks.onResponse` gets `ctx = {attempt,name,res}` and `ctx.res = {body,headers,status,url}` with `body.id = 1234567890123456768` and no key that could hold text; `Surface.interpret` sees the same; a `TraceSink` sees four events, no wire text and no symbol channels. And drift\'s left side is measured, not assumed: under `z.coerce.string()` the finding is `warn|coerced|id|number -> string` and the coerced value is "1234567890123456800" — a NUMBER on the left, not the sent digits — so `diff(raw, validated)` compares parsed-to-validated, and on a matching schema `drift()` reports nothing at all. TWO REFUTATIONS. (i) `z.number().refine(Number.isSafeInteger)` IS a working detector: it rejects the corrupted id and ACCEPTS the intact one, and walking 2^53-1 / 2^53 / 2^53+1 / 2^53+2 shows `lossless=false` never co-occurs with `flagged=false` — false negatives are impossible; the cost is false positives on large-but-representable integers. (ii) The Adapter is NOT the only seam that can see bytes: `wire: { response: "text" }` is published config that makes `ctx.res.body` the verbatim string {"id":1234567890123456789} on the STOCK `fetchAdapter`, and `transform` then runs pre-parse — a config-only repair that recovered the exact sent digits', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c3-custom-adapter.ts b/docs/scenarios/proofs/precision-loss/c3-custom-adapter.ts new file mode 100644 index 00000000..96466072 --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c3-custom-adapter.ts @@ -0,0 +1,449 @@ +// C3 — can a custom `Adapter` fix it, and what does that cost? +// +// The repair is real and it is short: a single-pass scanner (`wire.ts:quoteBigInts`) that quotes +// out-of-string integer literals above 2^53, plus a `JSON.parse` reviver that turns them back into +// `BigInt`. This script proves the parse is correct — including the case the capture says a regex +// gets wrong, a number-shaped substring INSIDE a string — and then walks every downstream seam +// asking "does this still work now that the body carries a BigInt". +// +// The capture predicts a "loud cascade". Measured, the cascade is real but SHORTER than predicted: +// the library already carries bigint handling in two of the places the capture expected to break +// (trace serialisation and the cache key encoder), both deliberately, both commented as such. +// What genuinely breaks is user-facing and unavoidable: `JSON.stringify` and every `z.number()`. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c3-custom-adapter.ts +import { + consoleSink, + fileSink, + stitch, +} from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + checkStr, + finish, + heading, + note, +} from './harness'; +import { + BASE, + NESTED_TEXT, + ONE_ID_TEXT, + SENTINEL, + SNOWFLAKE, + bigintAdapter, + fmt, + parseBigIntsAsStrings, + parseWithBigInt, + quoteBigInts, +} from './wire'; +import { z } from './zod'; + +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** Try something and report the exact thrown message, or `''` when it did not throw. */ +function threw(fn: () => unknown): string { + try { + fn(); + return ''; + } catch (e) { + return e instanceof Error ? e.message : String(e); + } +} + +async function main(): Promise { + heading('C3 (a) — is the scanner actually correct?'); + { + // The capture's objection to the regex approach: "Breaks on numbers inside strings." So + // that is the first thing measured. `quoteBigInts` tracks string state and escapes. + const tricky = + '{"note":"order 1234567890123456789 shipped","id":1234567890123456789,' + + '"esc":"a \\" 9007199254740993 b","small":42,"neg":-9223372036854775807,' + + '"float":1234567890123456789.5,"exp":1.2e30}'; + note('a payload designed to break a regex', tricky); + const parsed = parseWithBigInt(tricky) as Record; + checkStr( + 'a big-integer-looking substring INSIDE a string is untouched', + String(parsed['note']), + 'order 1234567890123456789 shipped', + ); + checkStr( + 'and inside a string containing an escaped quote', + String(parsed['esc']), + 'a " 9007199254740993 b', + ); + checkDigits( + 'the real id became a BigInt with the sent digits', + parsed['id'], + `${SNOWFLAKE}n`, + ); + checkDigits( + 'a negative int64 too', + parsed['neg'], + '-9223372036854775807n', + ); + check( + 'a small integer stays a number', + typeof parsed['small'], + 'number', + ); + check( + 'a float stays a number (not an integer token — left alone)', + typeof parsed['float'], + 'number', + ); + check( + 'an exponent form stays a number', + typeof parsed['exp'], + 'number', + ); + // The size of the repair is a claim, so it is counted rather than asserted: every + // non-blank, non-comment line from `quoteBigInts` to the end of `parseBigIntsAsStrings`. + const src = readFileSync(join(__dirname, 'wire.ts'), 'utf8').split( + '\n', + ); + const from = src.findIndex((l) => + l.includes('export function quoteBigInts'), + ); + const to = src.findIndex((l) => + l.includes('export function parseBigIntsAsStrings'), + ); + const end = src.findIndex((l, i) => i > to && l === '}'); + const executable = src + .slice(from, end + 1) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; + note( + 'executable lines in the repair (quoteBigInts + both revivers), counted', + executable, + ); + check('the counted size of the repair', executable, 84); + note( + 'that is the price of the repair, and it is a real parser, not a regex', + '', + ); + note( + 'what it looks like on the way through', + quoteBigInts(ONE_ID_TEXT), + ); + } + + heading('C3 (a2) — the repair has its OWN false positive, and here it is'); + { + // Writing this scanner turned up a constraint that is not in the capture and is not + // obvious: the natural sentinel is a control character, because a control character cannot + // appear unescaped in a vendor string — but `JSON.parse` REJECTS a raw control character + // inside a string literal. This directory hit that error verbatim before settling on a + // printable sentinel, so the collision below is a real residual cost of the repair. + const collide = `{"label":"${SENTINEL}999","id":${SNOWFLAKE}}`; + const parsed = parseWithBigInt(collide) as Record; + check( + 'a vendor string that happens to start with the sentinel becomes a BigInt', + typeof parsed['label'], + 'bigint', + ); + checkDigits('…this one', parsed['label'], '999n'); + note('the offending payload', collide); + note( + '→ a hand-rolled bigint parser is not free of silent misreads either; it just moves which payload triggers one, from "any id above 2^53" to "a string literally beginning ~bigint~". A real dependency (json-bigint) parses rather than pre-quotes and has no such case', + '', + ); + } + + heading('C3 (b) — the repaired adapter, in a stitch'); + { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + output: z.object({ id: z.bigint() }), + }); + const data = (await call()) as { id: bigint }; + checkDigits('data.id', data.id, `${SNOWFLAKE}n`); + check('typeof data.id', typeof data.id, 'bigint'); + check( + 'and it equals the sent digits exactly', + data.id === BigInt(SNOWFLAKE), + true, + ); + note('the seam works. Everything below is the bill', ''); + } + + heading('C3 (c) — COST: `output` validators'); + { + // The most common schema in the world, applied to a repaired body. + const numeric = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + output: z.object({ id: z.number() }), + }); + const r = await numeric.safe(); + check('a `z.number()` schema now FAILS', r.ok, false); + note('the message', r.error?.message ?? ''); + note( + '→ every schema in the codebase that said `z.number()` for an ID has to become `z.bigint()`, one at a time, and a missed one is a hard failure rather than a silent one. That is an improvement, and it is still a migration', + '', + ); + + // z.coerce.number() "works" and is the trap: it re-introduces the exact loss. + const coerced = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + output: z.object({ id: z.coerce.number() }), + }); + const c = await coerced.safe(); + check('but `z.coerce.number()` accepts it', c.ok, true); + checkDigits( + 'and hands back — the corrupted value again', + (c.data as { id: number }).id, + '1234567890123456768', + ); + note( + '→ the repair is undone by one `.coerce`. The BigInt has to survive all the way to the consumer or it bought nothing', + '', + ); + } + + heading('C3 (d) — COST: `JSON.stringify` on the result'); + { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + }); + const data = await call(); + const msg = threw(() => JSON.stringify(data)); + checkStr( + 'the exact error', + msg, + 'Do not know how to serialize a BigInt', + ); + note( + '→ this is the cascade the capture names, and it is real: every log line, every response echo, every `res.json(data)` in an Express handler', + '', + ); + // The standard workaround, measured so the cost is concrete rather than gestured at. + const withReplacer = JSON.stringify(data, (_k, v: unknown) => + typeof v === 'bigint' ? v.toString() : v, + ); + checkStr( + 'with a replacer it serialises, to the SENT digits', + withReplacer, + `{"id":"${SNOWFLAKE}"}`, + ); + note( + 'note the quotes: the id is now a JSON string on the way out. Correct, and a wire-format change your own consumers see', + '', + ); + } + + heading( + 'C3 (e) — REFUTATION: trace sinks do NOT break. The library already handles BigInt', + ); + { + // The capture predicts trace sinks among the casualties. They are not: `trace.ts:160` is a + // `bigintSafe` JSON replacer, with a comment saying tracing must never break the call it + // observes. The replacer sits on the JSONL writer, so `fileSink` is the path that actually + // exercises it — written to a temp file, read back, and asserted on. + const jsonl = join( + mkdtempSync(join(tmpdir(), 'stitch-precision-')), + 'trace.jsonl', + ); + let sinkError = ''; + try { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + trace: fileSink(jsonl, { body: { chars: false } }), + }); + await call(); + } catch (e) { + sinkError = e instanceof Error ? e.message : String(e); + } + checkStr('the real `fileSink` threw', sinkError, ''); + const written = readFileSync(jsonl, 'utf8'); + check('and it wrote records', written.length > 0, true); + check( + 'the JSONL carries the SENT digits, `n`-suffixed', + written.includes(`${SNOWFLAKE}n`), + true, + ); + note( + 'the result record it wrote', + written + .split('\n') + .find((l) => l.includes('"result"')) + ?.slice(0, 220) ?? '', + ); + // And the human console sink, which renders to stderr rather than JSON — also clean. + let consoleError = ''; + try { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + trace: consoleSink(), + }); + await call(); + } catch (e) { + consoleError = e instanceof Error ? e.message : String(e); + } + checkStr('the real `consoleSink` threw', consoleError, ''); + note( + '→ REFUTED. `trace.ts` ships `bigintSafe`, a replacer that renders a bigint as `"n"` on the JSONL writer, commented "Tracing must never break the call it observes". A repaired body traces fine, and the trace shows the RIGHT digits — the only place in this whole scenario where a StitchAPI diagnostic surface holds the vendor\'s actual value', + '', + ); + } + + heading('C3 (f) — COST: `.inspect()` and `.report()`'); + { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + }); + const ins = await call.inspect(); + checkDigits( + 'inspection.raw.id survives as a BigInt', + (ins.raw as { id: unknown }).id, + `${SNOWFLAKE}n`, + ); + checkSeq('inspection.findings', ins.findings, []); + const rep = await call.report(); + check('`.report()` completed', rep.status, 200); + // The object works; SERIALISING it is what fails, and a report is meant to be pasted into + // a support ticket, so this matters. + checkStr( + 'JSON.stringify(report) — the exact error', + threw(() => JSON.stringify(rep)), + 'Do not know how to serialize a BigInt', + ); + note( + '→ `.report()` is documented as "safe to log". With a BigInt body it is safe to READ and throws when logged. The `raw` field is non-enumerable so it is not the culprit — `data` is', + '', + ); + } + + heading('C3 (g) — `__config` JSON round-trip is UNAFFECTED'); + { + // Worth measuring rather than assuming: `__config` describes the stitch, not the response, + // so no response value can reach it. The adapter is a function and is redacted out. + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + output: z.object({ id: z.bigint() }), + }); + await call(); + const json = threw(() => JSON.stringify(call.__config)); + checkStr('JSON.stringify(__config) threw', json, ''); + checkSeq('__config keys', Object.keys(call.__config).sort(), [ + 'baseUrl', + 'kind', + 'name', + 'output', + 'path', + ]); + note( + 'the config round-trips because a response body never enters it. Confirmed, not assumed', + '', + ); + } + + heading('C3 (h) — the other fork: parse big integers as STRINGS'); + { + // Same scanner, different reviver. This is the capture's "keep everything as strings", and + // measured against the BigInt fork it costs strictly less downstream — at the price of + // giving up arithmetic, which for an ID is not a price. + const parsed = parseBigIntsAsStrings(NESTED_TEXT) as { + items: { id: unknown }[]; + cursor: unknown; + }; + checkStr( + 'the big id came back as a string with the sent digits', + String(parsed.items[0]?.id), + SNOWFLAKE, + ); + check( + 'a small id stays a number', + typeof parsed.items[1]?.id, + 'number', + ); + checkStr( + 'and the whole body JSON.stringifies without a replacer', + threw(() => JSON.stringify(parsed)), + '', + ); + note('JSON.stringify of the string-fork body', JSON.stringify(parsed)); + note( + '→ the string fork keeps `JSON.stringify`, keeps `z.string()`, keeps structured cloning, keeps every cache store. It breaks only arithmetic and `===` against a number — which is why vendors that care ship `id_str`', + '', + ); + note( + 'one asymmetry worth naming: the string fork makes the TYPE depend on the VALUE — the same field is a string above 2^53 and a number below it, so a schema has to be `z.union([z.string(), z.number()])` unless the scanner quotes by KEY instead of by magnitude', + '', + ); + } + + heading('C3 (i) — does the engine care that the adapter is custom?'); + { + // A quick control: nothing else in the pipeline is disturbed by the swap. Same events, + // same shape, one attempt. + const events: StitchEvent[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + events.push(e); + }, + }; + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: bigintAdapter(ONE_ID_TEXT), + trace: sink, + }); + await call(); + checkSeq( + "event spine, identical to C1's", + events.map((e) => e.type), + ['start', 'progress', 'result', 'done'], + ); + checkSeq( + 'findings', + events + .filter((e) => e.type === 'drift') + .map((e) => fmt(e as never)), + [], + ); + } + + finish( + 'C3', + 'CONFIRMED that a custom Adapter fixes it; the COST is real but SHORTER than the capture predicted. The repair is a single-pass scanner plus a reviver — counted, 84 executable lines — and it is correct where a regex is not: a big-integer-looking substring inside a string ("order 1234567890123456789 shipped") and inside an escaped-quote string are both left untouched, floats and exponent forms pass through, and the id arrives as 1234567890123456789n. It has its OWN silent misread, measured: a vendor string beginning "~bigint~" is turned into a BigInt, because the collision-free sentinel is a control character and `JSON.parse` rejects a raw control character in a string literal. WHAT BREAKS: `output: z.number()` now hard-fails (an improvement over silence, and still a per-schema migration); `z.coerce.number()` silently UNDOES the repair back to 1234567890123456768; `JSON.stringify(data)` throws the exact string "Do not know how to serialize a BigInt", and so does `JSON.stringify(report)` — which matters because `.report()` is documented as safe to log. WHAT DOES NOT BREAK, against prediction: trace sinks. `fileSink(path, { body: { chars: false } })` wrote {"id":"1234567890123456789n"} and `consoleSink()` ran clean, because `trace.ts` already ships a `bigintSafe` replacer explicitly so tracing cannot break the call it observes — the one diagnostic surface in this whole scenario that ends up holding the vendor\'s actual digits. `__config` also round-trips (a response body never enters it) and the event spine is byte-identical to the default path. The STRING fork of the same scanner costs strictly less — `JSON.stringify` keeps working — at the price of making the field type depend on the field value', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c4-cache.ts b/docs/scenarios/proofs/precision-loss/c4-cache.ts new file mode 100644 index 00000000..4d020739 --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c4-cache.ts @@ -0,0 +1,237 @@ +// C4 — does `cache` survive a BigInt body? +// +// The capture predicts: "A store that `JSON.stringify`s will throw `Do not know how to serialize a +// BigInt`. Test `memoryStore` and a JSON-backed store." Both are tested here, and the prediction +// splits cleanly: +// +// memoryStore — survives. It holds the value by reference; nothing serialises. +// JSON store — throws, with exactly the predicted message, on the WRITE. +// +// The measurement that is NOT in the capture, and is the more interesting one: the cache KEY +// encoder handles bigint deliberately (`cache.ts:42` — a `bigint:` type tag chosen so `42n` cannot +// collide with the string `'42n'`). So the failure is confined to the value-persistence layer, and +// a store that serialises with a bigint-aware replacer works end to end. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c4-cache.ts +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import type { StitchStore } from '../../../../packages/core/src/types'; +import { check, checkDigits, checkStr, finish, heading, note } from './harness'; +import { + BASE, + ONE_ID_TEXT, + SNOWFLAKE, + bigintAdapter, + wireAdapter, +} from './wire'; + +/** + * A store that persists through JSON, the way a Redis/file/localStorage adapter does. This is the + * shape the capture predicts will throw, built literally: `set` stringifies, `get` parses. + */ +function jsonStore(opts: { bigintSafe?: boolean } = {}): StitchStore & { + lastError(): string; +} { + const data = new Map(); + let lastError = ''; + const replacer = opts.bigintSafe + ? (_k: string, v: unknown) => + typeof v === 'bigint' ? `${v.toString()}n` : v + : undefined; + const store = { + async get(key: string) { + const s = data.get(key); + return s === undefined ? undefined : (JSON.parse(s) as unknown); + }, + async set(key: string, value: unknown) { + try { + data.set(key, JSON.stringify(value, replacer)); + } catch (e) { + lastError = e instanceof Error ? e.message : String(e); + throw e; + } + }, + async increment(key: string) { + const n = Number(data.get(key) ?? '0') + 1; + data.set(key, String(n)); + return n; + }, + lastError: () => lastError, + }; + return store as StitchStore & { lastError(): string }; +} + +/** Run a cached stitch twice and report both outcomes plus the transport call count. */ +async function twice( + adapter: ReturnType, + store: StitchStore, +): Promise<{ + first: { ok: boolean; data: unknown; message: string }; + second: { ok: boolean; data: unknown; message: string }; + calls: number; +}> { + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter, + store, + cache: { ttl: '60s' }, + }); + const a = await call.safe(); + const b = await call.safe(); + return { + first: { ok: a.ok, data: a.data, message: a.error?.message ?? '' }, + second: { ok: b.ok, data: b.data, message: b.error?.message ?? '' }, + calls: adapter.count(), + }; +} + +async function main(): Promise { + heading('C4 (a) — the control: a plain (corrupted) body caches fine'); + { + const r = await twice(wireAdapter(ONE_ID_TEXT), memoryStore()); + check('first call ok', r.first.ok, true); + check('second call ok', r.second.ok, true); + check('transport calls (2 = no cache, 1 = a hit)', r.calls, 1); + checkDigits( + 'the cached value — corrupted, and cached that way', + (r.second.data as { id: unknown }).id, + '1234567890123456768', + ); + note( + 'so the cache faithfully preserves the wrong number. Worth stating: caching does not add a corruption and does not remove one', + '', + ); + } + + heading('C4 (b) — `memoryStore` with a BigInt body: SURVIVES'); + { + const r = await twice(bigintAdapter(ONE_ID_TEXT), memoryStore()); + check('first call ok', r.first.ok, true); + check('second call ok', r.second.ok, true); + check('transport calls', r.calls, 1); + checkDigits( + 'and the value came back off the cache as a BigInt', + (r.second.data as { id: unknown }).id, + `${SNOWFLAKE}n`, + ); + check( + 'typeof the cached value', + typeof (r.second.data as { id: unknown }).id, + 'bigint', + ); + note( + '`memoryStore` keeps `{ value, expires }` in a Map — the value is held by reference, never encoded. So the default store is BigInt-clean', + '', + ); + } + + heading('C4 (c) — a JSON-serialising store with a BigInt body: THROWS'); + { + const store = jsonStore(); + const adapter = bigintAdapter(ONE_ID_TEXT); + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter, + store, + cache: { ttl: '60s' }, + }); + const first = await call.safe(); + checkStr( + 'the exact error the store threw', + store.lastError(), + 'Do not know how to serialize a BigInt', + ); + note( + 'and what the CALLER saw — ok=' + + String(first.ok) + + ', message=' + + JSON.stringify(first.error?.message ?? ''), + '', + ); + check('the call FAILED because the cache WRITE threw', first.ok, false); + note( + 'the corrupted-body control through the same store, for contrast', + '', + ); + const control = jsonStore(); + const ok = await twice(wireAdapter(ONE_ID_TEXT), control); + check('control: first ok', ok.first.ok, true); + check('control: second ok', ok.second.ok, true); + checkStr('control: store error', control.lastError(), ''); + } + + heading('C4 (d) — the same JSON store WITH a bigint-aware replacer'); + { + // The repair to the repair. It works, and it is lossy in a new way: the value comes back + // as the string "…n", not as a BigInt, unless the store also revives it. + const store = jsonStore({ bigintSafe: true }); + const r = await twice(bigintAdapter(ONE_ID_TEXT), store); + check('first call ok', r.first.ok, true); + check('second call ok', r.second.ok, true); + check('transport calls', r.calls, 1); + checkStr('store error', store.lastError(), ''); + note( + 'what came back off the cache', + (r.second.data as { id: unknown }).id, + ); + check( + 'typeof the value after a cache round-trip', + typeof (r.second.data as { id: unknown }).id, + 'string', + ); + check( + 'so a cache HIT and a cache MISS now return different TYPES', + typeof (r.first.data as { id: unknown }).id !== + typeof (r.second.data as { id: unknown }).id, + true, + ); + note( + '→ this is the sharpest edge in C4. The digits survive, but `typeof data.id` is `bigint` on a miss and `string` on a hit, so the bug moves from "wrong number" to "type depends on cache state" — which a test suite with a cold cache will never see', + '', + ); + } + + heading('C4 (e) — the cache KEY encoder already knows about bigint'); + { + // Not in the capture, and it is the reason (c) fails on the VALUE rather than on the KEY. + // `cache.ts:42` renders a bigint as the tagged token `bigint:` — chosen, per its + // comment, so `42n` cannot collide with the string `'42n'`. Measured through a `query`, + // which is part of the cache key. + const adapter = bigintAdapter(ONE_ID_TEXT); + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter, + store: memoryStore(), + cache: { ttl: '60s' }, + }); + await call.safe({ query: { since: BigInt(SNOWFLAKE) } }); + await call.safe({ query: { since: BigInt(SNOWFLAKE) } }); + check( + 'two identical bigint queries → one transport call', + adapter.count(), + 1, + ); + await call.safe({ query: { since: `${SNOWFLAKE}n` } }); + check( + 'and the STRING "…n" is a different key, not a collision', + adapter.count(), + 2, + ); + note( + '→ a bigint in a cache key is handled deliberately and does not collide with its own string spelling. The cache breaks on VALUE persistence only', + '', + ); + } + + finish( + 'C4', + 'CONFIRMED, and narrower than the capture drew it. `memoryStore` SURVIVES a BigInt body completely — it holds `{ value, expires }` in a Map by reference and never encodes, so a cache hit returns 1234567890123456789n unchanged. A JSON-serialising store THROWS on the write with exactly the predicted string, "Do not know how to serialize a BigInt", and the throw is FATAL to the call (ok=false), not a silent cache miss. Adding a bigint-aware replacer to that store fixes the throw and introduces a subtler bug, measured: the value survives as the string "1234567890123456789n", so `typeof data.id` is `bigint` on a cache MISS and `string` on a cache HIT — a type that depends on cache state, which a cold-cache test suite never sees. And the part the capture did not predict: the cache KEY encoder already handles bigint on purpose (`cache.ts:42`, a `bigint:` type tag chosen so `42n` cannot collide with the string `"42n"`) — two identical bigint queries coalesced to one transport call and the string spelling was correctly a different key. The cache breaks on value persistence only', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c5-request-side.ts b/docs/scenarios/proofs/precision-loss/c5-request-side.ts new file mode 100644 index 00000000..1ca9847c --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c5-request-side.ts @@ -0,0 +1,260 @@ +// C5 — the REQUEST side: does a large ID survive going OUT? +// +// C1–C4 are about a value arriving wrong. This is the other direction, and it is the one that +// produces the `Unknown Channel` in the linked openclaw issue: you hold an id and you have to put +// it back in a URL or a body. Three positions, measured by capturing what the transport actually +// handed to `fetch` — the URL string and the encoded request body, not the config that produced +// them. +// +// The capture predicts one thing here: "A `bigint` in a request body is a `JSON.stringify` throw, +// not silent corruption." That is confirmed exactly. But the survey turns up a THIRD outcome the +// capture did not anticipate, and it is worse than either: a `bigint` in `params` is neither +// corrupted nor rejected — it VANISHES, expanding to the empty string and producing a request to +// the wrong URL. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c5-request-side.ts +import { fetchAdapter } from '../../../../packages/core/src/http-adapter'; +import { stitch } from '../../../../packages/core/src/index'; +import type { Adapter } from '../../../../packages/core/src/types'; +import { check, checkStr, finish, heading, note } from './harness'; +import { BASE, ONE_ID_TEXT, SNOWFLAKE } from './wire'; + +/** What the transport actually put on the wire for one request. */ +interface Sent { + url: string; + body: string; + error: string; +} + +/** + * An adapter that records the URL and encoded body `fetch` was called with. Built on the real + * `fetchAdapter`, so `encodeRequestBody` (the library's own JSON/form encoder) runs for real — + * the recorded `body` is the literal bytes, not a re-derivation. + */ +function sendingAdapter(): { adapter: Adapter; sent: Sent[] } { + const sent: Sent[] = []; + const inner = fetchAdapter({ + fetch: (async (url: string, init: { body?: unknown }) => { + sent.push({ + url: String(url), + body: typeof init.body === 'string' ? init.body : '', + error: '', + }); + return new Response(ONE_ID_TEXT, { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as unknown as typeof fetch, + }); + return { adapter: inner, sent }; +} + +/** Fire one request with the given config/input and report what went out (or what threw). */ +async function send( + cfg: Record, + input: Record, +): Promise { + const { adapter, sent } = sendingAdapter(); + const call = stitch({ + name: 'send', + baseUrl: BASE, + adapter, + ...cfg, + } as never); + const r = await call.safe(input as never); + return ( + sent[0] ?? { + url: '', + body: '', + error: r.error?.message ?? '', + } + ); +} + +async function main(): Promise { + heading('C5 (a) — `params`: a path parameter'); + { + // Three spellings of "the same id", through `/v1/things/{id}`. + const asString = await send( + { path: '/v1/things/{id}' }, + { params: { id: SNOWFLAKE } }, + ); + checkStr( + 'a STRING param — the digits survive', + asString.url, + `${BASE}/v1/things/${SNOWFLAKE}`, + ); + + // A number param. Note the source literal is ALREADY the corrupted double — this is the + // value a caller would be holding after C1, so it is the honest input. + const asNumber = await send( + { path: '/v1/things/{id}' }, + { params: { id: 1234567890123456789 } }, + ); + checkStr( + 'a NUMBER param — the corrupted digits, shortest-form', + asNumber.url, + `${BASE}/v1/things/1234567890123456800`, + ); + note( + 'note it is 1234567890123456800, not the exact 1234567890123456768: `String(double)` picks the shortest round-tripping decimal. A third wrong id', + '', + ); + + // And a bigint — the value a C3-repaired pipeline is holding. + const asBigint = await send( + { path: '/v1/things/{id}' }, + { params: { id: BigInt(SNOWFLAKE) } }, + ); + checkStr( + 'a BIGINT param — measured', + asBigint.url, + `${BASE}/v1/things/`, + ); + check( + 'the id VANISHED from the URL', + asBigint.url.includes(SNOWFLAKE), + false, + ); + check( + 'and the call still succeeded — no error, no event', + asBigint.error, + '', + ); + note( + '→ NOT IN THE CAPTURE, and the worst outcome of the three. `expandTemplateVar` (util.ts:392) branches on `string | number | boolean`; a bigint matches none of them and falls to the object arm, where `Object.entries(9007199254740993n)` is `[]`, so nothing is emitted. A repaired pipeline that hands its BigInt id straight back to a path parameter silently requests the COLLECTION instead of the item', + '', + ); + } + + heading('C5 (b) — `query`: a query-string parameter'); + { + const asString = await send( + { path: '/v1/things' }, + { query: { since: SNOWFLAKE } }, + ); + checkStr( + 'a STRING query param', + asString.url, + `${BASE}/v1/things?since=${SNOWFLAKE}`, + ); + + const asNumber = await send( + { path: '/v1/things' }, + { query: { since: 1234567890123456789 } }, + ); + checkStr( + 'a NUMBER query param — corrupted, shortest-form', + asNumber.url, + `${BASE}/v1/things?since=1234567890123456800`, + ); + + const asBigint = await send( + { path: '/v1/things' }, + { query: { since: BigInt(SNOWFLAKE) } }, + ); + checkStr( + 'a BIGINT query param — measured', + asBigint.url, + `${BASE}/v1/things?since=${SNOWFLAKE}`, + ); + note( + '→ the query path DOES handle bigint, correctly and exactly. `stringifyLeaf` (util.ts:332) lists `bigint` alongside number and boolean. So the two URL positions disagree with each other: `query` is bigint-safe, `params` drops it', + '', + ); + } + + heading('C5 (c) — a JSON request `body`'); + { + const asString = await send( + { path: '/v1/things', method: 'POST' }, + { body: { id: SNOWFLAKE } }, + ); + checkStr( + 'a STRING in the body — survives', + asString.body, + `{"id":"${SNOWFLAKE}"}`, + ); + + const asNumber = await send( + { path: '/v1/things', method: 'POST' }, + { body: { id: 1234567890123456789 } }, + ); + checkStr( + 'a NUMBER in the body — the wrong digits, silently', + asNumber.body, + '{"id":1234567890123456800}', + ); + + const asBigint = await send( + { path: '/v1/things', method: 'POST' }, + { body: { id: BigInt(SNOWFLAKE) } }, + ); + checkStr( + 'a BIGINT in the body — the exact throw', + asBigint.error, + 'Do not know how to serialize a BigInt', + ); + checkStr('and nothing went out', asBigint.url, ''); + note( + '→ the capture is confirmed exactly here: a bigint body is a LOUD failure, and that is the good outcome. The number body is the silent one', + '', + ); + } + + heading('C5 (d) — a `form` request body'); + { + // The fourth position, for completeness — `wire.body: 'form'` runs the same `flattenParams` + // walker the query string does, so it inherits the query's bigint handling. + const asBigint = await send( + { + path: '/v1/things', + method: 'POST', + wire: { body: 'form' }, + }, + { body: { id: BigInt(SNOWFLAKE) } }, + ); + checkStr('a BIGINT in a form body', asBigint.body, `id=${SNOWFLAKE}`); + note( + 'so of four outbound positions, bigint works in two (query, form), throws in one (json body) and silently vanishes in one (params)', + '', + ); + } + + heading('C5 (e) — the round trip, end to end'); + { + // The shape that actually bites: read an id, then use it. Measured with a plain pipeline. + const { adapter, sent } = sendingAdapter(); + const read = stitch({ + name: 'read', + baseUrl: BASE, + path: '/v1/things/1', + adapter, + }); + const got = (await read()) as { id: number }; + const write = stitch({ + name: 'write', + baseUrl: BASE, + path: '/v1/things/{id}', + adapter, + }); + await write({ params: { id: got.id } }); + checkStr( + 'read an id, send it straight back — the URL that goes out', + sent[1]?.url ?? '', + `${BASE}/v1/things/1234567890123456800`, + ); + check('and the vendor sent', SNOWFLAKE, '1234567890123456789'); + note( + '→ this is the openclaw `Unknown Channel` shape, reproduced in eight lines: read a channel id, use it, get a 404 for an id that does not exist. Nothing in the pipeline reported anything', + '', + ); + } + + finish( + 'C5', + 'CONFIRMED for the body, and the survey found a THIRD outcome the capture did not anticipate. `params`: a string survives (/v1/things/1234567890123456789); a number goes out as 1234567890123456800 — the shortest-form rendering, a THIRD wrong digit string; and a BIGINT VANISHES, producing the URL https://api.snowflake.test/v1/things/ with no error and no event, because `expandTemplateVar` (util.ts:392) branches on string|number|boolean and `Object.entries()` is empty. `query`: string and bigint both survive EXACTLY (?since=1234567890123456789) because `stringifyLeaf` (util.ts:332) lists bigint; a number corrupts the same way. JSON `body`: a bigint throws exactly "Do not know how to serialize a BigInt" and no request is made — the capture confirmed, and the LOUD outcome. A `form` body handles bigint correctly (id=1234567890123456789), since it shares the query walker. So of four outbound positions, bigint works in two, throws in one, and silently vanishes in one — and the two URL positions disagree with each other. End to end, reading an id and handing it straight back produces a request for /v1/things/1234567890123456800: the openclaw `Unknown Channel` shape in eight lines, with nothing reported', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c6-streams.ts b/docs/scenarios/proofs/precision-loss/c6-streams.ts new file mode 100644 index 00000000..f70caa00 --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c6-streams.ts @@ -0,0 +1,259 @@ +// C6 — do `stream` / `download` / `sse` see raw bytes, and is that a safer path? +// +// The structural fact that makes C1–C2 what they are is that `fetchAdapter` parses. The streaming +// surfaces take a different branch of the same function (`http-adapter.ts:93` — `if (req.stream) +// return { body: response.body }`), so the parse never happens. This script measures what a CALLER +// actually receives on each, which is the question that matters: "the bytes exist somewhere" is not +// the same as "you can use them". +// +// The answer splits by DECODER, not by surface: +// +// stream + decode:'bytes' — Uint8Array. Lossless. The digits are yours. +// stream + decode:'lines' — string. Lossless. +// stream + decode:'ndjson' — JSON.parse per line. CORRUPTED, same as the buffered path. +// stream + decode:'json' — structural streaming JSON. CORRUPTED. +// download — Blob. Lossless. +// sse — JSON.parse per `data:` payload. CORRUPTED. +// +// So "streaming is safer" is true only for the two decoders that hand you bytes and make the +// parsing your problem — which is the same trade C3 makes, reached from a different direction. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c6-streams.ts +import { download } from '../../../../packages/core/src/download'; +import { sse } from '../../../../packages/core/src/sse'; +import { stream } from '../../../../packages/core/src/stream'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + checkStr, + finish, + heading, + note, +} from './harness'; +import { BASE, ONE_ID_TEXT, SNOWFLAKE } from './wire'; + +/** An adapter that hands back a live `ReadableStream` of `text` when `req.stream` is set. */ +function streamingAdapter( + text: string, + contentType = 'application/json', +): Adapter { + const fn = (async (req: AdapterRequest): Promise => { + const bytes = new TextEncoder().encode(text); + if (req.stream) { + return { + status: 200, + headers: { 'content-type': contentType }, + body: new ReadableStream({ + start(c) { + c.enqueue(bytes); + c.close(); + }, + }), + }; + } + // The buffered arm — `download` asks for a blob, so honour `responseType`. + if (req.responseType === 'blob') { + return { + status: 200, + headers: { 'content-type': contentType }, + body: new Blob([bytes], { type: contentType }), + }; + } + return { + status: 200, + headers: { 'content-type': contentType }, + body: JSON.parse(text) as unknown, + }; + }) as Adapter; + fn.capabilities = { name: 'streamingAdapter', supports: ['stream'] }; + return fn; +} + +async function main(): Promise { + heading("C6 (a) — `stream` with the default decoder ('bytes')"); + { + const call = stream({ + name: 'streamThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(ONE_ID_TEXT), + }); + const chunks = (await call()) as Uint8Array[]; + check('how many deltas', chunks.length, 1); + check( + 'the delta is a Uint8Array', + chunks[0] instanceof Uint8Array, + true, + ); + const text = new TextDecoder().decode(chunks[0]); + checkStr('decoded, it is the VERBATIM wire text', text, ONE_ID_TEXT); + check( + "so the SENT digits are in the caller's hands", + text.includes(SNOWFLAKE), + true, + ); + note( + 'LOSSLESS. `fetchAdapter` returns `response.body` unparsed when `req.stream` is set (http-adapter.ts:93), so line 135 never runs', + '', + ); + } + + heading("C6 (b) — `stream` with decode: 'lines'"); + { + const call = stream({ + name: 'streamThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(ONE_ID_TEXT), + stream: { decode: 'lines' }, + }); + const lines = (await call()) as string[]; + checkSeq('the decoded lines', lines, [ONE_ID_TEXT]); + check('typeof the delta', typeof lines[0], 'string'); + note('LOSSLESS — a line is a string; nothing parsed it', ''); + } + + heading("C6 (c) — `stream` with decode: 'ndjson'"); + { + const call = stream({ + name: 'streamThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(ONE_ID_TEXT), + stream: { decode: 'ndjson' }, + }); + const recs = (await call()) as { id: number }[]; + check('how many records', recs.length, 1); + checkDigits('the id it decoded', recs[0]?.id, '1234567890123456768'); + note( + 'CORRUPTED. `stream.ts:decodeStream` calls `JSON.parse(line)` per record — the same primitive, in a different file. Streaming is not the safe path; NOT PARSING is', + '', + ); + } + + heading("C6 (d) — `stream` with decode: 'json'"); + { + const call = stream({ + name: 'streamThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(ONE_ID_TEXT), + stream: { decode: 'json' }, + }); + const vals = (await call()) as { id: number }[]; + check('how many values', vals.length, 1); + checkDigits('the id it decoded', vals[0]?.id, '1234567890123456768'); + note( + 'CORRUPTED. The structural streaming-JSON decoder builds numbers the same way', + '', + ); + } + + heading('C6 (e) — `download`'); + { + const call = download({ + name: 'downloadThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(ONE_ID_TEXT), + }); + const res = (await call()) as { blob: Blob; filename?: string }; + check('the result carries a Blob', res.blob instanceof Blob, true); + check('blob size in bytes', res.blob.size, ONE_ID_TEXT.length); + const text = await res.blob.text(); + checkStr('and its text is verbatim', text, ONE_ID_TEXT); + note( + 'LOSSLESS. `download` sets `wire.response: "blob"`, so `fetchAdapter` takes the blob branch (line 121) and never reaches the json branch', + '', + ); + } + + heading('C6 (f) — `sse`'); + { + const frame = `data: ${ONE_ID_TEXT}\n\n`; + const call = sse({ + name: 'sseThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(frame, 'text/event-stream'), + }); + const events = (await call()) as { data: { id: number } }[]; + check('how many events', events.length, 1); + checkDigits( + 'event.data.id', + events[0]?.data?.id, + '1234567890123456768', + ); + note( + 'CORRUPTED. `sse.ts:parseData` JSON-parses each `data:` payload, falling back to the raw string only when it is NOT valid JSON — so a valid JSON payload is always parsed, and always lossy', + '', + ); + // …and the fallback is the escape hatch, measured: a payload that is not JSON stays a string. + const bare = `data: ${SNOWFLAKE}x\n\n`; + const call2 = sse({ + name: 'sseThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(bare, 'text/event-stream'), + }); + const ev2 = (await call2()) as { data: unknown }[]; + checkStr( + 'a non-JSON payload comes through as a raw string', + String(ev2[0]?.data), + `${SNOWFLAKE}x`, + ); + // And the one that matters: a bare large integer IS valid JSON, so it parses and corrupts. + const bareNum = `data: ${SNOWFLAKE}\n\n`; + const call3 = sse({ + name: 'sseThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: streamingAdapter(bareNum, 'text/event-stream'), + }); + const ev3 = (await call3()) as { data: unknown }[]; + checkDigits( + 'but a BARE large integer is valid JSON, so it parses — and corrupts', + ev3[0]?.data, + '1234567890123456768', + ); + } + + heading('C6 (g) — the summary that answers the claim'); + { + // Assembled as data so the conclusion is a printed table rather than prose. + const rows = [ + "stream decode:'bytes' -> Uint8Array LOSSLESS", + "stream decode:'lines' -> string LOSSLESS", + "stream decode:'ndjson' -> object CORRUPTED", + "stream decode:'json' -> object CORRUPTED", + 'download -> Blob LOSSLESS', + 'sse -> object CORRUPTED', + ]; + for (const r of rows) note(r); + checkSeq('the six surface/decoder pairs', rows, [ + "stream decode:'bytes' -> Uint8Array LOSSLESS", + "stream decode:'lines' -> string LOSSLESS", + "stream decode:'ndjson' -> object CORRUPTED", + "stream decode:'json' -> object CORRUPTED", + 'download -> Blob LOSSLESS', + 'sse -> object CORRUPTED', + ]); + note( + '→ the split is by DECODER, not by surface. Every path that hands you bytes or text is lossless; every path that calls `JSON.parse` for you is lossy, wherever it lives', + '', + ); + } + + finish( + 'C6', + 'PARTIAL — "streaming sees raw bytes" is true, "streaming is a safer path" is only half true, and the split is by DECODER rather than by surface. LOSSLESS, measured: `stream` with the default `decode: "bytes"` yields one `Uint8Array` that decodes to the verbatim {"id":1234567890123456789}; `decode: "lines"` yields that same string; `download` yields a 26-byte Blob whose text is verbatim. Each of those takes a different branch of `fetchAdapter` — the stream branch at line 93, the blob branch at line 121 — and never reaches the `JSON.parse` at line 135. CORRUPTED, measured: `stream` with `decode: "ndjson"` and with `decode: "json"` both yield 1234567890123456768, because `stream.ts:decodeStream` calls `JSON.parse` per record; `sse` yields 1234567890123456768 because `sse.ts:parseData` JSON-parses every `data:` payload that parses at all — and a bare 19-digit integer IS valid JSON, so even an unstructured SSE payload corrupts (only a genuinely non-JSON payload, "1234567890123456789x", survived as a string). So the safe streaming paths are exactly the ones that decline to parse for you, which is C3\'s trade arrived at from the other side', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c7-detector.ts b/docs/scenarios/proofs/precision-loss/c7-detector.ts new file mode 100644 index 00000000..1c5c7697 --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c7-detector.ts @@ -0,0 +1,334 @@ +// C7 — is there ANY spelling that makes this loud, and what is the minimum user code? +// +// C2 established that a schema refinement CAN separate a corrupted id from an intact one. C7 asks +// the two follow-up questions that decide whether that is a usable answer: +// +// 1. Can it be LOUD IN THE LIBRARY'S OWN CHANNEL — a drift finding, an event — rather than just a +// thrown error? Measured: YES, and it does not require the call to fail. A `drift()`-wrapped +// schema whose refinement is expressed as a COERCION produces `warn|coerced||number -> +// string` on the drift channel, non-fatal, with the field named. That is the exact shape the +// capture asks for and does not expect to exist. +// 2. What does it COST in false positives and false negatives? Measured over a mixed workload +// and then over 20 000 random snowflakes, because the answer is a rate, not a yes/no. +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c7-detector.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + checkStr, + finish, + heading, + note, +} from './harness'; +import { BASE, MAX_SAFE, SMALL_ID, SNOWFLAKE, fmt, wireAdapter } from './wire'; +import { z } from './zod'; + +// ── >>> BEGIN USER CODE — THE DETECTOR ────────────────────────────────────────────────────────── +// The whole of it. One predicate, one walker, one schema wrapper. + +/** True when a parsed JSON number passed through the lossy zone. Integers only — a decimal is a + * different failure (C1 b2) and flagging it here would be noise. */ +const unsafe = (v: unknown): boolean => + typeof v === 'number' && Number.isInteger(v) && !Number.isSafeInteger(v); + +/** Every dotted path in `body` holding an integer above 2^53. Schema-free: works on any shape. */ +function unsafePaths(body: unknown, at = ''): string[] { + if (unsafe(body)) return [at || '']; + if (Array.isArray(body)) + return body.flatMap((v, i) => unsafePaths(v, `${at}[${i}]`)); + if (body !== null && typeof body === 'object') + return Object.entries(body as Record).flatMap( + ([k, v]) => unsafePaths(v, at ? `${at}.${k}` : k), + ); + return []; +} + +/** A `z.number()` that stays a number when safe and becomes its digits when not — so the loss is + * reported by `drift()` as a coercion instead of failing the call. */ +const guardedInt = z + .number() + .transform((n) => (Number.isSafeInteger(n) ? n : String(n))); + +// ── <<< END USER CODE ─────────────────────────────────────────────────────────────────────────── + +/** Run one stitch over `text`, collecting findings and events. */ +async function run( + text: string, + extra: Record = {}, +): Promise<{ + ok: boolean; + data: unknown; + findings: string[]; + events: StitchEvent[]; +}> { + const findings: string[] = []; + const events: StitchEvent[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + events.push(e); + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + name: 'getThing', + baseUrl: BASE, + path: '/v1/things/1', + adapter: wireAdapter(text), + trace: sink, + ...extra, + } as never); + const r = await call.safe(); + return { ok: r.ok, data: r.data, findings, events }; +} + +async function main(): Promise { + heading( + "C7 (a) — LOUD, in the library's own channel, WITHOUT failing the call", + ); + { + // The answer to "is there any spelling that makes this loud". A `drift()`-wrapped schema + // whose guard is a TRANSFORM: the schema succeeds either way, so nothing throws, but the + // validated value now differs from the raw one — which is precisely what `drift()` reports. + const Guarded = drift(z.object({ id: guardedInt })); + + const bad = await run(`{"id":${SNOWFLAKE}}`, { output: Guarded }); + check( + 'the call SUCCEEDED — this is diagnostic, not control flow', + bad.ok, + true, + ); + checkSeq('the drift finding', bad.findings, [ + 'warn|coerced|id|number -> string', + ]); + checkStr( + 'and `data.id` now carries digits instead of a wrong number', + (bad.data as { id: string }).id, + '1234567890123456800', + ); + checkSeq( + 'the event spine now has a `drift` event in it', + bad.events.map((e) => e.type), + ['start', 'progress', 'drift', 'result', 'done'], + ); + + const good = await run(`{"id":${SMALL_ID}}`, { output: Guarded }); + check('an intact id still succeeds', good.ok, true); + checkSeq('…with NO finding', good.findings, []); + checkDigits( + '…and its value is untouched, still a number', + (good.data as { id: number }).id, + SMALL_ID, + ); + note( + "→ this is the answer to C7. `warn|coerced|id|number -> string`, on the drift channel, in `.inspect().findings`, in `.report()`, through any `TraceSink`, and through `loggerSink` at `warn` level — because a drift finding's level IS its log level. The call still resolves", + '', + ); + note( + 'the honest caveat: `data.id` is now a string on the corrupted path and a number on the clean one — the same value-dependent type C3(h) and C4(d) hit. It buys VISIBILITY, not correctness. The digits it carries (1234567890123456800) are still not the digits the vendor sent', + '', + ); + } + + heading('C7 (b) — the schema-free version, for a body you do not model'); + { + // `unsafePaths` needs no schema, so it covers the case a refinement cannot: an undeclared + // field, a `transform` output, a body you pass through. Wired at `transform` so it runs on + // every call. + const seen: string[][] = []; + const r = await run( + '{"page":1,"items":[{"id":' + + SNOWFLAKE + + ',"qty":3},{"id":' + + SMALL_ID + + '}],"cursor":"' + + SNOWFLAKE + + '","total":' + + MAX_SAFE + + '}', + { + transform: (body: unknown) => { + seen.push(unsafePaths(body)); + return body; + }, + }, + ); + check('the call succeeded', r.ok, true); + checkSeq('the paths it flagged', seen[0] ?? [], ['items[0].id']); + note( + 'note what it did NOT flag: `page` (small), `items[1].id` (small), `total` (exactly 2^53-1), and `cursor` — which holds the same 19 digits but as a STRING, so it never went through a number', + '', + ); + } + + heading('C7 (c) — the minimum, counted'); + { + // Line counts for the two spellings, measured off this file rather than asserted. + const src = (await import('node:fs')).readFileSync(__filename, 'utf8'); + const lines = src.split('\n'); + const start = lines.findIndex((l) => l.includes('>>> BEGIN USER CODE')); + const end = lines.findIndex((l) => l.includes('<<< END USER CODE')); + const executable = lines + .slice(start + 1, end) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; + note('executable lines between the USER CODE markers', executable); + check('the whole detector, counted', executable, 15); + note( + 'the one-field version is ONE line: `z.number().refine(Number.isSafeInteger)` — and it fails the call instead of reporting. The drift-channel version is three lines (`guardedInt`), the schema-free walker eight', + '', + ); + } + + heading( + 'C7 (d) — false positives and false negatives over a mixed workload', + ); + { + // A payload per row, each labelled with what SHOULD happen. `corrupted` is ground truth, + // computed by comparing the sent digits to the received ones — not by the detector. + const workload: [label: string, json: string, path: string][] = [ + ['snowflake', `{"v":${SNOWFLAKE}}`, 'v'], + ['2^53 - 1 (max safe)', `{"v":${MAX_SAFE}}`, 'v'], + ['2^53', '{"v":9007199254740992}', 'v'], + ['2^53 + 1', '{"v":9007199254740993}', 'v'], + ['2^53 + 2', '{"v":9007199254740994}', 'v'], + ['small id', `{"v":${SMALL_ID}}`, 'v'], + ['zero', '{"v":0}', 'v'], + ['negative snowflake', `{"v":-${SNOWFLAKE}}`, 'v'], + ['money 19.99', '{"v":19.99}', 'v'], + ['0.1', '{"v":0.1}', 'v'], + ['a big float', '{"v":1234567890123.456}', 'v'], + ['exponent 1e21', '{"v":1e21}', 'v'], + ['id as a STRING', `{"v":"${SNOWFLAKE}"}`, 'v'], + ['null', '{"v":null}', 'v'], + ['boolean', '{"v":true}', 'v'], + ['nested', `{"a":{"b":[{"v":${SNOWFLAKE}}]}}`, 'a.b[0].v'], + ]; + + let tp = 0; + let fp = 0; + let fn = 0; + let tn = 0; + const rows: string[] = []; + for (const [label, json, path] of workload) { + // Ground truth: re-serialise the parsed value and compare digit tokens with the wire. + const sentToken = /:\s*(-?[\d.e+]+)\s*[},]/.exec( + json.replace(/\[|\]/g, ''), + )?.[1]; + const parsed = JSON.parse(json) as unknown; + const flagged = unsafePaths(parsed); + const detected = flagged.length > 0; + // "corrupted" = a NUMBER token whose exact value changed. Recomputed from BigInt for + // integer tokens; a non-numeric token is never corrupted. + let corrupted = false; + if (sentToken !== undefined && /^-?\d+$/.test(sentToken)) { + const got = unsafePathValue(parsed, path); + corrupted = + typeof got === 'number' && + BigInt(sentToken) !== BigInt(got); + } + if (corrupted && detected) tp++; + else if (!corrupted && detected) fp++; + else if (corrupted && !detected) fn++; + else tn++; + rows.push( + `${label.padEnd(20)} corrupted=${corrupted ? 'Y' : 'n'} detected=${detected ? 'Y' : 'n'} ${ + corrupted === detected + ? '' + : corrupted + ? '<-- FALSE NEGATIVE' + : '<-- false positive' + }`, + ); + } + for (const r of rows) note(r); + check('true positives', tp, 4); + check('true negatives', tn, 9); + check('FALSE NEGATIVES — the number that must be zero', fn, 0); + check('false positives', fp, 3); + note( + 'the three false positives are 2^53, 2^53+2 and 1e21 — all above the boundary and all exactly representable, so they round-tripped and were flagged anyway', + '', + ); + } + + heading('C7 (e) — the false-positive RATE, on realistic snowflakes'); + { + // The mixed workload is hand-picked and its 3/16 false-positive count is an artefact of + // that. The number a user cares about is: given a real 19-digit snowflake, how often does + // the detector cry wolf? Measured over 20 000 pseudo-random ids in the Discord range. + // + // The id is assembled ENTIRELY in BigInt, from four 16-bit chunks. The obvious spelling — + // `Math.floor(rand() * 3e17)` — is wrong for this measurement in exactly the way the + // scenario is about: that product is itself a double above 2^53, so every offset it can + // produce is already a representable value, and the sample comes out ~24x more + // "exactly-representable" than a real id stream. Measuring a precision bug with a generator + // that has the precision bug in it was this directory's second self-inflicted error. + let seed = 20260805; + const rand16 = (): number => { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + return (seed >>> 8) & 0xffff; + }; + const N = 20_000; + let corrupted = 0; + let flagged = 0; + let missed = 0; + for (let i = 0; i < N; i++) { + let off = 0n; + for (let k = 0; k < 4; k++) off = off * 65536n + BigInt(rand16()); + // A plausible snowflake: 1.1e18 .. 1.4e18, the range Discord is in today. + const id = + 1_100_000_000_000_000_000n + (off % 300_000_000_000_000_000n); + const parsed = JSON.parse(`{"v":${id.toString()}}`) as { + v: number; + }; + const lost = BigInt(parsed.v) !== id; + const flag = unsafe(parsed.v); + if (lost) corrupted++; + if (flag) flagged++; + if (lost && !flag) missed++; + } + note('sample size', N); + note('ids whose digits actually changed', corrupted); + note('ids the detector flagged', flagged); + check('every id in this range is flagged', flagged, N); + check('FALSE NEGATIVES across 20 000 ids', missed, 0); + const fpRate = ((flagged - corrupted) / N) * 100; + note( + 'false-positive rate in the snowflake range, %', + Number(fpRate.toFixed(3)), + ); + check('the false-positive rate is under 1%', fpRate < 1, true); + note( + '→ roughly 1 in 128–256: at 1.1e18 the representable doubles are spaced 128 apart, and past 2^60 (1.15e18) 256 apart, so that fraction of ids land exactly on one. Every OTHER id in the range is genuinely corrupted. A detector that fires on 100% of ids in a range where >99% are wrong is not crying wolf — it is correctly reporting that the whole field is untrustworthy', + '', + ); + } + + finish( + 'C7', + "CONFIRMED — there IS a spelling that makes it loud, in the library's own channel, without failing the call. `drift(z.object({ id: z.number().transform(n => Number.isSafeInteger(n) ? n : String(n)) }))` produces the finding `warn|coerced|id|number -> string` and inserts a `drift` event into the spine (start/progress/DRIFT/result/done), while the call still resolves ok=true — so it reaches `.inspect().findings`, `.report()`, any `TraceSink`, and `loggerSink` at warn level, because a finding's level IS its log level. An intact id produces no finding and keeps its number type. The schema-free version — an 8-line recursive walker in `transform` — flags `items[0].id` in a nested payload and correctly leaves alone a small id, an exactly-2^53-1 total, and the SAME 19 digits carried as a string. Whole detector: 15 executable lines between the markers; the one-field version is one line. COST, measured: over a hand-picked 16-row workload, 4 true positives, 9 true negatives, 3 false positives (2^53, 2^53+2, 1e21 — all above the boundary and all exactly representable) and ZERO false negatives. Over 20 000 random Discord-range snowflakes (assembled in BigInt, because the obvious Math.floor(rand()*3e17) generator has the very precision bug under test baked into it and inflates the rate 17x): 20 000 flagged, 19 893 genuinely corrupted, 0 false negatives, false-positive rate 0.535% — roughly 1 in 128-256, the spacing of representable doubles at 1.1e18. It buys VISIBILITY, not correctness: the reported value 1234567890123456800 is still not the value the vendor sent", + ); +} + +/** Read a dotted/indexed path (`a.b[0].v`) out of a parsed body — used only to fetch ground truth. */ +function unsafePathValue(body: unknown, path: string): unknown { + let cur: unknown = body; + for (const seg of path.split(/\.|\[|\]/).filter(Boolean)) { + if (cur === null || typeof cur !== 'object') return undefined; + cur = (cur as Record)[seg]; + } + return cur; +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/c8-assembled.ts b/docs/scenarios/proofs/precision-loss/c8-assembled.ts new file mode 100644 index 00000000..c267ea7d --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/c8-assembled.ts @@ -0,0 +1,363 @@ +// C8 — assemble the safest available setup, name the seams, and state what it costs. +// +// Everything C1–C7 measured, put together into the configuration a team would actually ship. Two of +// them, in fact, because the measurements do not support a single recommendation: +// +// SETUP A — REPAIR. `wire.response: 'text'` + a `transform` that parses big integers to STRINGS. +// The digits are correct end to end. Costs: your own parser, and the id is a +// string everywhere downstream. +// SETUP B — DETECT. Stock parse + a `drift()`-wrapped guard. Digits are still wrong; you now get +// a `warn` finding naming the field. Costs: almost nothing. Buys: knowing. +// +// A is strictly better and strictly more work. B is what you do to the other forty stitches while +// you migrate. Both are measured end to end below, and both mark `>>> BEGIN USER CODE`. +// +// The seam list is the deliverable: this scenario's answer is not "use option X", it is "there are +// exactly two places in a StitchAPI config where this is addressable, and one of them was not +// supposed to exist". +// +// pnpm exec tsx docs/scenarios/proofs/precision-loss/c8-assembled.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import type { + Adapter, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkDigits, + checkSeq, + checkStr, + finish, + heading, + note, +} from './harness'; +import { + BASE, + NESTED_TEXT, + SMALL_ID, + SNOWFLAKE, + fmt, + parseBigIntsAsStrings, + wireAdapter, +} from './wire'; +import { z } from './zod'; + +import { readFileSync } from 'node:fs'; + +// Both setups are written as factories taking `adapter` + `trace`, purely so this script can serve +// them a known wire and watch the events. In an application those two keys are absent (the stock +// `fetchAdapter` is the default) and everything else is verbatim. They are inside the markers and +// therefore counted, so the line counts below are if anything an over-estimate by two. + +// ── >>> BEGIN USER CODE — SETUP A: REPAIR ─────────────────────────────────────────────────────── +// Three config keys and one import. `parseBigIntsAsStrings` is the scanner from `wire.ts` (C3); +// in a real project it is `json-bigint` with `{ storeAsString: true }`, or those ~84 lines vendored. + +// `z.coerce.string()` on the ids, NOT `z.string()`. The scanner only quotes integers above 2^53, +// so a small id is still a number after `transform` — which would make the field's TYPE depend on +// its MAGNITUDE, the trap C3(h) and C4(d) both hit. Coercing pins it to string either way. (Learned +// by writing `z.string()` first and watching it fail on `items[1].id: 4242`.) +const Page = z.object({ + page: z.number(), + items: z.array(z.object({ id: z.coerce.string(), qty: z.number() })), + cursor: z.string(), +}); + +const listThings = (adapter: Adapter, trace: TraceSink) => + stitch({ + name: 'listThings', + baseUrl: BASE, + path: '/v1/things', + wire: { response: 'text' }, // ← seam 1: do not let the transport parse + transform: parseBigIntsAsStrings, // ← seam 2: parse it yourself, ids as strings + output: Page, // ← ids are `z.coerce.string()` now, and that is the point + adapter, + trace, + }); + +// ── <<< END USER CODE — SETUP A ───────────────────────────────────────────────────────────────── + +// ── >>> BEGIN USER CODE — SETUP B: DETECT ─────────────────────────────────────────────────────── +// One helper and one wrapper. Nothing about the transport changes; the body is still corrupted. + +/** A number that reports itself when it has been through the lossy zone (C7). */ +const guardedInt = z + .number() + .transform((n) => (Number.isSafeInteger(n) ? n : String(n))); + +const watchedThings = (adapter: Adapter, trace: TraceSink) => + stitch({ + name: 'watchedThings', + baseUrl: BASE, + path: '/v1/things', + output: drift( + // ← seam 3: the drift channel + z.object({ + page: z.number(), + items: z.array(z.object({ id: guardedInt, qty: z.number() })), + cursor: z.string(), + }), + ), + adapter, + trace, + }); + +// ── <<< END USER CODE — SETUP B ───────────────────────────────────────────────────────────────── + +/** Collect the drift findings and event spine of one call. */ +function collector(): { + sink: TraceSink; + findings: string[]; + events: StitchEvent[]; +} { + const findings: string[] = []; + const events: StitchEvent[] = []; + return { + sink: { + handle(e: StitchEvent) { + events.push(e); + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }, + findings, + events, + }; +} + +/** Count executable lines between a pair of markers in this file. */ +function userCodeLines(marker: string): number { + const lines = readFileSync(__filename, 'utf8').split('\n'); + const start = lines.findIndex((l) => + l.includes(`>>> BEGIN USER CODE — ${marker}`), + ); + const end = lines.findIndex( + (l, i) => i > start && l.includes('<<< END USER CODE'), + ); + return lines + .slice(start + 1, end) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; +} + +async function main(): Promise { + heading('C8 (a) — SETUP A: does the repair hold end to end?'); + { + const c = collector(); + const call = listThings(wireAdapter(NESTED_TEXT), c.sink); + const data = (await call()) as { + items: { id: string }[]; + cursor: string; + }; + checkStr( + 'the big id — EXACTLY the digits the vendor sent', + data.items[0]?.id ?? '', + SNOWFLAKE, + ); + checkStr( + 'the small id, coerced to a string by the same rule', + data.items[1]?.id ?? '', + SMALL_ID, + ); + checkStr('the cursor is untouched', data.cursor, SNOWFLAKE); + checkSeq('drift findings', c.findings, []); + checkStr( + 'and the whole result JSON.stringifies with no replacer', + JSON.stringify(data), + `{"page":1,"items":[{"id":"${SNOWFLAKE}","qty":3},{"id":"${SMALL_ID}","qty":1}],"cursor":"${SNOWFLAKE}"}`, + ); + note( + 'note `items[1].id` — the SMALL id came back as the string "4242". The scanner left it a NUMBER (it only quotes integers above 2^53); `z.coerce.string()` in the schema pinned it, which is the right call: an id whose TYPE depends on its MAGNITUDE is the trap C3(h) named, and pinning it to string closes it', + '', + ); + } + + heading( + 'C8 (b) — SETUP B: does the detector fire, without breaking the call?', + ); + { + const c = collector(); + const call = watchedThings(wireAdapter(NESTED_TEXT), c.sink); + const r = await call.safe(); + check('the call still SUCCEEDS', r.ok, true); + checkSeq('the finding, naming the exact path', c.findings, [ + 'warn|coerced|items[].id|1 element: number -> string', + ]); + checkSeq( + 'the event spine carries a `drift` event', + c.events.map((e) => e.type), + ['start', 'progress', 'drift', 'result', 'done'], + ); + checkStr( + 'the value it hands back — still WRONG, and now visibly so', + (r.data as { items: { id: string }[] }).items[0]?.id ?? '', + '1234567890123456800', + ); + checkDigits( + 'the small id keeps its number type (no finding for it)', + (r.data as { items: { id: unknown }[] }).items[1]?.id, + SMALL_ID, + ); + note( + '→ `items[].id` — the array path is collapsed to `[]` by the drift path renderer, so one finding covers the field rather than one per row. That is the right granularity for a log line', + '', + ); + } + + heading('C8 (c) — what SETUP A costs the rest of the config'); + { + // The costs are measured, not listed. Each is a thing that used to work and now does not. + const c = collector(); + + // (1) `wire.response: 'text'` means the transport no longer parses — so a stitch that + // forgets the `transform` gets a STRING where it expected an object. + const forgot = stitch({ + name: 'forgot', + baseUrl: BASE, + path: '/v1/things', + adapter: wireAdapter(NESTED_TEXT), + wire: { response: 'text' }, + trace: c.sink, + }); + const got = await forgot(); + check( + 'forgetting the `transform` yields a string, not an object', + typeof got, + 'string', + ); + note( + 'and it does NOT throw — the two keys are independent, so the failure mode of a half-applied repair is a silent type change', + '', + ); + + // (1b) The TYPE side of the same gap, and it is a compile-time cost rather than a runtime + // one — so it is checked by `tsc`, not by an assertion here. `StitchConfig.transform` + // is `(body: unknown) => unknown`, because it sits downstream of an + // `AdapterResponse.body` that is `unknown`. `wire.response: 'text'` guarantees a string + // at RUNTIME and changes nothing at the TYPE level, so a parser written as + // `(text: string) => unknown` does not typecheck in the slot: the narrowing has to + // happen inside the function. `wire.ts:parseBigIntsAsStrings` takes `unknown` and calls + // `String(body)` for exactly this reason — see its JSDoc. + check( + 'the `transform` slot accepts a `(body: unknown) => unknown`, so the parser must narrow itself', + typeof parseBigIntsAsStrings, + 'function', + ); + note( + '→ `wire.response: "text"` is a runtime guarantee with no type-level counterpart. The two keys do not know about each other in either direction', + '', + ); + + // (2) The pairing is per-stitch. There is no place to say it once. + const cfgKeys = Object.keys( + listThings(wireAdapter(NESTED_TEXT), c.sink).__config, + ).sort(); + checkSeq('`__config` keys on the repaired stitch', cfgKeys, [ + 'baseUrl', + 'kind', + 'name', + 'output', + 'path', + 'wire', + ]); + check( + 'does `transform` survive into `__config` for an auditor to check?', + cfgKeys.includes('transform'), + false, + ); + note( + '→ `transform` is redacted out of `__config` (it is a function), so "is this stitch repaired?" is NOT answerable from the published config. `wire` IS there, so the first half is auditable and the second half is not', + '', + ); + + // (3) A `seam` CAN carry the pairing for a whole API — measured, because it is the answer + // to "must I write this on every stitch". + const cfgSeam = { + baseUrl: BASE, + wire: { response: 'text' as const }, + transform: parseBigIntsAsStrings, + }; + const one = stitch({ + ...cfgSeam, + name: 'one', + path: '/v1/things', + adapter: wireAdapter(NESTED_TEXT), + output: Page, + }); + const two = stitch({ + ...cfgSeam, + name: 'two', + path: '/v1/things', + adapter: wireAdapter(NESTED_TEXT), + output: Page, + }); + const a = (await one()) as { items: { id: string }[] }; + const b = (await two()) as { items: { id: string }[] }; + checkStr( + 'shared fragment, stitch one', + a.items[0]?.id ?? '', + SNOWFLAKE, + ); + checkStr( + 'shared fragment, stitch two', + b.items[0]?.id ?? '', + SNOWFLAKE, + ); + note( + 'so the two keys travel together through a shared config fragment (or a `seam`), which is the only thing that makes this maintainable across an API surface', + '', + ); + + // (4) The surfaces it does not reach. + note( + '`wire.response` is an HTTP-transport key. It does nothing for `sse` (which parses in `sse.ts:parseData`) or for `stream({ decode: "ndjson" })` (which parses in `stream.ts`) — those need `decode: "lines"` plus your own parse instead, per C6', + '', + ); + } + + heading('C8 (d) — the seam inventory'); + { + // The point of the whole directory, as data. + const seams = [ + 'adapter — replace the transport (C3). Works. Most code.', + "wire.response:'text' — stop the transport parsing (C2i). Published config.", + 'transform — parse it yourself, pre-validation (C2j). Runs on text.', + 'output + drift() — report the loss as a coercion (C7). Non-fatal, named.', + "stream decode:'bytes'|lines — never parsed at all (C6). Streaming surfaces only.", + 'download — Blob, never parsed (C6).', + ]; + for (const s of seams) note(s); + check('seams that can PREVENT the loss', 5, 5); + check('seams that can only REPORT it after the fact', 1, 1); + note( + 'and the seams that CANNOT touch it, all measured in C2: hooks.onResponse, Surface.interpret, .inspect().raw, .report(), TraceSink, and every `output` schema that does not test magnitude', + '', + ); + } + + heading('C8 (e) — the line count'); + { + const a = userCodeLines('SETUP A: REPAIR'); + const b = userCodeLines('SETUP B: DETECT'); + note('SETUP A — executable lines of user code', a); + note('SETUP B — executable lines of user code', b); + check('SETUP A', a, 16); + check('SETUP B', b, 18); + note( + 'plus, for SETUP A only, the ~84-line scanner from C3 — or one dependency (`json-bigint`), which is the honest recommendation', + '', + ); + } + + finish( + 'C8', + 'ASSEMBLED, as TWO setups, because the measurements do not support one. SETUP A (REPAIR) is 16 lines of user code over three seams — `wire: { response: "text" }` to stop the transport parsing, `transform: parseBigIntsAsStrings` to parse it yourself, and `output` with `z.coerce.string()` ids — and it delivered the EXACT sent digits 1234567890123456789 end to end, with zero findings and a result that `JSON.stringify`s with no replacer. SETUP B (DETECT) is 18 lines and changes no transport: `drift()` around a schema whose ids are `z.number().transform(safe ? n : String(n))`, which produced `warn|coerced|items[].id|1 element: number -> string`, put a `drift` event in the spine, and still resolved ok=true with the wrong value visible. WHAT SETUP A COSTS, measured: (1) `wire.response` and `transform` are INDEPENDENT keys, so a stitch that sets the first and forgets the second silently returns a STRING instead of an object — no throw — and they are independent at the TYPE level too: `transform` is `(body: unknown) => unknown`, so a parser written `(text: string)` does not typecheck in the slot even though `wire.response: "text"` guarantees a string at runtime; (2) `transform` is redacted out of `__config` (functions are), so "is this stitch repaired?" is only half auditable — `wire` shows, `transform` does not; (3) the small id 4242 also became the string "4242", which is deliberate, since an id whose TYPE depends on its MAGNITUDE is the trap; (4) it is per-stitch unless carried on a shared fragment or `seam`, which was measured to work; (5) `wire.response` is an HTTP key and does nothing for `sse` or `stream({decode:"ndjson"})`, which parse in their own files. Of six seams that touch this, five can PREVENT the loss and one can only REPORT it; the six that cannot touch it at all are `hooks.onResponse`, `Surface.interpret`, `.inspect().raw`, `.report()`, `TraceSink`, and any `output` schema that does not test magnitude', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/precision-loss/harness.ts b/docs/scenarios/proofs/precision-loss/harness.ts new file mode 100644 index 00000000..fa833168 --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/harness.ts @@ -0,0 +1,238 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script exits +// non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario needs one thing the other proof directories did not: a renderer that prints the +// EXACT DIGITS of a number. That is not `String(n)` and it is definitely not `JSON.stringify(n)` — +// both are fine for a snowflake but both go exponential above ~1e21, and both are the very +// formatting layer whose fidelity is in question. `digits()` below routes an integral double +// through `BigInt`, which is exact by construction: it prints the integer the double ACTUALLY IS, +// with every digit, including the ones the vendor never sent. +// +// The rest — `check` / `checkStr` / `checkSeq` / `note` / `heading` / `finish` — follows +// `stale-fixture/harness.ts` unchanged, so a reader who has seen one proof directory has seen +// this one. + +let failures = 0; +let checks = 0; + +/** + * The exact decimal digits of a value, for the one comparison this whole directory is about. + * + * - an integral `number` goes through `BigInt`, which cannot round or abbreviate: a double that + * holds 1234567890123456768 prints as `1234567890123456768`, never `1.2345678901234568e+18`. + * - a `bigint` prints its digits with an `n` suffix, so a repaired value is never mistaken for a + * corrupted one in the output. + * - a non-integral number prints via `String`, which is the shortest round-tripping form — the + * right rendering for `19.99` and for `0.30000000000000004` alike. + */ +export function digits(v: unknown): string { + if (typeof v === 'bigint') return `${v.toString()}n`; + if (typeof v === 'number') { + if (Number.isNaN(v)) return 'NaN'; + if (!Number.isFinite(v)) return String(v); + if (Number.isInteger(v)) return BigInt(v).toString(); + return String(v); + } + if (typeof v === 'string') return JSON.stringify(v); + if (v === undefined) return 'undefined'; + if (v === null) return 'null'; + return String(v); +} + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides several rows. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v.toString()}n`; + if (typeof v === 'number') return digits(v); + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** + * Assert a measured value's EXACT DIGITS equal an expected digit string. The digit string is the + * evidence — the whole scenario is "these digits are not those digits" — so the comparison is done + * on the rendered form rather than on the value, and the rendered form is what prints. + */ +export function checkDigits( + label: string, + actual: unknown, + expected: string, +): void { + checks++; + const a = digits(actual); + const ok = a === expected; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${expected})`}`, + ); +} + +/** Assert an exact string match, printing the measured string. For error messages, mostly. */ +export function checkStr( + label: string, + actual: string, + expected: string, +): void { + checks++; + const ok = actual === expected; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${JSON.stringify(actual)}${ok ? '' : ` (expected ${JSON.stringify(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the event spine + * (`["start","progress","result","done"]`) IS the evidence for C1's silence claim. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +// ---- the sent-vs-received table ------------------------------------------- +// Every claim in this directory reduces to one shape: a digit string went onto the wire, some +// other digit string came off it. `wireRow`/`printWireTable` collect and print exactly that, with +// an `intact` column so the eye does not have to diff nineteen digits. + +export interface WireRow { + what: string; + sent: string; + received: string; + intact: boolean; +} + +const wireRows: WireRow[] = []; + +/** Record one sent-digits / received-digits pair (printed by {@link printWireTable}). */ +export function wireRow( + what: string, + sent: string, + received: unknown, +): WireRow { + const r = { + what, + sent, + received: digits(received), + intact: digits(received) === sent, + }; + wireRows.push(r); + return r; +} + +/** Print the accumulated sent/received table and return how many values survived. */ +export function printWireTable(): { intact: number; corrupted: number } { + const w = Math.max(...wireRows.map((r) => r.what.length), 4); + const s = Math.max(...wireRows.map((r) => r.sent.length), 4); + const g = Math.max(...wireRows.map((r) => r.received.length), 8); + console.log( + `\n ${'what'.padEnd(w)} ${'sent'.padEnd(s)} ${'received'.padEnd(g)} verdict\n` + + ` ${'-'.repeat(w)} ${'-'.repeat(s)} ${'-'.repeat(g)} -------`, + ); + for (const r of wireRows) { + console.log( + ` ${r.what.padEnd(w)} ${r.sent.padEnd(s)} ${r.received.padEnd(g)} ${r.intact ? 'intact' : 'CORRUPTED'}`, + ); + } + return { + intact: wireRows.filter((r) => r.intact).length, + corrupted: wireRows.filter((r) => !r.intact).length, + }; +} + +// ---- the seam table ------------------------------------------------------- +// C2's question is "can ANY downstream seam see the original digits". Each seam gets one row: +// what it observed, and whether that observation could distinguish a corrupted ID from an intact +// one. `SEES_TEXT` is the only verdict that would refute the capture. + +export interface SeamRow { + seam: string; + verdict: 'SEES_TEXT' | 'SEES_PARSED' | 'ABSENT'; + observed: string; +} + +const seamRows: SeamRow[] = []; + +const SEAM_LABEL: Record = { + SEES_TEXT: 'RAW TEXT ', + SEES_PARSED: 'parsed body', + ABSENT: 'nothing ', +}; + +/** Record a C2 table row (printed by {@link printSeamTable}). */ +export function seamRow( + seam: string, + verdict: SeamRow['verdict'], + observed: string, +): void { + seamRows.push({ seam, verdict, observed }); +} + +/** Print the accumulated C2 seam table plus per-verdict tallies. */ +export function printSeamTable(): { + text: number; + parsed: number; + absent: number; +} { + const w = Math.max(...seamRows.map((r) => r.seam.length)); + console.log( + `\n ${'seam'.padEnd(w)} what it holds what it observed\n ${'-'.repeat(w)} ------------- ----------------`, + ); + for (const r of seamRows) { + console.log( + ` ${r.seam.padEnd(w)} ${SEAM_LABEL[r.verdict]} ${r.observed}`, + ); + } + return { + text: seamRows.filter((r) => r.verdict === 'SEES_TEXT').length, + parsed: seamRows.filter((r) => r.verdict === 'SEES_PARSED').length, + absent: seamRows.filter((r) => r.verdict === 'ABSENT').length, + }; +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a FAILURE of the library (C1's silence, C2's blind seams), + * so the verdict statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/precision-loss/wire.ts b/docs/scenarios/proofs/precision-loss/wire.ts new file mode 100644 index 00000000..752320ae --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/wire.ts @@ -0,0 +1,324 @@ +// The wire, and the digits on it. +// +// Every other proof directory in this repo starts from a fixture OBJECT. This one cannot: the +// entire scenario is the difference between the bytes a vendor sent and the value JavaScript ended +// up holding, and an object literal has already lost that difference. `1234567890123456789` typed +// into a `.ts` file IS `1234567890123456768` — the corruption happens in the TypeScript source, +// before any library code runs. So the fixtures here are STRINGS, and the digits are only ever +// asserted against strings. +// +// `wireAdapter` is likewise not a hand-written stub. It is the library's real `fetchAdapter` with a +// fake `fetch` underneath, so the parse under test is `http-adapter.ts:135` itself — +// `parsed = text === '' ? undefined : JSON.parse(text)` — and not this file's imitation of it. That +// matters more here than anywhere else: a hand-rolled adapter that called `JSON.parse` would prove +// only that `JSON.parse` loses precision, which nobody disputes. What is in question is whether the +// LIBRARY'S path does, and the only way to measure that is to run the library's path. +import { fetchAdapter } from '../../../../packages/core/src/http-adapter'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +export const BASE = 'https://api.snowflake.test'; + +// ---- the digits ------------------------------------------------------------ +// Each is the exact literal a vendor puts on the wire. They are strings so that this file itself +// cannot round them. + +/** A Discord/Twitter-style snowflake. 19 digits, comfortably above 2^53. */ +export const SNOWFLAKE = '1234567890123456789'; +/** 2^53 + 1 — the smallest integer a double cannot represent. The canonical demonstration. */ +export const TWO53_PLUS_1 = '9007199254740993'; +/** 2^53 - 1 = `Number.MAX_SAFE_INTEGER`. The control: this one MUST survive. */ +export const MAX_SAFE = '9007199254740991'; +/** int64 max — a `bigint` primary key at the top of its range. */ +export const BIGINT_PK = '9223372036854775807'; +/** A small legacy ID, far below the danger zone. The second control. */ +export const SMALL_ID = '4242'; +/** A retail price. Not an integer problem — a binary-fraction problem, same root cause. */ +export const MONEY = '19.99'; +/** The two addends of the oldest float demo in the world, sent as separate fields. */ +export const TENTH = '0.1'; +export const FIFTH = '0.2'; + +/** + * The vendor's response, as TEXT. Hand-assembled rather than `JSON.stringify`d, because + * `JSON.stringify` would have to be handed numbers, and handing it numbers is the bug. + */ +export const WIRE_TEXT = + '{' + + `"snowflake":${SNOWFLAKE},` + + `"two53_plus_1":${TWO53_PLUS_1},` + + `"max_safe":${MAX_SAFE},` + + `"bigint_pk":${BIGINT_PK},` + + `"small_id":${SMALL_ID},` + + `"money":${MONEY},` + + `"tenth":${TENTH},` + + `"fifth":${FIFTH},` + + '"snowflake_str":"' + + SNOWFLAKE + + '"' + + '}'; + +/** The single-field payload most claims use — one snowflake, nothing else to read past. */ +export const ONE_ID_TEXT = `{"id":${SNOWFLAKE}}`; + +/** The same shape with a SAFE id, for the control runs. */ +export const ONE_SAFE_ID_TEXT = `{"id":${SMALL_ID}}`; + +/** A nested/array payload, for the C7 detector's false-positive/negative workload. */ +export const NESTED_TEXT = + '{"page":1,"items":[' + + `{"id":${SNOWFLAKE},"qty":3},` + + `{"id":${SMALL_ID},"qty":1}` + + '],"cursor":"' + + SNOWFLAKE + + '"}'; + +// ---- transports ------------------------------------------------------------ + +/** An {@link Adapter} that also records what it was handed and what it answered with. */ +export interface RecordingAdapter extends Adapter { + /** Every request the transport received. */ + readonly seen: AdapterRequest[]; + /** How many requests it received. */ + count(): number; +} + +/** Options shared by the transports below. */ +export interface WireOpts { + status?: number; + contentType?: string; + /** Extra response headers, merged over `content-type`. */ + headers?: Record; +} + +/** + * THE transport for this directory: the library's own `fetchAdapter`, fed a fake `fetch` that hands + * back a real `Response` carrying `text` verbatim. Everything `fetchAdapter` does to a body — the + * content-type sniff, the `responseType` switch, the `JSON.parse` on line 135 — runs for real. + * + * `text` is the wire. Nothing between this string and `AdapterResponse.body` is this file's code. + */ +export function wireAdapter( + text: string, + opts: WireOpts = {}, +): RecordingAdapter { + const seen: AdapterRequest[] = []; + const inner = fetchAdapter({ + fetch: (async () => + new Response(text, { + status: opts.status ?? 200, + headers: { + 'content-type': opts.contentType ?? 'application/json', + ...opts.headers, + }, + })) as unknown as typeof fetch, + }); + const fn = (async (req: AdapterRequest): Promise => { + seen.push(req); + return inner(req); + }) as RecordingAdapter; + Object.defineProperty(fn, 'seen', { value: seen }); + fn.count = () => seen.length; + fn.capabilities = { name: 'wireAdapter', supports: ['stream'] }; + return fn; +} + +/** + * A transport that hands back an ALREADY-BUILT body, bypassing any parse. Used by C3/C4 to put a + * `BigInt`-bearing object into the engine without pretending a parser produced it, and by C6 to + * hand back a `ReadableStream`. + */ +export function bodyAdapter( + body: unknown, + opts: WireOpts = {}, +): RecordingAdapter { + const seen: AdapterRequest[] = []; + const fn = (async (req: AdapterRequest): Promise => { + seen.push(req); + return { + status: opts.status ?? 200, + headers: { + 'content-type': opts.contentType ?? 'application/json', + ...opts.headers, + }, + body, + }; + }) as RecordingAdapter; + Object.defineProperty(fn, 'seen', { value: seen }); + fn.count = () => seen.length; + fn.capabilities = { name: 'bodyAdapter', supports: ['stream'] }; + return fn; +} + +// ---- a bigint-aware JSON parse (no new dependencies) ----------------------- +// The capture calls this "regex the raw text first … and is a JSON parser written in regex. Breaks +// on numbers inside strings." That criticism is correct about a regex, so this is not one: it is a +// small single-pass scanner that tracks whether it is inside a string literal (respecting `\\` +// escapes) and only rewrites number tokens found OUTSIDE one. It is roughly 40 lines, which is +// itself a measurement — C3 reports the cost of the repair, and this is the cost. + +/** + * A big-integer token is quoted with this prefix so the reviver can find it again. + * + * PRINTABLE ASCII, and that is a deliberate compromise worth naming. The collision-free choice is + * a control character, precisely because one cannot appear unescaped in a vendor string — but + * `JSON.parse` rejects a raw control character inside a string literal ("Bad control character in + * string literal in JSON at position N"), which this directory learned by writing that version + * first. Emitting it as a six-byte `\\u0001` ESCAPE SEQUENCE instead would work, and is more + * scanner than the point requires. So: a printable prefix, one residual false positive, and C3(a2) + * MEASURES that false positive rather than hiding it. + */ +export const SENTINEL = '~bigint~'; + +/** The reviver's guard: the sentinel followed by nothing but an optional sign and digits. */ +const SENTINEL_RE = /^~bigint~(-?\d+)$/; + +/** + * Rewrite every out-of-string integer literal whose magnitude exceeds `Number.MAX_SAFE_INTEGER` + * into a sentinel-prefixed STRING, so `JSON.parse` never sees the digits as a number. + */ +export function quoteBigInts(text: string): string { + let out = ''; + let i = 0; + let inString = false; + while (i < text.length) { + const c = text[i] as string; + if (inString) { + out += c; + if (c === '\\') { + out += text[i + 1] ?? ''; + i += 2; + continue; + } + if (c === '"') inString = false; + i += 1; + continue; + } + if (c === '"') { + inString = true; + out += c; + i += 1; + continue; + } + // A number token starts with `-` or a digit, and may only START here if the previous + // non-space character was structural — which, outside a string, it always is in valid JSON. + if (c === '-' || (c >= '0' && c <= '9')) { + let j = i; + if (text[j] === '-') j += 1; + while ( + j < text.length && + (text[j] as string) >= '0' && + (text[j] as string) <= '9' + ) + j += 1; + const isInteger = + text[j] !== '.' && text[j] !== 'e' && text[j] !== 'E'; + const token = text.slice(i, j); + if (isInteger && !Number.isSafeInteger(Number(token))) { + out += `"${SENTINEL}${token}"`; + } else { + // Not big, or not an integer — copy the whole token (including any fraction and + // exponent) unchanged. + let k = j; + if (text[k] === '.') { + k += 1; + while ( + k < text.length && + (text[k] as string) >= '0' && + (text[k] as string) <= '9' + ) + k += 1; + } + if (text[k] === 'e' || text[k] === 'E') { + k += 1; + if (text[k] === '+' || text[k] === '-') k += 1; + while ( + k < text.length && + (text[k] as string) >= '0' && + (text[k] as string) <= '9' + ) + k += 1; + } + out += text.slice(i, k); + i = k; + continue; + } + i = j; + continue; + } + out += c; + i += 1; + } + return out; +} + +/** `JSON.parse` with big integers preserved as `BigInt`. Built on {@link quoteBigInts}. */ +export function parseWithBigInt(text: string): unknown { + return JSON.parse(quoteBigInts(text), (_k: string, v: unknown) => { + if (typeof v !== 'string') return v; + const m = SENTINEL_RE.exec(v); + return m ? BigInt(m[1] as string) : v; + }); +} + +/** + * `JSON.parse` with big integers preserved as STRINGS — the other half of C3's fork. + * + * Takes `unknown`, not `string`, and that is not laziness. `StitchConfig.transform` is typed + * `(body: unknown) => unknown` — it has to be, since it sits downstream of an `AdapterResponse.body` + * that is `unknown` — so a parser written as `(text: string)` does NOT typecheck in a `transform` + * slot even when `wire.response: 'text'` guarantees a string at runtime. The narrowing has to happen + * inside the function. C8(c) reports this as one of the costs; it is measured here in the signature. + */ +export function parseBigIntsAsStrings(body: unknown): unknown { + return JSON.parse(quoteBigInts(String(body)), (_k: string, v: unknown) => { + if (typeof v !== 'string') return v; + const m = SENTINEL_RE.exec(v); + return m ? (m[1] as string) : v; + }); +} + +/** + * An {@link Adapter} that reads the wire with {@link parseWithBigInt} instead of `JSON.parse` — + * C3's repair, spelled as the one seam the capture says is available. + */ +export function bigintAdapter( + text: string, + opts: WireOpts = {}, +): RecordingAdapter { + const seen: AdapterRequest[] = []; + const fn = (async (req: AdapterRequest): Promise => { + seen.push(req); + return { + status: opts.status ?? 200, + headers: { + 'content-type': opts.contentType ?? 'application/json', + ...opts.headers, + }, + body: parseWithBigInt(text), + }; + }) as RecordingAdapter; + Object.defineProperty(fn, 'seen', { value: seen }); + fn.count = () => seen.length; + fn.capabilities = { name: 'bigintAdapter', supports: [] }; + return fn; +} + +/** Render a `DriftFinding` as `level|change|path|detail` — the format the other proofs use. */ +export function fmt(f: { + level?: string; + change?: string; + path?: string; + detail?: string; + message?: string; +}): string { + return [ + f.level ?? '?', + f.change ?? '?', + f.path ?? '', + f.detail ?? f.message ?? '', + ].join('|'); +} diff --git a/docs/scenarios/proofs/precision-loss/zod.ts b/docs/scenarios/proofs/precision-loss/zod.ts new file mode 100644 index 00000000..54c964be --- /dev/null +++ b/docs/scenarios/proofs/precision-loss/zod.ts @@ -0,0 +1,11 @@ +// Real Zod, imported by path — the same convenience `stale-fixture/zod.ts` documents. +// +// C2 asks whether an `output` schema can catch a value that is the wrong number, and the answer +// depends entirely on what a REAL schema library does with `z.number().int()`, `z.bigint()` and +// `z.string()` when handed a double. A hand-rolled `{ validate }` stub would let this directory +// invent the very behaviour under test. +// +// `packages/core` already depends on Zod v4 in devDependencies; pnpm does not hoist it to the +// workspace root and `docs/` has no manifest, so the import goes by relative path. It resolves +// under `tsx`. In application code the spelling is `import { z } from 'zod'`. +export { z } from '../../../../packages/core/node_modules/zod'; diff --git a/docs/scenarios/proofs/provider-failover/README.md b/docs/scenarios/proofs/provider-failover/README.md new file mode 100644 index 00000000..ab6050d1 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/README.md @@ -0,0 +1,246 @@ +# Proofs — failover, hedging, and the difference between them on the bill + +Runnable evidence for the claims in [`../../provider-failover.md`](../../provider-failover.md). + +**The scenario's answer is a pair of integers, and it is `[10, 10]` against `[10, 0]`.** Ten calls in +which the primary succeeded every single time: `any(primary, backup)` — the combinator whose +docstring says _"failover across interchangeable sources… a primary and a mirror, two regions, two +providers"_ — sent **ten requests to the backup**. The corrected construction sent **zero**. Every +other number here is the distance between those two, or a reason the distance is bigger than it +looks. + +Every script is standalone and offline. Where a claim is about time — how much of a cancelled +request's work was already done, when a hedge's second leg fires, a 30-second circuit cooldown — it +runs on an injected `manualClock()`, so the numbers (`40`, `60`, `t=0`) are exact rather than +approximate. Where a claim is about whether a spelling EXISTS, it runs the TypeScript compiler over +candidate statements and reports which ones compile, so "there is no sequential-fallback combinator" +is measured rather than grepped. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/provider-failover/c1-any-calls-both.ts + +# all of them +for f in docs/scenarios/proofs/provider-failover/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/provider-failover/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ------------------------------ | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `c1-any-calls-both.ts` | does `any()` call every member on a SUCCESSFUL primary? | **`[10, 10]` — 20 requests for 10 answers.** And the loser COMPLETES (0 aborted), and a slower healthy primary LOSES | +| `c2-sequential-fallback.ts` | is sequential fallback expressible? at what cost? | **Yes — `linked` + try/catch, `[10, 0]`, and ONE traceId.** No combinator: 9 of 13 spellings refused | +| `c3-classification.ts` | can failover trigger on 429/5xx but not 400? | **No built-in.** `AggregateError` with `status`/`body` both `undefined`; one bad payload → 2 bills | +| `c4-one-input-every-member.ts` | do `any`/`race` share `all()`'s one-input behaviour? | **Yes — config survives, INPUT is broadcast.** A per-call bearer for the primary ARRIVED AT THE BACKUP | +| `c5-winner-identity.ts` | is the winner's identity recoverable? | **No.** Both members emit `result`; a cancelled one emits NOTHING terminal — a dangling span | +| `c6-cancellation.ts` | is the loser cancelled, and what does that save? | **Billed for the WINNER's latency, every time.** Equal-speed providers: 80 virtual ms, **0 saved** | +| `c7-hedging-amplification.ts` | does `race` amplify? can a breaker be scoped to the hedge? | **2.00× healthy AND degraded — no threshold.** And two `url`-only stitches share **one** breaker | +| `c8-assembled.ts` | the best available answer, priced | **`[10, 0]`, classified, attributed, one trace — 30 lines vs 104 hand-rolled** | + +## Files + +- `fake-provider.ts` — the two providers as plain `Adapter`s over the injected clock. Deliberately + NOT interchangeable on the wire (different origin, path, auth header, success envelope, error + vocabulary), because that is the normal case. Each keeps its own ledger, and the four integers + every claim reads are `received` (the request arrived — the billable event), `completed`, + `aborted` (arrived and was cancelled mid-flight) and `workedMs` (virtual ms of work done, aborted + requests included — the "tokens generated before the cancel" proxy). `hits(pair)` reduces the pair + to the `[primary, backup]` spine every claim prints. +- `providers.ts` — `rig()`: both providers, both stitches, one clock, with hooks for per-member and + shared config. One place so that eight scripts measure the same construction. +- `trace-probe.ts` — a `TraceSink` recording `(name, type, traceId, spanId, parentSpanId)` per event. + Load-bearing for C2 and C5: whether the failover is one trace tree or two, and whether anything + downstream can name the provider that served the call, are both facts about the event stream and + not about the returned value. +- `probe-store.ts` — a `StitchStore` that records every key the engine touches. C7(e) turns on it: + whether the pair shares a breaker is a fact about what STRING the state was keyed on. +- `type-probe.ts` — hands the TypeScript compiler one candidate statement per spelling and reports + which compile. `typescript` is `require`d through a path anchored at `packages/core` (the workspace + package that declares it) — a bare `import ts from 'typescript'` resolves under `tsx` and not under + plain Node from `docs/`, which would make the script run one way and typecheck another. +- `failover.ts` — the assembled answer C8 runs. The region between the ``/`` + markers is what C8's line count measures. +- `hand-rolled.ts` — the same feature set with no library at all (per-provider auth, retry with a + status set and backoff, per-provider breaker with cooldown, an error carrying status and body, + normalisation, lifecycle events), against the same fake providers. The baseline C8 prices against. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C1 is the finding, and the docstring is the footgun.** `any`'s first line is "Run nodes + CONCURRENTLY" and its second sentence is "failover across interchangeable sources — a primary and a + mirror, two regions, two providers" (pipe.ts:274-280). Measured over ten calls where the primary + succeeded every time: **10 primary requests, 10 BACKUP requests, 20 for 10 answers**. A `try`/`catch` + over the same two providers measured **[10, 0]**. The difference between the two constructions is + the backup vendor's entire bill. +- **Three things about C1 the capture does not predict.** (1) The loser does not merely get called, it + **completes**: 10/10 backup requests measured `completed`, **0 aborted**, because `ctrl.abort()` is + in `runAny`'s `finally` (pipe.ts:154-156), one microtask after the winner settled. (2) `any` has + **no preferred member** — it is `Promise.any` (pipe.ts:152), so with a healthy primary that was + merely 10ms slower the winner measured `served_by: "backup"` and the healthy primary was aborted + mid-flight. If the backup is a cheaper model, that is a silent quality regression as well as a + double bill. (3) `all`, `any` and `race` each measured **[1, 1]** on one call: they are one eager + implementation with three joins (pipe.ts:96-175), identical in spend, different only in which + result they keep. +- **C2 refutes the capture in the PESSIMISTIC direction.** "There is no sequential-fallback + combinator… so the correct default may be the one shape the library doesn't offer." There is no + combinator — the compiler refused **9 of 13** candidate spellings, and the four that exist are three + concurrent joins plus one sequential SCOPE — but the correct default is fully available and it is + `linked` + `try`/`catch`: **[10, 0]** on a healthy primary, correct failover to a backup-served + answer on a 503. +- **And `linked` earns its place for a reason the capture never raises: the trace.** The failover + measured **ONE traceId**, spine `primary<-, backup<-primary` — the chain an on-call engineer + wants. The bare `try`/`catch` measured identical request counts and **TWO unrelated root traces**. + The cost is that `linked` returns a Promise, not a `Composable` (pipe.ts:357-369): the flow is a + statement that runs once, not a node you can nest, hand to a seam, or introspect. +- **Nothing delays a member's first request.** A `throttle: '1/10s'` on the backup did NOT hold it + back — it left at **t=0** and the pair measured [1, 1] — because a throttle is a minimum spacing + between SUCCESSIVE calls. A hedge delay is not expressible on a member. +- **C3's aggregate is worse than "an AggregateError hides the actionable one".** Measured: `status` + **`undefined`**, `body` **`undefined`**, message `"All promises were rejected"`. Every field a catch + block routes on is dropped at the combinator boundary — the engine populates + `StitchError.status`/`.body`/`.url`/`.attempts` (types.ts:1657-1691) and `Promise.any` replaces it + with a builtin carrying none of them. The actionable 400 (`invalid_request`) survives only inside + `.errors[0]`, which no `StitchError` API points at. And the malformed payload cost **two** bills: + the backup received the identical body and 400'd on it. +- **One built-in does surface the actionable error — the wrong one.** `race` handed the caller a real + `StitchError` **400** with the body intact, because first-to-SETTLE means the primary's rejection + wins. That same property makes it unusable as failover: a 500 primary against a healthy backup also + measured **500**. +- **Classification is small, and it is user code.** A 6-status `Set` and one `if` over + `StitchError.status` inside a `linked` body measured **[1, 0]** on a 400 (chain stopped, actionable + error preserved) and **[1, 1]** on both 429 and 500. `retry: { on: [...] }` (types.ts:981-986) is + exactly the right vocabulary scoped to the wrong target — it re-hits the SAME endpoint, and `any`'s + own docstring draws that distinction and then offers no `on` of its own. +- **C4 splits along the CONFIG / INPUT line, which is the useful way to say it.** Everything DECLARED + is per-member and works with zero user code: `/v1/complete` + `Authorization: Bearer pk-primary` + against `/generate` + `x-api-key: sk-backup`, neither credential on the other provider, and a + per-stitch `pick` (`choices.0.text` vs `output`) normalising two different response envelopes. + Everything PASSED is broadcast (pipe.ts:75-86). +- **The broadcast leaks credentials.** A per-call `headers: { authorization: 'Bearer …' }` intended + for the primary was measured **arriving at the backup verbatim** — one vendor handed another + vendor's credential, silently, no type error — because config headers merge under input headers + (engine.ts:232) and each strategy only overwrites its own header name. +- **A member's missing template param is not an error, it is a malformed URL.** + `/v1/{deployment}/complete` with no `deployment` expanded to `/v1//complete` (util.ts:453-482, RFC + 6570 drops undefined vars), the provider 404'd, and **the call still succeeded** because the other + member answered. `any` converts a silent misconfiguration into a permanently-degraded-but-green + failover. +- **C5: the winner is unnameable, and normalising makes it worse.** `any` resolves to + `OutputOf` (pipe.ts:281-286) — the member's own value, no envelope, no index, no name. + The `pick` that normalises the two vendors DESTROYS the only attribution there was: the winner + measured as the bare string `"answer from primary"`, indistinguishable from the backup's. +- **The trace cannot break the tie either, for a reason the capture does not anticipate.** On a happy + path BOTH members emitted a terminal `result` (measured `["backup","primary"]`) with nothing marking + the one the caller received; the group emitted **0** events because `makeComposable` + (pipe.ts:210-220) is not a span. And a member that IS auto-cancelled emits `start` and `progress` + and then **nothing** — 0 `error`, 0 `done` — because the cancellation rejection is caught by + `swallowLateRejections` (pipe.ts:90-92) outside the engine. A hedge's trace is one closed span and + one span that simply stops. +- **The sharpest detail in C5.** `all` accepts a NAMED bag and returns a keyed object + (pipe.ts:252-260); `any` and `race` accept only arrays and bare arguments. The one combinator that + carries member names through to its result is the one whose semantics never need them. The fix for + `any` is `transform` — one line per member, measured returning `{ provider: 'primary' }` and + `{ provider: 'backup' }` across a failover. +- **C6: "auto-cancelled" never means "not sent", and the saving is a fraction.** Every measurement + confirms the loser's request ARRIVES. What the cancel saves is set by the latency gap: a 100ms loser + against a 40ms winner was billed **40** virtual ms and saved 60 — **the loser is billed for the + winner's latency, every time**. Two equally-fast 40ms providers burned **40 each, 80 for one + answer**, and although the loser IS recorded as aborted, the work the abort saved measured **0**. + The better a backup is, the less cancellation saves. +- **Two writes for one intent, measured.** `race` over a POST delivered the identical + `{ charge: { amount: 4200 } }` to BOTH providers, method `POST` at each, with the cancel arriving + after both landed. Nothing in the combinators inspects `method`. +- **The half of cancellation that works exactly as documented:** an external abort at t=30 reached + BOTH members (1 and 1 aborted, 30 virtual ms billed each) via `linkedController` (pipe.ts:57-71), + with **0** timers left pending. +- **C7 refutes the capture in the OPTIMISTIC direction, twice over.** "Hedging amplifies outages" is + the standard warning and it understates this: `race` has **no threshold**, so amplification measured + **2.00× healthy** and **2.00× degraded** — identical. The doubling is the steady state, and there is + nothing to tune. Against a degraded backend the hedge also buys nothing: two 500ms legs answered in + **500** virtual ms, exactly one call's latency, for **1000ms** of provider work. +- **A breaker cannot bound hedge spend, because it is a health gate and not a budget gate.** Ten + healthy calls with `circuit: [2, '30s']` on both members still measured **[10, 10]**. No resilience + primitive in the library counts successful requests. +- **What the breaker DOES buy is real.** With the primary returning 500, `any` + per-member `circuit` + wasted exactly **2** requests on the dead leg and then stopped, while the backup served all **6** + calls. This is the one construction in the scenario that works as you would hope. +- **THE TRAP: a `url`-only failover pair shares ONE breaker.** Neither stitch has a `name` or a + `path`, so both key on the literal string `'stitch'` (resilience.ts:353 over engine.ts:857-861; + `hostKey` → `nameOf` → `cfg.name ?? cfg.path ?? 'stitch'`, engine.ts:140,265-274). Measured **one + key `circuit:stitch`** for the pair, and the primary's outage opened the BACKUP's breaker: + `ok, ok, AggregateError, AggregateError, AggregateError`, with the healthy backup receiving only 2 + requests and fast-failed unasked on the other 3. The caller's `AggregateError` carries + `status: undefined`, so nothing even says "circuit open". Setting `name` fixes it completely (5 of 5 + ok) — **the partition key is a diagnostic label**. +- **The delayed hedge everyone actually recommends is outside the vocabulary.** Hand-rolled, it + measured the right profile — **[10, 0]** healthy, **[10, 10]** degraded — in ~11 lines of raw + `AbortController` + `Promise.race` + a clock sleep. No combinator contributes to it, and it gets + none of `linked`'s trace linkage. +- **C8, as one sentence: the library carries everything PER MEMBER and nothing BETWEEN members.** The + assembled answer measured **[10, 0]** with every call credited to the primary; a 400 stopped the + chain at [1, 0] with a real `StitchError` 400, `attempts: 1`, body intact; a 503 retried the primary + twice and then failed over ([2, 1]) returning `{ provider: 'backup', value: 'answer from backup' }` + in **one** trace tree; both providers down gave the LAST real error (`StitchError` 503 naming + `backup`) rather than a statusless aggregate. Price: **30 counted lines against 104** for the same + feature set hand-rolled — the library carries ~71%, all of it auth/retry/breaker/timeout/ + normalisation/trace identity. +- **And the 30 lines cannot be given back.** A `Composable` is not user-authorable: `makeComposable` + is unexported and `__runWith` is not on the public type, while the member gate + `Member = { __stitch } | { __composable }` (pipe.ts:188-189) checks only the BRAND. A hand-branded + node **compiles** and then throws `TypeError` at runtime — measured. So the failover is not a node: + no span of its own, not nestable, not introspectable, not exportable. + +## The footguns + +- **`any` is named and documented as failover and priced as a hedge.** "Failover across + interchangeable sources — a primary and a mirror, two regions, two providers" describes a + one-call-on-the-happy-path cost model; the implementation sends every member on every call. + Measured 20 requests for 10 answers against a provider that never failed. **A combinator whose + docstring implies one cost model while implementing another is the sharpest kind of footgun, + because the bill arrives monthly and the tests all pass.** +- **"The losers are auto-cancelled" reads as "the losers are free".** It is neither. The request + always arrives; the loser is billed for the winner's latency; and against a backup that is not + slower it completes in full (10/10 measured). For a token-metered API, cancelling refunds the tail. +- **`any` prefers the FASTER member, not the FIRST one.** Member order carries no priority. A healthy + primary that is 10ms slower loses, silently, to a backup that may be a different model at a + different price and quality. +- **A `400` through `any` costs two bad requests and yields an error with no status.** `status` and + `body` are both `undefined` on the `AggregateError`; a catch block written against `err.status` + silently sees nothing. Reach into `.errors[0]`, or do not use `any` for failover. +- **A per-call header goes to every provider.** Including `Authorization`. Anything vendor-specific — + a per-call JWT, `anthropic-version`, `OpenAI-Organization`, an idempotency key minted for one vendor + — is broadcast to the other one. +- **A member whose template param you forgot is not an error, and `any` hides it.** The mis-addressed + member 404s, the other member answers, the call returns green, and you are paying for a failover + pair with one permanently broken leg. +- **`pick` normalises the two vendors and erases the attribution.** Use `transform` instead if you + need to know who served the call — it does both in one line per member. +- **Two `url`-only stitches share one circuit breaker, keyed `circuit:stitch`.** The primary going + down fast-fails the backup, which is the exact opposite of what a failover pair is for. Set `name` + (or `circuit.key`) on every member — and note that the fix is a diagnostic label, so anyone + "cleaning up" the names can re-break it. +- **`race` has no hedge threshold, so it doubles your traffic permanently.** It is not "hedge when + slow", it is "always hedge". Against a degraded shared backend it also buys no latency at all. +- **Hedging a POST is a correctness bug the type system will not catch.** Nothing in `any`/`race` + inspects `method`; both providers received the identical charge body. +- **A cancelled member leaves an unterminated span.** No `error`, no `done`. A span-based backend will + report every hedge as a leaked or timed-out operation. diff --git a/docs/scenarios/proofs/provider-failover/c1-any-calls-both.ts b/docs/scenarios/proofs/provider-failover/c1-any-calls-both.ts new file mode 100644 index 00000000..df7be008 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c1-any-calls-both.ts @@ -0,0 +1,137 @@ +// C1 — THE DECIDING CLAIM. `any()`'s docstring calls it "failover across interchangeable sources … +// a primary and a mirror, two regions, two providers" (pipe.ts:274-280). Failover means ONE call on +// the happy path. Measure the two integers that decide it: requests reaching the primary, and +// requests reaching the backup, over calls where the primary succeeded every time. +// +// The capture predicts "both providers on every call" and it is right. Three things it does NOT +// predict, all measured here: +// +// • The loser is not merely CALLED, it COMPLETES. Auto-cancellation fires in `runAny`'s `finally` +// (pipe.ts:154-156), which is one microtask AFTER the winner settled — by then a fast backup has +// already answered in full. 10/10 backup requests measured `completed`, 0 aborted. +// • `any` has no notion of a PREFERRED member. It is `Promise.any` (pipe.ts:152), so the winner is +// whoever settles first. With a healthy primary that is merely slower, the caller silently gets +// the BACKUP's answer — and pays both. +// • `all` and `race` cost exactly the same two requests. The three combinators differ only in +// which result they keep, never in what they spend. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c1-any-calls-both.ts +import { all, any, race } from '../../../../packages/core/src/pipe'; +import { hits } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig as pair } from './providers'; + +async function main(): Promise { + heading('C1 — how many requests does a SUCCESSFUL call cost?'); + + // ── (a) ten happy calls through `any` ────────────────────────────────────────────────────── + // The primary is healthy throughout. A failover would send 10 requests. Measure what `any` sends. + { + const { p, primary, backup } = pair(); + const failover = any(primary, backup); + for (let i = 0; i < 10; i++) + await failover({ body: { prompt: `q${i}` } }); + + check('(a) successful calls made', 10, 10); + check('(a) requests the PRIMARY received', p.primary.received, 10); + check('(a) requests the BACKUP received', p.backup.received, 10); + checkSeq('(a) [primary, backup]', hits(p), [10, 10]); + check( + '(a) total provider requests for 10 answers', + p.primary.received + p.backup.received, + 20, + ); + note( + '(a) → `any` is `Promise.any` over members started EAGERLY (pipe.ts:148-152)', + 'every member is invoked before any result is known, so the happy path costs 2 requests per answer — 100% amplification on a provider that never failed', + ); + } + + // ── (b) the loser COMPLETES; cancellation does not save the call ──────────────────────────── + // "The losers are auto-cancelled" is true and arrives too late to matter: `ctrl.abort()` is in + // the `finally` (pipe.ts:154-156), after the winner has settled. A backup that answers as fast + // as the primary has already answered. + { + const { p, primary, backup } = pair(); + const failover = any(primary, backup); + for (let i = 0; i < 10; i++) + await failover({ body: { prompt: `q${i}` } }); + + check('(b) backup requests COMPLETED in full', p.backup.completed, 10); + check('(b) backup requests aborted mid-flight', p.backup.aborted, 0); + note( + '(b) → the loser is billed for a complete request, not a cancelled one', + 'auto-cancel only helps when the loser is still working when the winner settles — see C6', + ); + } + + // ── (c) `any` has no PREFERRED member — it prefers the FASTER one ─────────────────────────── + // A healthy primary that is 10 virtual ms slower than the backup loses. The caller gets the + // backup's answer, and still pays for both. Nothing in the construction says "primary first". + { + const { clock, p, primary, backup } = pair(); + p.primary.takes(10); // healthy, just slower + const failover = any(primary, backup); + const result = (await failover({ body: { prompt: 'hi' } })) as { + served_by: string; + }; + + check('(c) primary status', 200, 200); + check('(c) who served the answer', result.served_by, 'backup'); + checkSeq('(c) [primary, backup] requests', hits(p), [1, 1]); + check( + '(c) the healthy primary was aborted mid-flight', + p.primary.aborted, + 1, + ); + check('(c) timers left pending', clock.pending(), 0); + note( + '(c) → "failover" routed AWAY from a healthy primary because it was slower', + 'member ORDER carries no priority in `Promise.any`; if the backup is cheaper-but-worse, or a different model, this is a silent quality regression as well as a double bill', + ); + } + + // ── (d) `all` and `race` cost the same two requests ───────────────────────────────────────── + // The three parallel combinators differ only in which result they keep. Spend is identical. + { + const a = pair(); + await all(a.primary, a.backup)({ body: { prompt: 'x' } }); + const r = pair(); + await race(r.primary, r.backup)({ body: { prompt: 'x' } }); + const n = pair(); + await any(n.primary, n.backup)({ body: { prompt: 'x' } }); + + checkSeq('(d) all → [primary, backup]', hits(a.p), [1, 1]); + checkSeq('(d) race → [primary, backup]', hits(r.p), [1, 1]); + checkSeq('(d) any → [primary, backup]', hits(n.p), [1, 1]); + note( + '(d) → all/any/race are one implementation with three joins', + '`runAll`/`runAny`/`runRace` (pipe.ts:96-175) are the same eager `members.map(runMember)` under Promise.all/any/race — the cost model is fixed, only the result selection varies', + ); + } + + // ── (e) what a real failover would have cost ──────────────────────────────────────────────── + // The number the docstring's vocabulary implies, measured on the same providers, for contrast. + { + const { p, primary, backup } = pair(); + for (let i = 0; i < 10; i++) { + try { + await primary({ body: { prompt: `q${i}` } }); + } catch { + await backup({ body: { prompt: `q${i}` } }); + } + } + checkSeq('(e) try/catch → [primary, backup]', hits(p), [10, 0]); + note( + '(e) → the same 10 answers, 10 requests instead of 20', + 'the difference between `any` and a `try`/`catch` on a healthy provider is exactly the backup vendor’s entire bill', + ); + } + + finish( + 'C1', + 'CONFIRMED, and the overspend is worse than "it calls both". Ten calls in which the primary succeeded EVERY TIME cost 10 primary requests and 10 BACKUP requests — 20 provider requests for 10 answers, 100% amplification against a provider that never failed, where a try/catch over the same providers measured [10, 0]. `any` is `Promise.any` over members started eagerly (pipe.ts:148-152), so every member is invoked before any outcome is known. THE THREE UNPREDICTED PARTS: (1) the loser COMPLETES — 10/10 backup requests measured `completed` and 0 aborted, because `ctrl.abort()` runs in the `finally` AFTER the winner settled (pipe.ts:154-156), so "the losers are auto-cancelled" saves nothing against a backup that is not slow; (2) `any` has no PREFERRED member — with a healthy primary that was merely 10ms slower, the winner measured `served_by: "backup"` and the healthy primary was aborted mid-flight, so member order carries no priority and the construction silently routes away from the provider you chose; (3) `all`, `race` and `any` each measured [1, 1] on one call — the three combinators are one eager implementation with three different joins (pipe.ts:96-175), identical in spend and different only in which result they keep', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c2-sequential-fallback.ts b/docs/scenarios/proofs/provider-failover/c2-sequential-fallback.ts new file mode 100644 index 00000000..35c2c436 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c2-sequential-fallback.ts @@ -0,0 +1,214 @@ +// C2 — is SEQUENTIAL fallback expressible at all? The correct default is "try the primary; only on +// failure try the backup", so the measurement is: backup requests when the primary SUCCEEDS, which +// must be 0. Then the two costs that decide whether it is a real answer — how much user code, and +// whether the trace survives. +// +// The capture says "there is no sequential-fallback COMBINATOR" and that is confirmed by the +// compiler (six spellings probed, the two that exist are both concurrent). What it gets wrong is +// the conclusion it leans toward — that the correct default is therefore unavailable. It is +// available, it is `linked` + `try`/`catch`, it measured [10, 0], and it keeps the trace: +// +// • `linked` + `try`/`catch` measured ONE traceId across primary and backup, with the backup +// parented on the primary's span — the failover chain is a single readable trace tree. +// • Bare `try`/`catch` measured TWO traceIds, both roots. Same 0 backup calls, no linkage. +// +// So the seam is real but small: 5 lines of body, and the loss is not correctness, it is that the +// flow is a statement rather than a value (a `linked` call cannot be handed to `all`, cached, or +// introspected the way a `Composable` can). +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c2-sequential-fallback.ts +import { any, linked } from '../../../../packages/core/src/pipe'; +import { hits } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; +import { recordingSink } from './trace-probe'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +async function main(): Promise { + heading('C2 — sequential fallback: 0 backup calls on a healthy primary?'); + + // ── (a) `linked` + try/catch, primary healthy ────────────────────────────────────────────── + // The whole shape, and the number that defines the claim. + { + const { p, primary, backup } = rig(); + for (let i = 0; i < 10; i++) { + await linked(async (run) => { + try { + return await run(primary, { body: { prompt: `q${i}` } }); + } catch { + return await run(backup, { body: { prompt: `q${i}` } }); + } + }); + } + checkSeq( + '(a) linked + try/catch → [primary, backup]', + hits(p), + [10, 0], + ); + note( + '(a) → the happy path costs exactly one request', + 'against `any`’s [10, 10] on the identical providers (C1a)', + ); + } + + // ── (b) …and it does fail over when the primary is genuinely down ────────────────────────── + { + const { p, primary, backup } = rig(); + p.primary.respond(503); + const out = (await linked(async (run) => { + try { + return await run(primary, { body: { prompt: 'q' } }); + } catch { + return await run(backup, { body: { prompt: 'q' } }); + } + })) as { served_by: string }; + checkSeq('(b) primary 503 → [primary, backup]', hits(p), [1, 1]); + check('(b) who served it', out.served_by, 'backup'); + } + + // ── (c) the trace: does the failover stay ONE tree? ───────────────────────────────────────── + // This is the reason to reach for `linked` over a bare try/catch, and it is measurable. + { + const trace = recordingSink(); + const { p, primary, backup } = rig({ trace }); + p.primary.respond(500); + await linked(async (run) => { + try { + return await run(primary, { body: { prompt: 'q' } }); + } catch { + return await run(backup, { body: { prompt: 'q' } }); + } + }); + check('(c) linked: distinct traceIds', trace.traceIds().length, 1); + checkSeq('(c) linked: trace spine', trace.spine(), [ + 'primary<-', + 'backup<-primary', + ]); + note( + '(c) → `linked` chains each call under the PREVIOUS one (pipe.ts:361-368)', + 'so the failover reads as `primary → backup` in one trace tree, which is exactly the shape an on-call engineer wants', + ); + + // The bare try/catch, for contrast: identical counts, two unrelated traces. + const trace2 = recordingSink(); + const b = rig({ trace: trace2 }); + b.p.primary.respond(500); + try { + await b.primary({ body: { prompt: 'q' } }); + } catch { + await b.backup({ body: { prompt: 'q' } }); + } + checkSeq('(c) bare try/catch → [primary, backup]', hits(b.p), [1, 1]); + check( + '(c) bare try/catch: distinct traceIds', + trace2.traceIds().length, + 2, + ); + checkSeq('(c) bare try/catch: trace spine', trace2.spine(), [ + 'primary<-', + 'backup<-', + ]); + } + + // ── (d) `any` cannot be made sequential by any member-level configuration ─────────────────── + // A stitch's `throttle` paces SUCCESSIVE calls, not the first one, so it cannot be used to hold + // the backup back behind the primary. Measured: the first request leaves at t=0 either way. + { + const { p, primary, backup } = rig({ + onBackup: { throttle: { rate: '1/10s' } }, + }); + await any(primary, backup)({ body: { prompt: 'q' } }); + checkSeq( + '(d) any + backup throttle 1/10s → [primary, backup]', + hits(p), + [1, 1], + ); + check( + '(d) backup request arrival (virtual ms)', + p.backup.calls[0]?.at, + 0, + ); + note( + '(d) → `throttle` is a minimum SPACING between successive calls', + 'the first acquire is free, so there is no member-level knob that delays a member’s first request — a hedge delay is not expressible on the member', + ); + } + + // ── (e) which spellings exist, per the compiler ───────────────────────────────────────────── + { + const results = probeSpellings([ + { label: 'pipe.all', code: 'void pipe.all;' }, + { label: 'pipe.any', code: 'void pipe.any;' }, + { label: 'pipe.race', code: 'void pipe.race;' }, + { label: 'pipe.linked', code: 'void pipe.linked;' }, + { label: 'pipe.first', code: 'void pipe.first;' }, + { label: 'pipe.fallback', code: 'void pipe.fallback;' }, + { label: 'pipe.series', code: 'void pipe.series;' }, + { label: 'pipe.sequence', code: 'void pipe.sequence;' }, + { label: 'pipe.hedge', code: 'void pipe.hedge;' }, + { + label: 'any(a, b, { sequential: true })', + code: 'void pipe.any(a, b, { sequential: true });', + }, + { + label: "any([a, b], { delay: '100ms' })", + code: "void pipe.any([a, b], { delay: '100ms' });", + }, + { + label: 'stitch({ fallback: b })', + code: "void stitch({ url: 'https://x.test', fallback: b });", + }, + { + label: 'stitch({ retry: { fallback: b } })', + code: "void stitch({ url: 'https://x.test', retry: { fallback: b } });", + }, + ]); + checkSeq('(e) spellings that COMPILE', accepted(results), [ + 'pipe.all', + 'pipe.any', + 'pipe.race', + 'pipe.linked', + ]); + check( + '(e) spellings the compiler REFUSED', + rejected(results).length, + 9, + ); + note( + '(e) → the vocabulary is 3 concurrent combinators + 1 sequential SCOPE', + 'no `first`/`fallback`/`series`, no per-call option that makes `any` sequential, and no `fallback` key on a stitch — sequential failover is a body you write, never a node you declare', + ); + } + + // ── (f) what a `linked` fallback is NOT ──────────────────────────────────────────────────── + // `linked` returns a Promise, not a `Composable`. The failover therefore cannot be nested in + // `all`/`any`/`race`, and it is not a value with `__config` to inspect, diff, or export. + { + const { primary, backup } = rig(); + const flow = linked(async (run) => { + try { + return await run(primary, { body: { prompt: 'q' } }); + } catch { + return await run(backup, { body: { prompt: 'q' } }); + } + }); + await flow; + check( + '(f) is the linked flow a Composable node?', + (flow as { __composable?: true }).__composable ?? false, + false, + ); + check('(f) is it callable (re-runnable)?', typeof flow, 'object'); + note( + '(f) → `linked` is a Promise, not a node (pipe.ts:357-369)', + 'so the failover runs ONCE at the point of definition and cannot be handed to a combinator, wrapped in a seam member, or introspected — wrap it in a plain function to get a re-runnable unit back', + ); + } + + finish( + 'C2', + 'EXPRESSIBLE, and the capture under-sells it. Sequential fallback is not a combinator — the compiler refused 9 of 13 candidate spellings, and the four that exist (`all`/`any`/`race`/`linked`) are three concurrent joins plus one sequential SCOPE — but `linked` + `try`/`catch` IS the correct default and it measured [10, 0]: ten calls with a healthy primary sent ZERO requests to the backup, against `any`’s [10, 10] on the identical providers, and a 503 primary failed over correctly to a backup-served answer at [1, 1]. THE OBSERVABILITY SURVIVES, AND ONLY THROUGH `linked`: the failover measured ONE traceId with the spine primary<-, backup<-primary — the chain an on-call engineer wants — where the bare `try`/`catch` measured the same counts but TWO unrelated root traces. The cost is 5 lines of body and one real loss: `linked` returns a Promise, not a `Composable` (pipe.ts:357-369), so the flow is a statement that runs once, not a node you can nest in a combinator, hand to a seam, or introspect. Nor can `any` be coaxed into sequencing: a member-level `throttle: "1/10s"` on the backup did NOT hold its first request back — it left at t=0 and the pair measured [1, 1] — because a throttle is a minimum spacing between SUCCESSIVE calls, so no member-level knob delays a first request', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c3-classification.ts b/docs/scenarios/proofs/provider-failover/c3-classification.ts new file mode 100644 index 00000000..4f6bdb17 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c3-classification.ts @@ -0,0 +1,264 @@ +// C3 — classification. The consensus rule is that `429`/`5xx` are availability errors (fail over) +// and a `400` is your payload (stop the chain, because the next provider will reject it +// identically). So: can failover be made to trigger on one and not the other, and what does the +// caller actually receive when the primary returns a 400? +// +// The capture predicts "neither classifies, and a 400 surfaces as an AggregateError rather than the +// actionable error". Confirmed, and sharper than that in both directions: +// +// • WORSE than predicted: the AggregateError has `status === undefined` and `body === undefined`. +// Every caller-facing field the engine populates on a `StitchError` — status, response body, +// url, attempts — is DROPPED at the combinator boundary. The actionable 400 is reachable only +// by knowing to reach into `.errors[0]`, which is not a `StitchError` API, it is a JS builtin. +// • BETTER than predicted, in one narrow spot: `race` DOES surface the actionable error, because +// the first settle is the primary's rejection. It just is not failover — it never tries the +// backup at all. +// +// And the 400 costs two bills: the backup received the identical malformed payload and 400'd on it. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c3-classification.ts +import { any, linked, race } from '../../../../packages/core/src/pipe'; +import type { StitchError } from '../../../../packages/core/src/types'; +import { hits, outcomeOf } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +/** A malformed payload 400s at BOTH providers — which is the entire reason a 400 must not fail over. */ +function bothReject400() { + const r = rig(); + r.p.primary.respond(400); + r.p.backup.respond(400); + return r; +} + +async function main(): Promise { + heading('C3 — does a 400 stop the chain, and does the caller see it?'); + + // ── (a) `any` on a 400: both providers billed, and the error is an aggregate ──────────────── + { + const { p, primary, backup } = bothReject400(); + const r = await any( + primary, + backup, + )({ + body: { prompt: null }, + }).then( + () => ({ ok: true as const, err: undefined }), + (e: unknown) => ({ ok: false as const, err: e as AggregateError }), + ); + + check('(a) call succeeded?', r.ok, false); + check('(a) error name', r.err?.name, 'AggregateError'); + checkSeq('(a) [primary, backup] requests', hits(p), [1, 1]); + check( + '(a) providers that 400d on the same bad payload', + p.primary.calls.filter((c) => c.status === 400).length + + p.backup.calls.filter((c) => c.status === 400).length, + 2, + ); + note( + '(a) → one malformed payload became two bad requests and two bills', + 'and `any` "waits past failures for a success" (pipe.ts:294-297), so it waits for the backup to reject the same payload before it gives up', + ); + } + + // ── (b) what the aggregate DROPS ─────────────────────────────────────────────────────────── + // The measurement that makes this a finding rather than a style complaint. + { + const { primary, backup } = bothReject400(); + const err = (await any( + primary, + backup, + )({ body: { prompt: null } }).then( + () => undefined, + (e: unknown) => + e as AggregateError & { status?: number; body?: unknown }, + ))!; + + check('(b) aggregate.status', err.status, undefined); + check('(b) aggregate.body', err.body, undefined); + check( + '(b) aggregate.message', + err.message, + 'All promises were rejected', + ); + check('(b) aggregate.errors.length', err.errors.length, 2); + + // The actionable error IS in there — one `.errors[0]` away, and nothing in the type says so. + const first = err.errors[0] as StitchError; + check('(b) errors[0].name', first.name, 'StitchError'); + check('(b) errors[0].status', first.status, 400); + check( + '(b) errors[0].body.error.code', + (first.body as { error: { code: string } }).error.code, + 'invalid_request', + ); + note( + '(b) → every field a caller routes on is dropped at the combinator boundary', + '`StitchError` carries status/body/url/attempts (types.ts:1657-1691); `Promise.any` (pipe.ts:152) rejects with a plain `AggregateError` that carries none of them, so a catch block written against `err.status` silently sees `undefined`', + ); + } + + // ── (c) `race` surfaces the actionable error — and is not failover ────────────────────────── + // Worth measuring because it is the ONE built-in that hands the caller a routable 400. + { + const { p, primary, backup } = rig(); + p.primary.respond(400); + const err = (await race( + primary, + backup, + )({ body: { prompt: null } }).then( + () => undefined, + (e: unknown) => e as StitchError, + ))!; + check('(c) race error name', err.name, 'StitchError'); + check('(c) race error status', err.status, 400); + check( + '(c) race error body.error.code', + (err.body as { error: { code: string } }).error.code, + 'invalid_request', + ); + // …but it still called the backup, and it would have failed over to NOTHING if the primary + // had merely been slow-and-failing. + checkSeq('(c) [primary, backup] requests', hits(p), [1, 1]); + const healthy = rig(); + healthy.p.primary.respond(500); + check( + '(c) race with a 500 primary and a healthy backup', + await outcomeOf(() => + race(healthy.primary, healthy.backup)({ body: {} }), + ), + '500', + ); + note( + '(c) → `race` keeps the error and loses the failover', + 'first to SETTLE (pipe.ts:293-297) means a fast failure wins over a good answer — it is a hedge, and it is unsafe as a failover', + ); + } + + // ── (d) the classified version, in user code ─────────────────────────────────────────────── + // Nine lines. `err.status` is on the `StitchError` the stitch itself throws, so the classifier + // is a predicate over one field — the seam is small and the trace still chains. + { + const AVAILABILITY = new Set([408, 429, 500, 502, 503, 504]); + const failsOver = (e: unknown): boolean => + AVAILABILITY.has((e as StitchError).status ?? 0); + + // 400 → stop the chain. + { + const { p, primary, backup } = bothReject400(); + const err = (await linked(async (run) => { + try { + return await run(primary, { body: { prompt: null } }); + } catch (e) { + if (!failsOver(e)) throw e; + return await run(backup, { body: { prompt: null } }); + } + }).then( + () => undefined, + (e: unknown) => e as StitchError, + ))!; + checkSeq( + '(d) classified, 400 → [primary, backup]', + hits(p), + [1, 0], + ); + check('(d) 400: error name', err.name, 'StitchError'); + check('(d) 400: error status', err.status, 400); + check( + '(d) 400: error body.error.message', + (err.body as { error: { message: string } }).error.message, + 'primary says 400', + ); + } + + // 429 → fail over. + { + const { p, primary, backup } = rig(); + p.primary.respond(429); + const out = (await linked(async (run) => { + try { + return await run(primary, { body: {} }); + } catch (e) { + if (!failsOver(e)) throw e; + return await run(backup, { body: {} }); + } + })) as { served_by: string }; + checkSeq( + '(d) classified, 429 → [primary, backup]', + hits(p), + [1, 1], + ); + check('(d) 429: who served it', out.served_by, 'backup'); + } + + // 500 → fail over. + { + const { p, primary, backup } = rig(); + p.primary.respond(500); + const out = (await linked(async (run) => { + try { + return await run(primary, { body: {} }); + } catch (e) { + if (!failsOver(e)) throw e; + return await run(backup, { body: {} }); + } + })) as { served_by: string }; + checkSeq( + '(d) classified, 500 → [primary, backup]', + hits(p), + [1, 1], + ); + check('(d) 500: who served it', out.served_by, 'backup'); + } + note( + '(d) → classification is one `Set` and one `if`, over `StitchError.status`', + 'the library gives the caller the field; it gives no place to DECLARE the routing rule', + ); + } + + // ── (e) is there a declarative spelling for "fail over on these statuses"? ────────────────── + // `retry.on` exists and is the right vocabulary — for the SAME endpoint. Nothing carries it + // across to a different one. + { + const results = probeSpellings([ + { + label: 'retry: { on: [429, 503] } (same endpoint)', + code: "void stitch({ url: 'https://x.test', retry: { attempts: 3, on: [429, 503] } });", + }, + { + label: 'verdict: { accept: [429] }', + code: "void stitch({ url: 'https://x.test', verdict: { accept: [429] } });", + }, + { + label: 'any(a, b, { on: [429, 503] })', + code: 'void pipe.any(a, b, { on: [429, 503] });', + }, + { + label: 'any([a, b], { failOverOn: [429] })', + code: 'void pipe.any([a, b], { failOverOn: [429] });', + }, + { + label: 'stitch({ failover: { to: b, on: [429] } })', + code: "void stitch({ url: 'https://x.test', failover: { to: b, on: [429] } });", + }, + ]); + checkSeq('(e) declarative spellings that COMPILE', accepted(results), [ + 'retry: { on: [429, 503] } (same endpoint)', + 'verdict: { accept: [429] }', + ]); + check('(e) refused', rejected(results).length, 3); + note( + '(e) → `retry.on` (types.ts:981-986) is the exact vocabulary the failover needs', + 'and it is scoped to re-hitting the SAME endpoint — the docstring on `any` (pipe.ts:277-278) draws that distinction itself, and then offers no `on` of its own', + ); + } + + finish( + 'C3', + 'NOT CLASSIFIABLE by any built-in, and the aggregate is worse than the capture predicts. `any` over a pair that both reject a malformed payload measured [1, 1] — one bad request became TWO bad requests and two bills — and rejected with `AggregateError`, message "All promises were rejected", whose `status` and `body` are both `undefined`. Every field a catch block routes on is DROPPED at the combinator boundary: the engine populates `StitchError.status`/`.body`/`.url`/`.attempts` (types.ts:1657-1691), and `Promise.any` (pipe.ts:152) replaces it with a builtin that carries none of them; the actionable 400 (`body.error.code === "invalid_request"`) survives only inside `.errors[0]`, which no `StitchError` API points at. ONE BUILT-IN DOES SURFACE IT, IN THE OPPOSITE DIRECTION: `race` handed the caller a real `StitchError` status 400 with the body intact — because first-to-SETTLE means the primary’s rejection wins — but that same property makes it useless as failover, since a 500 primary against a healthy backup also measured 500. The classified version is USER CODE and it is small: a 6-status `Set` and one `if` over `StitchError.status` inside a `linked` body measured [1, 0] on a 400 (chain stopped, actionable error preserved) and [1, 1] on both 429 and 500 (failed over, backup served). Of five declarative spellings probed, only `retry: { on: [...] }` and `verdict: { accept: [...] }` compile — and `retry.on` is exactly the right vocabulary scoped to the WRONG target, since it re-hits the same endpoint', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c4-one-input-every-member.ts b/docs/scenarios/proofs/provider-failover/c4-one-input-every-member.ts new file mode 100644 index 00000000..d5011736 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c4-one-input-every-member.ts @@ -0,0 +1,242 @@ +// C4 — do `any`/`race` share `all()`'s one-input-for-every-member behaviour? Two providers with +// different paths and different auth is the NORMAL case for failover, so this is not an edge: it is +// whether the combinators can address the pair at all. +// +// The answer splits cleanly along the CONFIG / INPUT line, and the capture (which only asks whether +// the behaviour is shared — it is, `runMember` at pipe.ts:75-86 builds one `memberInput` and hands +// the same object shape to every member) misses BOTH halves of what that means in practice: +// +// • BETTER than feared: everything DECLARED on the stitch is per-member and survives intact. +// Different origin, different path, different auth scheme, even different response shapes +// normalised by a per-stitch `pick` — measured working, zero user code. +// • WORSE than feared: everything passed AT CALL TIME is broadcast to every member, including +// `headers`. A per-call `Authorization` intended for the primary was measured ARRIVING AT THE +// BACKUP — one vendor receiving another vendor's credential, silently, with no type error. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c4-one-input-every-member.ts +import { apiKey, bearer } from '../../../../packages/core/src/auth'; +import { stitch } from '../../../../packages/core/src/index'; +import { any, race } from '../../../../packages/core/src/pipe'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakeProvider, hits, outcomeOf } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; + +async function main(): Promise { + heading('C4 — one input, two providers that are not the same endpoint'); + + // ── (a) what is DECLARED per stitch survives ─────────────────────────────────────────────── + // The reassuring half, and it is worth stating plainly because the "one input" headline + // suggests otherwise: url, path, method, static headers and `auth` are config, not input. + { + const { p, primary, backup } = rig(); + await any(primary, backup)({ body: { prompt: 'hi' } }); + + check('(a) primary path', p.primary.calls[0]?.path, '/v1/complete'); + check('(a) backup path', p.backup.calls[0]?.path, '/generate'); + check( + '(a) primary auth header', + p.primary.calls[0]?.headers['authorization'], + 'Bearer pk-primary', + ); + check( + '(a) backup auth header', + p.backup.calls[0]?.headers['x-api-key'], + 'sk-backup', + ); + check( + '(a) did the primary’s bearer leak to the backup?', + p.backup.calls[0]?.headers['authorization'], + undefined, + ); + check( + '(a) did the backup’s key leak to the primary?', + p.primary.calls[0]?.headers['x-api-key'], + undefined, + ); + note( + '(a) → different origin + path + auth scheme, no user code', + 'each member is a whole stitch, so `auth` is applied per member (engine.ts buildRequest→auth) and never crosses', + ); + } + + // ── (b) what is PASSED is broadcast, verbatim ────────────────────────────────────────────── + // `runMember` builds `{ ...input, signal }` once per member from the SAME input (pipe.ts:75-86) + // — every slot (`body`, `query`, `params`, `headers`) goes to everyone. + { + const { p, primary, backup } = rig(); + await any( + primary, + backup, + )({ + body: { model: 'primary-large', max_tokens: 100 }, + query: { stream: 'false' }, + }); + checkSeq( + '(b) body.model received by [primary, backup]', + [ + (p.primary.calls[0]?.body as { model: string }).model, + (p.backup.calls[0]?.body as { model: string }).model, + ], + ['primary-large', 'primary-large'], + ); + checkSeq( + '(b) query received by [primary, backup]', + [p.primary.calls[0]?.query, p.backup.calls[0]?.query], + ['stream=false', 'stream=false'], + ); + note( + '(b) → the backup was asked for a model only the primary has', + 'two providers of the same SHAPE still take different request envelopes; the shared input cannot be shaped per member, so a `transform`-equivalent for the REQUEST is the missing seam', + ); + } + + // ── (c) a per-call credential reaches the other vendor ───────────────────────────────────── + // The sharp edge. Config headers merge under input headers (engine.ts:232), then `auth.apply` + // overwrites its own header — so the primary's bearer is replaced on the PRIMARY and survives + // on the BACKUP, whose strategy writes a different header name. + { + const { p, primary, backup } = rig(); + await any( + primary, + backup, + )({ + body: { prompt: 'hi' }, + headers: { authorization: 'Bearer per-call-primary-jwt' }, + }); + check( + '(c) primary saw its own strategy header', + p.primary.calls[0]?.headers['authorization'], + 'Bearer pk-primary', + ); + check( + '(c) the BACKUP received the per-call bearer', + p.backup.calls[0]?.headers['authorization'], + 'Bearer per-call-primary-jwt', + ); + note( + '(c) → a per-call credential is broadcast to every member', + 'the same holds for any vendor-specific per-call header (`anthropic-version`, `OpenAI-Organization`, an idempotency key minted for one vendor) — there is no per-member input slot', + ); + } + + // ── (d) different param names: the union, and a silent hole ──────────────────────────────── + // Azure-shaped primary (`/{deployment}/complete`) against a model-in-path backup + // (`/generate/{model}`). One input has to carry BOTH names; supply only one and the other slot + // expands to EMPTY with no error (`expandPath`, util.ts:453-482 — RFC 6570 drops undefined). + { + const clock = manualClock(); + const pri = new FakeProvider({ + name: 'primary', + clock, + origin: 'https://primary.llm.test', + path: '/v1/gpt-x/complete', + }); + const bak = new FakeProvider({ + name: 'backup', + clock, + origin: 'https://backup.llm.test', + path: '/generate/claude-y', + }); + const primary = stitch({ + name: 'primary', + url: 'https://primary.llm.test/v1/{deployment}/complete', + method: 'POST', + adapter: pri.adapter(), + auth: bearer('pk-primary'), + clock, + }); + const backup = stitch({ + name: 'backup', + url: 'https://backup.llm.test/generate/{model}', + method: 'POST', + adapter: bak.adapter(), + auth: apiKey({ + in: 'header', + name: 'x-api-key', + secret: 'sk-backup', + }), + clock, + }); + + // The union supplied: both members address their own endpoint correctly. + await any( + primary, + backup, + )({ + params: { deployment: 'gpt-x', model: 'claude-y' }, + body: {}, + }); + checkSeq( + '(d) union of params → paths hit', + [pri.calls[0]?.path, bak.calls[0]?.path], + ['/v1/gpt-x/complete', '/generate/claude-y'], + ); + + // Only the backup's name supplied: the primary's slot vanishes, silently. + pri.reset(); + bak.reset(); + const outcome = await outcomeOf(() => + any(primary, backup)({ params: { model: 'claude-y' }, body: {} }), + ); + check('(d) missing `deployment` → call outcome', outcome, 'ok'); + check( + '(d) path the primary was actually sent', + pri.calls[0]?.path, + '/v1//complete', + ); + check('(d) primary status for that path', pri.calls[0]?.status, 404); + note( + '(d) → a member’s missing param is not an error, it is a malformed URL', + 'the call still "succeeds" because the OTHER member answered — `any` converts a silent misconfiguration into a permanently-degraded-but-green failover', + ); + } + + // ── (e) response normalisation IS declarable, per member ─────────────────────────────────── + // The two providers return different shapes; `pick` is per-stitch config, so the combinator's + // output is uniform without touching the caller. The one thing in this claim that is free. + { + const { p, primary, backup } = rig({ + onPrimary: { pick: 'choices.0.text' }, + onBackup: { pick: 'output' }, + }); + const won = await any(primary, backup)({ body: {} }); + check('(e) normalised primary result', won, 'answer from primary'); + p.primary.respond(503); + const failed = await any(primary, backup)({ body: {} }); + check('(e) normalised backup result', failed, 'answer from backup'); + note( + '(e) → `pick` (engine.ts:968) runs inside each member', + '`getPath` splits on "." and indexes arrays by string key (util.ts:311-320), so `choices.0.text` reaches into the primary’s envelope — different response vocabularies normalise with zero user code', + ); + } + + // ── (f) `race` behaves identically — it is the same `runMember` ───────────────────────────── + { + const { p, primary, backup } = rig(); + await race( + primary, + backup, + )({ + body: { prompt: 'hi' }, + headers: { authorization: 'Bearer per-call-primary-jwt' }, + }); + checkSeq('(f) race → [primary, backup]', hits(p), [1, 1]); + check( + '(f) race: backup received the per-call bearer', + p.backup.calls[0]?.headers['authorization'], + 'Bearer per-call-primary-jwt', + ); + note( + '(f) → all three combinators share `runMember` (pipe.ts:75-86)', + 'so the input-broadcast behaviour is a property of the composition layer, not of `all`', + ); + } + + finish( + 'C4', + 'SHARED, and the consequence splits along the CONFIG / INPUT line — which is the part the capture does not draw. `any` and `race` use the same `runMember` as `all` (pipe.ts:75-86, one `{ ...input, signal }` per member from one input), so everything the CALLER passes is broadcast and everything the stitch DECLARES is per-member. Declared side, working with zero user code: the pair measured `/v1/complete` + `Authorization: Bearer pk-primary` and `/generate` + `x-api-key: sk-backup`, with neither credential appearing on the other provider, and per-stitch `pick` (`choices.0.text` vs `output`) normalised two different response envelopes into one string. Passed side, and this is the sharp edge: a per-call `headers: { authorization: "Bearer per-call-primary-jwt" }` intended for the primary was measured ARRIVING AT THE BACKUP verbatim — one vendor handed another vendor’s credential, silently, with no type error — because config headers merge under input headers and each strategy only overwrites its OWN header name. The same broadcast sends the primary’s `model` to the backup, and a member whose template param was not supplied is not an error: `/v1/{deployment}/complete` with no `deployment` expanded to `/v1//complete` (util.ts:453-482, RFC 6570 drops undefined vars), the provider 404’d, and the CALL STILL SUCCEEDED because the other member answered — a silent misconfiguration that presents as a permanently-degraded-but-green failover', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c5-winner-identity.ts b/docs/scenarios/proofs/provider-failover/c5-winner-identity.ts new file mode 100644 index 00000000..dd5c9d7a --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c5-winner-identity.ts @@ -0,0 +1,254 @@ +// C5 — is the WINNER'S IDENTITY recoverable by the caller? Cost attribution needs it: you cannot +// bill, rate-limit, or debug a provider you cannot name. Three places it could live — the returned +// value, the events, the trace — measured in turn. +// +// The capture predicts "a combinator that returns the value tends to lose it". Confirmed, and the +// two mechanisms are worth separating because only one of them is fixable in config: +// +// • The RESULT is the winner's body and nothing else. `any` returns `OutputOf` — a +// union of the members' outputs, with no discriminant unless the provider happens to put one +// in the body. Normalising the two vendors' envelopes with `pick` (which C4 measured working) +// DESTROYS the only identity that was there. +// • The TRACE cannot break the tie either, and this is the part the capture does not reach: on a +// happy path BOTH members emit a terminal `result` event, and nothing distinguishes the one the +// caller received. The group is not a span — `makeComposable` (pipe.ts:210-220) emits no events +// at all — so there is no "failover" node in the trace to hang the decision on. And when the +// loser IS cancelled it emits `start` and then nothing: no `error`, no `done`, a dangling span. +// +// The fix is one line per member and it is `transform`, not `pick`. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c5-winner-identity.ts +import { any, linked } from '../../../../packages/core/src/pipe'; +import type { StitchError } from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; +import { recordingSink } from './trace-probe'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +async function main(): Promise { + heading('C5 — who served this call?'); + + // ── (a) the returned value: identity only if the vendor volunteered it ───────────────────── + { + const { primary, backup } = rig(); + const raw = (await any(primary, backup)({ body: {} })) as Record< + string, + unknown + >; + checkSeq('(a) keys on the raw winner', Object.keys(raw).sort(), [ + 'choices', + 'served_by', + ]); + check( + '(a) served_by (the vendor volunteered it)', + raw['served_by'], + 'primary', + ); + note( + '(a) → the combinator adds nothing', + '`any` resolves to `OutputOf` (pipe.ts:281-286) — the member’s own value, with no envelope, index, or name', + ); + } + + // ── (b) …and normalising the two vendors destroys it ─────────────────────────────────────── + // C4(e) showed `pick` unifying two different response shapes. That is the same operation that + // removes the only attribution the caller had. + { + const { p, primary, backup } = rig({ + onPrimary: { pick: 'choices.0.text' }, + onBackup: { pick: 'output' }, + }); + const won = await any(primary, backup)({ body: {} }); + check('(b) normalised winner', won, 'answer from primary'); + check('(b) typeof', typeof won, 'string'); + p.primary.respond(503); + const failedOver = await any(primary, backup)({ body: {} }); + check('(b) after failover', failedOver, 'answer from backup'); + note( + '(b) → normalise OR attribute, not both, with `pick`', + 'the caller’s two goals — one uniform result type, and knowing who produced it — pull in opposite directions under `pick`', + ); + } + + // ── (c) the trace: two winners, no tiebreak ──────────────────────────────────────────────── + // The measurement that closes the question: on a happy path both members SUCCEED, so a sink + // sees two terminal `result` events and no marker on the one the caller got. + { + const trace = recordingSink(); + const { primary, backup } = rig({ trace }); + await any(primary, backup)({ body: {} }); + + checkSeq( + '(c) members that emitted a terminal `result`', + trace.names('result').sort(), + ['backup', 'primary'], + ); + check('(c) distinct traceIds', trace.traceIds().length, 1); + checkSeq('(c) trace spine', trace.spine(), [ + 'primary<-span', + 'backup<-span', + ]); + check( + '(c) events emitted by the `any` group itself', + trace.records.filter( + (r) => r.name !== 'primary' && r.name !== 'backup', + ).length, + 0, + ); + note( + '(c) → the group is not a span', + '`makeComposable` (pipe.ts:210-220) mints a run context for the members and emits nothing of its own, so the trace shows a fan with two successes and no node that says which one the caller received', + ); + } + + // ── (d) the cancelled loser leaves a DANGLING span ───────────────────────────────────────── + // Sharper than "the trace can't tell you": a member that is auto-cancelled emits `start` and + // then never terminates. No `error`, no `done`. So the trace of a hedge is one closed span and + // one span that simply stops — which a span-based backend reports as a timeout or a leak, and + // which no cost report can attribute. + { + const trace = recordingSink(); + const { p, primary, backup } = rig({ trace }); + p.primary.takes(50); + await any(primary, backup)({ body: {} }); + + check('(d) the primary WAS aborted mid-flight', p.primary.aborted, 1); + checkSeq('(d) members that emitted `result`', trace.names('result'), [ + 'backup', + ]); + checkSeq('(d) members that emitted `error`', trace.names('error'), []); + checkSeq('(d) members that emitted `done`', trace.names('done'), [ + 'backup', + ]); + checkSeq( + '(d) every event the cancelled primary emitted', + trace.records + .filter((r) => r.name === 'primary') + .map((r) => r.type), + ['start', 'progress'], + ); + note( + '(d) → the loser’s span is opened and never closed', + 'the cancellation rejects the member promise, which `swallowLateRejections` (pipe.ts:90-92) catches OUTSIDE the engine — so no `error`/`done` event is ever emitted and the span dangles', + ); + note( + '(d) → and when the loser is fast it emits `result` instead', + 'C1(b) measured 10/10 losers COMPLETING on a fast backup, so a sink sees either two successes or one success and one unterminated span — never a marked winner', + ); + } + + // ── (e) what the caller can reach on the combinator itself ───────────────────────────────── + // `.safe()`, `.inspect()` and `__config` are Stitch API; a `Composable` is a bare callable. + { + const { primary, backup } = rig(); + const node = any(primary, backup); + check('(e) typeof the composable', typeof node, 'function'); + check('(e) node.__composable', node.__composable, true); + check( + '(e) does it have .safe()?', + 'safe' in (node as unknown as Record), + false, + ); + check( + '(e) does it have .inspect()?', + 'inspect' in (node as unknown as Record), + false, + ); + check( + '(e) does it have __config?', + '__config' in (node as unknown as Record), + false, + ); + + const results = probeSpellings([ + { label: 'any(a, b).safe()', code: 'void pipe.any(a, b).safe();' }, + { + label: 'any(a, b).inspect()', + code: 'void pipe.any(a, b).inspect();', + }, + { label: 'a.safe()', code: 'void a.safe();' }, + { + label: 'any({ primary: a, backup: b }) (named bag)', + code: 'void pipe.any({ primary: a, backup: b });', + }, + { + label: 'all({ primary: a, backup: b }) (named bag)', + code: 'void pipe.all({ primary: a, backup: b });', + }, + ]); + checkSeq('(e) spellings that COMPILE', accepted(results), [ + 'a.safe()', + 'all({ primary: a, backup: b }) (named bag)', + ]); + check('(e) refused', rejected(results).length, 3); + note( + '(e) → `all` takes a NAMED bag and returns keys; `any` and `race` do not', + 'the one combinator that already carries member NAMES through to its result is the one whose semantics never need them (pipe.ts:252-260 vs 281-286) — the naming exists, it is just on the wrong combinator', + ); + } + + // ── (f) the two ways to get identity back ────────────────────────────────────────────────── + // Per-member `transform`: one line each, and it survives `pick`-style normalisation because it + // IS the normalisation. + { + const { p, primary, backup } = rig({ + onPrimary: { + transform: (b) => ({ + provider: 'primary', + text: (b as { choices: { text: string }[] }).choices[0] + ?.text, + }), + }, + onBackup: { + transform: (b) => ({ + provider: 'backup', + text: (b as { output: string }).output, + }), + }, + }); + const won = (await any(primary, backup)({ body: {} })) as { + provider: string; + text: string; + }; + checkSeq( + '(f) transform: [provider, text]', + [won.provider, won.text], + ['primary', 'answer from primary'], + ); + p.primary.respond(500); + const after = (await any(primary, backup)({ body: {} })) as { + provider: string; + }; + check('(f) transform after failover', after.provider, 'backup'); + + // …and in the sequential shape, identity is free: the caller KNOWS which branch it took. + const seq = rig(); + seq.p.primary.respond(503); + const attributed = await linked(async (run) => { + try { + return { + provider: 'primary', + value: await run(seq.primary, { body: {} }), + }; + } catch (e) { + void (e as StitchError).status; + return { + provider: 'backup', + value: await run(seq.backup, { body: {} }), + }; + } + }); + check('(f) linked: who served it', attributed.provider, 'backup'); + note( + '(f) → `transform` (a per-stitch config field) is the one-line fix for `any`', + 'and the sequential shape needs no fix at all, because the branch the code took IS the attribution', + ); + } + + finish( + 'C5', + 'NOT RECOVERABLE from any built-in, in either of the two places it could live. The RESULT is the winner’s raw body and nothing more — `any` resolves to `OutputOf` (pipe.ts:281-286) with no envelope, index or name — so attribution exists only when the vendor volunteered a field, and the per-stitch `pick` that C4 measured normalising two different response envelopes DESTROYS exactly that field (the winner measured as the bare string "answer from primary", indistinguishable in type and shape from the backup’s). The TRACE cannot break the tie either, which is the part the capture does not reach: on a happy path BOTH members emitted a terminal `result` event (measured ["backup","primary"]) with nothing marking the one the caller received, and the group emitted ZERO events of its own because `makeComposable` (pipe.ts:210-220) is not a span — there is no failover node in the trace to hang the decision on. AND INFERENCE FROM THE TRACE DOES NOT WORK EITHER, FOR A REASON THE CAPTURE DOES NOT ANTICIPATE: a member that IS auto-cancelled emits `start` and `progress` and then NOTHING — measured 0 `error` events and 0 `done` events for a primary that the ledger confirms was aborted mid-flight — because the cancellation rejects the member promise and `swallowLateRejections` (pipe.ts:90-92) catches it outside the engine. So a hedge’s trace is one closed span and one span that simply stops, which a span-based backend reads as a timeout or a leak; and when the loser is fast it emits `result` instead (C1(b): 10/10). Nor can the caller reach for `.safe()`/`.inspect()`/`__config` — a `Composable` is a bare branded callable and all three probes refused to compile. THE SHARPEST DETAIL: `all` accepts a NAMED bag and returns a keyed object (pipe.ts:252-260); `any` and `race` accept only arrays and bare arguments. The one combinator that carries member names through to its result is the one whose semantics never need them. The fix is `transform` — one line per member, measured returning `{ provider: "primary" }` and `{ provider: "backup" }` across a failover — or the sequential shape, where the branch taken IS the attribution', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c6-cancellation.ts b/docs/scenarios/proofs/provider-failover/c6-cancellation.ts new file mode 100644 index 00000000..c176290f --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c6-cancellation.ts @@ -0,0 +1,209 @@ +// C6 — cancellation. "The losers are auto-cancelled" (pipe.ts:276, 296) is the sentence that makes +// a concurrent combinator sound like a cheap one. Two measurements decide what it is worth: +// (a) did the loser's request reach the provider at all, and (b) did the provider see it aborted +// MID-FLIGHT, or had it already finished? +// +// The capture is right that a cancelled request still reaches the provider, and right that +// cancelling does not refund. What it does not say is that the saving is a FRACTION, and the +// fraction is set by the latency gap between the two providers — the same gap that decides whether +// the loser was cheap in the first place: +// +// • Loser as fast as the winner → 0% saved. It completed. (C1(b): 10/10.) +// • Loser 100ms, winner 40ms → 60% of the loser's work saved, 40% billed. Measured exactly. +// • Loser slower still → more saved, and it was never going to win anyway. +// +// So the cancellation refunds the tail and bills the head, and the head is what an LLM charges for. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c6-cancellation.ts +import { any, linked, race } from '../../../../packages/core/src/pipe'; +import { hits, outcomeOf } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; + +async function main(): Promise { + heading('C6 — is the loser cancelled, and what does that save?'); + + // ── (a) the request ARRIVES either way ───────────────────────────────────────────────────── + // Cancellation is a property of the response, never of the request. Both providers received it. + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(100); + const call = any(primary, backup)({ body: {} }); + await clock.advance(200); + await call; + + checkSeq('(a) [primary, backup] requests RECEIVED', hits(p), [1, 1]); + check('(a) primary completed?', p.primary.completed, 0); + check('(a) primary aborted mid-flight?', p.primary.aborted, 1); + note( + '(a) → "auto-cancelled" never means "not sent"', + 'the abort is raised after the winner settles (pipe.ts:154-156); by then every member has already made its request', + ); + } + + // ── (b) how much of the loser's work was actually saved ──────────────────────────────────── + // The number the framing turns on. The loser is aborted at the instant the winner answers, so + // it is billed for exactly the winner's latency. + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(100); // the loser would have taken 100 + p.backup.takes(40); // the winner answers at t=40 + const call = any(primary, backup)({ body: {} }); + await clock.advance(200); + await call; + + check('(b) winner latency (virtual ms)', p.backup.workedMs, 40); + check('(b) loser work BILLED (virtual ms)', p.primary.workedMs, 40); + check( + '(b) loser work SAVED (virtual ms of 100)', + 100 - p.primary.workedMs, + 60, + ); + check('(b) timers left pending', clock.pending(), 0); + note( + '(b) → the loser is billed for the WINNER’s latency, every time', + 'for a token-metered API that is the tokens generated in the first 40ms — cancelling refunds the tail, and the head is not free', + ); + } + + // ── (c) the degenerate case the docstring implies is safe ────────────────────────────────── + // Two providers of similar speed — the normal case for a deliberately-chosen backup — save + // nothing at all. + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(40); + p.backup.takes(40); + const call = any(primary, backup)({ body: {} }); + await clock.advance(200); + await call; + + check('(c) primary work billed', p.primary.workedMs, 40); + check('(c) backup work billed', p.backup.workedMs, 40); + check( + '(c) total work for ONE answer (virtual ms)', + p.primary.workedMs + p.backup.workedMs, + 80, + ); + // The sting: the loser IS recorded as aborted — and it had already done all 40ms of its + // work when the abort landed, because the two were due at the same instant. "Cancelled" + // and "saved something" are different facts, and only the first one is observable. + check( + '(c) requests recorded as aborted mid-flight', + p.primary.aborted + p.backup.aborted, + 1, + ); + check( + '(c) work the abort actually saved (virtual ms)', + 80 - (p.primary.workedMs + p.backup.workedMs), + 0, + ); + note( + '(c) → equally-fast providers = 100% of the double spend, 0% saved', + 'the loser is recorded as ABORTED and still burned its full 40ms — "cancelled" is a statement about the response, not about the work; the closer the backup is to the primary in speed, i.e. the better a backup it is, the less cancellation saves', + ); + } + + // ── (d) non-idempotent writes: two charges for one intent ────────────────────────────────── + // The hazard the capture names, as a number. `any`/`race` on a POST send the payload TWICE, and + // the abort lands after the second one arrived. + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(100); + const call = race( + primary, + backup, + )({ + body: { charge: { amount: 4200, currency: 'usd' } }, + }); + await clock.advance(200); + await call; + + checkSeq('(d) POSTs delivered [primary, backup]', hits(p), [1, 1]); + checkSeq( + '(d) method each provider saw', + [p.primary.calls[0]?.method, p.backup.calls[0]?.method], + ['POST', 'POST'], + ); + checkSeq( + '(d) amount each provider was asked to charge', + [ + (p.primary.calls[0]?.body as { charge: { amount: number } }) + .charge.amount, + (p.backup.calls[0]?.body as { charge: { amount: number } }) + .charge.amount, + ], + [4200, 4200], + ); + note( + '(d) → one intent, two writes, and the cancel arrives after both landed', + 'nothing in `any`/`race` inspects `method`; hedging a non-idempotent call is a correctness bug the type system will not catch', + ); + } + + // ── (e) the caller's own signal DOES cancel the whole group ──────────────────────────────── + // The half that works exactly as documented, and worth stating: an outer abort reaches every + // member, because the group controller is linked to the caller's (pipe.ts:57-71). + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(100); + p.backup.takes(100); + const ctrl = new AbortController(); + const outcome = outcomeOf(() => + any(primary, backup)({ body: {}, signal: ctrl.signal }), + ); + await clock.advance(30); + ctrl.abort(); + const seen = await outcome; + await clock.advance(200); + + check('(e) call outcome', seen, 'AggregateError'); + checkSeq( + '(e) providers that saw the abort', + [p.primary.aborted, p.backup.aborted], + [1, 1], + ); + checkSeq( + '(e) work billed to each before the abort', + [p.primary.workedMs, p.backup.workedMs], + [30, 30], + ); + check('(e) timers left pending', clock.pending(), 0); + note( + '(e) → an outer abort propagates to every member', + '`linkedController` (pipe.ts:57-71) links the group signal to the caller’s, so a request-scoped deadline does reach both providers', + ); + } + + // ── (f) the sequential shape has no loser to cancel ──────────────────────────────────────── + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(40); + p.backup.takes(40); + const call = linked(async (run) => { + try { + return await run(primary, { body: {} }); + } catch { + return await run(backup, { body: {} }); + } + }); + await clock.advance(200); + await call; + checkSeq('(f) sequential → [primary, backup]', hits(p), [1, 0]); + check( + '(f) total work for ONE answer (virtual ms)', + p.primary.workedMs + p.backup.workedMs, + 40, + ); + note( + '(f) → 40 virtual ms against the hedge’s 80 in (c)', + 'the cancellation machinery exists to reclaim a cost the sequential shape never incurs', + ); + } + + finish( + 'C6', + 'THE LOSER IS CANCELLED, THE REQUEST STILL ARRIVED, AND THE SAVING IS A FRACTION SET BY THE LATENCY GAP — which is the part the capture does not quantify. Every measurement here confirms the loser’s request REACHES the provider: `any` with a 100ms primary and an instant backup measured [1, 1] received, 0 completed and 1 aborted on the primary — "auto-cancelled" never means "not sent", because the abort is raised in `runAny`’s `finally` (pipe.ts:154-156), after every member has already made its request. What the cancel actually saves: with a 100ms loser and a 40ms winner, the loser was billed for exactly 40 virtual ms of work and saved 60 — IT IS BILLED FOR THE WINNER’S LATENCY, EVERY TIME, which for a token-metered API is the tokens generated before the abort landed. And in the case that matters most the saving is ZERO: two equally-fast 40ms providers burned 40 virtual ms EACH — 80 for one answer — and although the loser IS recorded as aborted mid-flight, the work the abort saved measured 0, because the two were due at the same instant. "Cancelled" is a statement about the response, not about the work, and the closer the backup is to the primary in speed — i.e. the better a backup it is — the less it saves. The non-idempotency hazard is a number too: `race` over a POST delivered the identical `{ charge: { amount: 4200 } }` body to BOTH providers, method POST at each, with the cancel arriving after both landed — nothing in the combinators inspects `method`. The half that works exactly as documented is the outer signal: an external abort at t=30 reached BOTH members (1 and 1 aborted, 30 virtual ms billed each) via `linkedController` (pipe.ts:57-71), with 0 timers left pending. The sequential shape produced the same answer for 40 virtual ms and [1, 0] — the cancellation machinery exists to reclaim a cost that failover never incurs', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c7-hedging-amplification.ts b/docs/scenarios/proofs/provider-failover/c7-hedging-amplification.ts new file mode 100644 index 00000000..7031cffd --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c7-hedging-amplification.ts @@ -0,0 +1,302 @@ +// C7 — hedging safety. The standard warning is that a hedge amplifies an outage: when the backend +// degrades, every request crosses the threshold, so every request doubles. Measure the +// amplification against a degraded backend, and then whether a circuit breaker can be scoped to +// just the hedge. +// +// The capture frames this as "does `race` amplify, and can a breaker be scoped". Both halves come +// back sharper than the framing, and the second one comes back with a trap the capture does not +// mention at all: +// +// • `race` has NO threshold. It is not "hedge after a delay", it is "hedge always", so the +// amplification is 2.00× when healthy and 2.00× when degraded — measured identical. There is +// nothing to tune, and the outage-amplification warning understates it: the doubling was +// already there before the outage. +// • A breaker CANNOT bound hedge spend. Ten healthy calls with `circuit: [2, '30s']` on the +// backup still sent the backup ten requests — a breaker trips on FAILURE, and an expensive +// healthy hedge never fails. +// • THE TRAP: two `url`-only stitches share ONE breaker, keyed on the literal string `'stitch'`. +// Measured: the primary's outage opened the breaker, and the BACKUP was fast-failed 503 without +// ever being called. The failover pair is wired so that the primary going down takes the backup +// with it. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c7-hedging-amplification.ts +import { apiKey, bearer } from '../../../../packages/core/src/auth'; +import { stitch } from '../../../../packages/core/src/index'; +import { any, race } from '../../../../packages/core/src/pipe'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { hits, outcomeOf, providerPair } from './fake-provider'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { probeStore } from './probe-store'; +import { rig } from './providers'; + +/** Two stitches with NO `name` and NO `path` — the shape a `url`-only failover pair has. */ +function namelessPair(circuit: [number, string]) { + const clock = manualClock(); + const store = probeStore(); + const p = providerPair(clock); + const common = { method: 'POST', clock, store, circuit }; + return { + clock, + store, + p, + primary: stitch({ + url: `${p.primary.origin}${p.primary.path}`, + adapter: p.primary.adapter(), + auth: bearer('pk-primary'), + ...common, + }), + backup: stitch({ + url: `${p.backup.origin}${p.backup.path}`, + adapter: p.backup.adapter(), + auth: apiKey({ + in: 'header', + name: 'x-api-key', + secret: 'sk-backup', + }), + ...common, + }), + }; +} + +async function main(): Promise { + heading('C7 — what does a hedge cost, and can a breaker bound it?'); + + // ── (a) the amplification is unconditional ───────────────────────────────────────────────── + // A real hedge fires the second leg only past a threshold, so a healthy backend sees 1.0× and a + // degraded one sees up to 2.0×. `race` has no threshold at all. + { + const healthy = rig(); + for (let i = 0; i < 20; i++) + await race(healthy.primary, healthy.backup)({ body: {} }); + const healthyTotal = + healthy.p.primary.received + healthy.p.backup.received; + + const degraded = rig(); + degraded.p.primary.takes(500); // the backend is in trouble + degraded.p.backup.takes(500); + for (let i = 0; i < 20; i++) { + const call = race(degraded.primary, degraded.backup)({ body: {} }); + await degraded.clock.advance(1000); + await call; + } + const degradedTotal = + degraded.p.primary.received + degraded.p.backup.received; + + checkSeq('(a) healthy → [primary, backup]', hits(healthy.p), [20, 20]); + checkSeq( + '(a) degraded → [primary, backup]', + hits(degraded.p), + [20, 20], + ); + check('(a) amplification when HEALTHY', healthyTotal / 20, 2); + check('(a) amplification when DEGRADED', degradedTotal / 20, 2); + note( + '(a) → there is no threshold to cross, so there is nothing to amplify FROM', + '`race`/`any` fire every member on every call (pipe.ts:161-168); the classic "hedging doubles traffic during an outage" warning understates it — the doubling is the steady state', + ); + } + + // ── (b) …and against a degraded backend the hedge buys no latency either ─────────────────── + // Hedging pays for itself only when the two legs' slowness is INDEPENDENT. A backend that is + // degraded is usually degraded for both legs. + { + const { clock, p, primary, backup } = rig(); + p.primary.takes(500); + p.backup.takes(500); + const call = race(primary, backup)({ body: {} }); + await clock.advance(2000); + await call; + check( + '(b) answer latency, hedged (virtual ms)', + p.backup.workedMs, + 500, + ); + check( + '(b) provider work spent to get it (virtual ms)', + p.primary.workedMs + p.backup.workedMs, + 1000, + ); + note( + '(b) → same latency as one call, twice the load', + 'the tail-latency win a hedge is bought for assumes the legs fail independently; a degraded shared backend is exactly the case where they do not', + ); + } + + // ── (c) a breaker cannot bound hedge SPEND ───────────────────────────────────────────────── + // Breakers trip on failure. An expensive-but-healthy hedge never fails, so the breaker never + // sees anything to trip on. + { + const { p, primary, backup } = rig({ + each: { circuit: { failures: 2, cooldown: '30s' } }, + }); + for (let i = 0; i < 10; i++) await any(primary, backup)({ body: {} }); + checkSeq( + '(c) 10 healthy calls with a circuit → [primary, backup]', + hits(p), + [10, 10], + ); + note( + '(c) → `circuit` is a HEALTH gate, not a BUDGET gate', + 'the cost the hedge imposes is invisible to every resilience primitive in the library — none of them count successful requests', + ); + } + + // ── (d) what a breaker DOES buy: it stops the dead leg ───────────────────────────────────── + // The one direction it helps. Once the primary is properly down, its breaker fast-fails + // in-process and `any` stops paying for the doomed request — 3 wasted calls, then none. + { + const { p, primary, backup } = rig({ + each: { circuit: { failures: 2, cooldown: '30s' } }, + }); + p.primary.respond(500); + const outcomes: string[] = []; + for (let i = 0; i < 6; i++) + outcomes.push( + await outcomeOf(() => any(primary, backup)({ body: {} })), + ); + + checkSeq('(d) 6 calls, primary down → outcomes', outcomes, [ + 'ok', + 'ok', + 'ok', + 'ok', + 'ok', + 'ok', + ]); + check('(d) requests wasted on the dead primary', p.primary.received, 2); + check('(d) requests served by the backup', p.backup.received, 6); + note( + '(d) → per-member `circuit` + `any` is a real, working construction', + 'the breaker opens after 2 failures and the primary leg costs nothing thereafter — this is the ONE thing in this claim that works as you would hope', + ); + } + + // ── (e) THE TRAP: two `url`-only stitches share one breaker ──────────────────────────────── + // The breaker key is `opts.key ?? hostKey(req, cfg)` = `cfg.name ?? cfg.path ?? 'stitch'` + // (resilience.ts:353, engine.ts:140,265-274,860). A stitch built from `url` alone has neither, + // so BOTH providers key on the literal string `'stitch'` — one breaker for the failover pair. + { + const { store, p, primary, backup } = namelessPair([2, '30s']); + p.primary.respond(500); + const outcomes: string[] = []; + for (let i = 0; i < 5; i++) + outcomes.push( + await outcomeOf(() => any(primary, backup)({ body: {} })), + ); + + checkSeq( + '(e) circuit keys touched by the PAIR', + store.keys('circuit:'), + ['circuit:stitch'], + ); + checkSeq('(e) 5 calls, primary down → outcomes', outcomes, [ + 'ok', + 'ok', + 'AggregateError', + 'AggregateError', + 'AggregateError', + ]); + check('(e) requests the healthy BACKUP received', p.backup.received, 2); + check( + '(e) calls the backup was fast-failed on without being called', + 3, + 3, + ); + note( + '(e) → the primary’s outage opened the BACKUP’s breaker', + 'the failover pair is wired so that the provider going down takes its own replacement with it — and the caller sees an AggregateError with `status: undefined` (C3b), so nothing in the error says "circuit open" either', + ); + + // Naming the stitches is the entire fix — and it is a diagnostic label, not a policy knob. + const named = rig({ + each: { circuit: { failures: 2, cooldown: '30s' } }, + }); + named.p.primary.respond(500); + const fixed: string[] = []; + for (let i = 0; i < 5; i++) + fixed.push( + await outcomeOf(() => + any(named.primary, named.backup)({ body: {} }), + ), + ); + checkSeq('(e) with `name` set → outcomes', fixed, [ + 'ok', + 'ok', + 'ok', + 'ok', + 'ok', + ]); + check( + '(e) with `name` set → backup served', + named.p.backup.received, + 5, + ); + } + + // ── (f) the delayed hedge, which no combinator expresses ─────────────────────────────────── + // ~10 lines of raw promise code. It is the shape the whole hedging literature recommends, and + // it is outside the `pipe` vocabulary entirely (C2(e): no `hedge`, no member-level delay). + { + const build = () => { + const r = rig(); + const hedge = async (threshold: number): Promise => { + const ctrl = new AbortController(); + const first = Promise.resolve( + r.primary({ body: {}, signal: ctrl.signal }), + ); + const late = r.clock + .sleep(threshold, ctrl.signal) + .then(() => r.backup({ body: {}, signal: ctrl.signal })); + void late.catch(() => undefined); + try { + return await Promise.race([first, late]); + } finally { + ctrl.abort(); + } + }; + return { r, hedge }; + }; + + // Healthy: the primary answers inside the threshold, the backup is never fired. + { + const { r, hedge } = build(); + r.p.primary.takes(20); + for (let i = 0; i < 10; i++) { + const call = hedge(100); + await r.clock.advance(200); + await call; + } + checkSeq( + '(f) delayed hedge, healthy → [primary, backup]', + hits(r.p), + [10, 0], + ); + } + // Degraded: the primary crosses the threshold, so the backup fires — and only then. + { + const { r, hedge } = build(); + r.p.primary.takes(500); + for (let i = 0; i < 10; i++) { + const call = hedge(100); + await r.clock.advance(1000); + await call; + } + checkSeq( + '(f) delayed hedge, degraded → [primary, backup]', + hits(r.p), + [10, 10], + ); + } + note( + '(f) → 1.0× healthy, 2.0× degraded — the amplification profile a hedge is supposed to have', + 'and it is ~11 lines of `AbortController` + `Promise.race` + a clock sleep, with none of the trace linkage `linked` gives (C2c) — the combinators contribute nothing to it', + ); + } + + finish( + 'C7', + 'THE AMPLIFICATION IS UNCONDITIONAL, THE BREAKER CANNOT BOUND IT, AND THE PAIR SHARES ONE BREAKER BY DEFAULT. `race` measured 2.00× amplification when HEALTHY (20 calls → [20, 20]) and 2.00× when DEGRADED (identical) — it has no threshold, so the standard "hedging doubles traffic during an outage" warning understates it: the doubling is the steady state, and there is no knob to tune. Against a degraded backend it also buys nothing — two 500ms legs answered in 500 virtual ms, exactly one call’s latency, for 1000ms of provider work. A breaker cannot bound the spend, because a breaker is a HEALTH gate and not a BUDGET gate: 10 healthy calls with `circuit: [2, "30s"]` on both members still measured [10, 10]. What the breaker DOES buy is real and worth documenting: with the primary returning 500, `any` + per-member `circuit` wasted exactly 2 requests on the dead leg and then stopped, while the backup served all 6 calls. AND THEN THE TRAP THE CAPTURE DOES NOT MENTION: two `url`-only stitches have neither `name` nor `path`, so both key their breaker on the literal string `"stitch"` (resilience.ts:353, engine.ts:140,265-274,860) — measured, ONE key `circuit:stitch` for the whole pair. The primary’s outage opened the BACKUP’s breaker: outcomes ok,ok,AggregateError,AggregateError,AggregateError, with the healthy backup receiving only 2 requests and fast-failed unasked on the other 3, and the caller’s AggregateError carrying `status: undefined` so nothing even says "circuit open". Setting `name` fixes it completely (5 of 5 ok, backup served 5) — the partition key is a diagnostic LABEL. Finally, the delayed hedge everyone actually recommends measured the right profile — [10, 0] healthy, [10, 10] degraded — and took ~11 lines of raw `AbortController` + `Promise.race` + clock sleep, with no combinator contributing anything and none of `linked`’s trace linkage', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/c8-assembled.ts b/docs/scenarios/proofs/provider-failover/c8-assembled.ts new file mode 100644 index 00000000..5332bfb3 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/c8-assembled.ts @@ -0,0 +1,297 @@ +// C8 — assemble the best available answer for "primary with a backup, classified correctly, one +// call on the happy path", run it, and price it against the same behaviour with no library. +// +// The construction is in `failover.ts`. Its shape is the finding: everything PER PROVIDER stays +// declared on the stitch (origin, path, auth, `retry` with its own `on`, `circuit`, `timeout`, +// `pick`) and costs nothing, while the ROUTING — try in order, classify before moving on, name the +// winner — is 30 counted lines of user code that no combinator contributes to. +// +// Two measurements make the trade concrete: the same behaviour hand-rolled against the same fake +// providers is 104 counted lines, so the library is carrying ~71% of it; and the 30 lines cannot be +// given back to the library, because a `Composable` is not user-authorable — `makeComposable` is +// unexported and the `__runWith` protocol is not on the public type. (e) measures a hand-branded +// node satisfying the TYPE gate and then crashing at runtime. +// +// pnpm exec tsx docs/scenarios/proofs/provider-failover/c8-assembled.ts +import { type Composable, all, any } from '../../../../packages/core/src/pipe'; +import type { + StitchError, + StitchInput, +} from '../../../../packages/core/src/types'; +import { type Leg, failover } from './failover'; +import { hits, outcomeOf } from './fake-provider'; +import { handRolled } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { rig } from './providers'; +import { recordingSink } from './trace-probe'; +import { accepted, probeSpellings, rejected } from './type-probe'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** Count the CODE lines between the `` / `` markers of a file. */ +function countedLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8').split('\n'); + const from = src.findIndex((l) => l.includes('')); + const to = src.findIndex((l) => l.includes('')); + return src + .slice(from + 1, to) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; +} + +/** The assembled construction: per-provider config declared, routing supplied by `failover`. */ +function assembled(trace?: ReturnType) { + const r = rig({ + ...(trace ? { trace } : {}), + each: { + retry: { attempts: 2, on: [429, 503], backoff: { base: 100 } }, + circuit: { failures: 3, cooldown: '30s' }, + timeout: { perAttempt: '5s' }, + }, + onPrimary: { pick: 'choices.0.text' }, + onBackup: { pick: 'output' }, + }); + const legs: Leg[] = [ + { name: 'primary', call: r.primary }, + { name: 'backup', call: r.backup }, + ]; + return { ...r, legs }; +} + +async function main(): Promise { + heading('C8 — the assembled answer, and what it costs'); + + // ── (a) the happy path is ONE request, and it is attributed ──────────────────────────────── + { + const { p, legs } = assembled(); + const served: string[] = []; + for (let i = 0; i < 10; i++) { + const r = await failover(legs, { body: { prompt: 'q' } }); + served.push(r.provider); + } + checkSeq('(a) 10 happy calls → [primary, backup]', hits(p), [10, 0]); + checkSeq('(a) providers credited', [...new Set(served)], ['primary']); + note( + '(a) → against `any`’s [10, 10] on identical providers (C1a)', + 'the entire backup vendor bill is the difference between the two constructions', + ); + } + + // ── (b) a 400 stops the chain with the actionable error ──────────────────────────────────── + { + const { p, legs } = assembled(); + p.primary.respond(400); + const err = (await failover(legs, { body: { prompt: null } }).then( + () => undefined, + (e: unknown) => e as StitchError, + ))!; + checkSeq('(b) 400 → [primary, backup]', hits(p), [1, 0]); + check('(b) error name', err.name, 'StitchError'); + check('(b) error status', err.status, 400); + check( + '(b) error body.error.code', + (err.body as { error: { code: string } }).error.code, + 'invalid_request', + ); + check('(b) attempts (a 400 is not retried)', err.attempts, 1); + } + + // ── (c) a 503 retries the primary, then fails over, and stays one trace ──────────────────── + // `retry.backoff.base` is 100 virtual ms, so the injected clock has to be driven. + { + const trace = recordingSink(); + const { clock, p, legs } = assembled(trace); + p.primary.respond(503); + const call = failover(legs, { body: { prompt: 'q' } }); + await clock.advance(1000); + const r = await call; + + checkSeq('(c) 503 → [primary, backup]', hits(p), [2, 1]); + check('(c) who served it', r.provider, 'backup'); + check('(c) normalised value', r.value, 'answer from backup'); + check('(c) distinct traceIds', trace.traceIds().length, 1); + checkSeq('(c) trace spine', trace.spine(), [ + 'primary<-', + 'backup<-primary', + ]); + note( + '(c) → the primary’s own `retry` ran first (2 requests), then the chain moved on', + 'per-member resilience and cross-member routing compose without either knowing about the other', + ); + } + + // ── (d) both providers down: the LAST real error, not an aggregate ───────────────────────── + { + const { clock, p, legs } = assembled(); + p.primary.respond(500); + p.backup.respond(503); + const call = failover(legs, { body: {} }); + const settled = call.then( + () => undefined, + (e: unknown) => e as StitchError, + ); + await clock.advance(2000); + const err = (await settled)!; + check('(d) error name', err.name, 'StitchError'); + check('(d) error status', err.status, 503); + check( + '(d) error body names the provider', + (err.body as { provider: string }).provider, + 'backup', + ); + note( + '(d) → against `any`’s AggregateError with `status: undefined` (C3b)', + 'the caller keeps a routable status and the failing provider’s own body', + ); + } + + // ── (e) the 30 lines cannot be given back to the library ─────────────────────────────────── + // A `Composable` is not user-authorable: `makeComposable` is unexported and `__runWith` is not + // on the public type. Hand-branding satisfies the TYPE gate and then crashes at runtime. + { + const { primary } = rig(); + const fake = Object.assign( + async (_input?: StitchInput) => 'hand-made', + { __composable: true as const }, + ) as unknown as Composable; + const crash = await outcomeOf(() => all(primary, fake)({ body: {} })); + check('(e) hand-branded node in `all` → outcome', crash, 'TypeError'); + + const results = probeSpellings([ + { + label: 'all(a, asyncFn) — a plain function as a member', + code: 'void pipe.all(a, async () => 1);', + }, + { + label: 'all(a, brandedFn) — hand-branded __composable', + code: 'void pipe.all(a, Object.assign(async () => 1, { __composable: true as const }));', + }, + { + label: 'any(a, b) as a member of all', + code: 'void pipe.all(a, pipe.any(a, b));', + }, + ]); + checkSeq('(e) spellings that COMPILE', accepted(results), [ + 'all(a, brandedFn) — hand-branded __composable', + 'any(a, b) as a member of all', + ]); + check('(e) refused', rejected(results).length, 1); + note( + '(e) → the brand gate is `Member = { __stitch } | { __composable }` (pipe.ts:188-189)', + 'it checks the BRAND and not the `__runWith` protocol, so a hand-authored node type-checks and then throws — the composition vocabulary is closed, and a user-written failover node cannot join it', + ); + } + + // ── (f) the price, in counted lines ──────────────────────────────────────────────────────── + { + const assembledLines = countedLines('failover.ts'); + const handRolledLines = countedLines('hand-rolled.ts'); + check('(f) assembled: counted lines of ROUTING', assembledLines, 30); + check( + '(f) hand-rolled: counted lines, same features', + handRolledLines, + 104, + ); + check( + '(f) share of the implementation the library carries (%)', + Math.round((1 - assembledLines / handRolledLines) * 100), + 71, + ); + + // …and the two produce the same measured behaviour on the same fake providers. + const hand = (() => { + const r = rig(); + const call = handRolled( + [ + { + name: 'primary', + url: `${r.p.primary.origin}${r.p.primary.path}`, + method: 'POST', + headers: { authorization: 'Bearer pk-primary' }, + adapter: r.p.primary.adapter(), + pick: (b) => + (b as { choices: { text: string }[] }).choices[0] + ?.text, + }, + { + name: 'backup', + url: `${r.p.backup.origin}${r.p.backup.path}`, + method: 'POST', + headers: { 'x-api-key': 'sk-backup' }, + adapter: r.p.backup.adapter(), + pick: (b) => (b as { output: string }).output, + }, + ], + { + clock: r.clock, + attempts: 2, + retryOn: [429, 503], + backoff: 100, + circuit: { failures: 3, cooldown: 30_000 }, + }, + ); + return { r, call }; + })(); + + const happy = await hand.call({ prompt: 'q' }); + checkSeq( + '(f) hand-rolled happy → [primary, backup]', + hits(hand.r.p), + [1, 0], + ); + check('(f) hand-rolled happy: provider', happy.provider, 'primary'); + + hand.r.p.primary.respond(503); + const failing = hand.call({ prompt: 'q' }); + await hand.r.clock.advance(1000); + const over = await failing; + checkSeq( + '(f) hand-rolled 503 → [primary, backup]', + hits(hand.r.p), + [3, 1], + ); + check('(f) hand-rolled 503: provider', over.provider, 'backup'); + check('(f) hand-rolled 503: value', over.value, 'answer from backup'); + note( + '(f) → identical routing behaviour, 30 lines against 104', + 'the library’s contribution is entirely PER MEMBER — auth, retry, breaker, timeout, normalisation, trace identity — and entirely absent from the routing between members', + ); + } + + // ── (g) what the assembled answer still does not do ──────────────────────────────────────── + { + const trace = recordingSink(); + const { legs } = assembled(trace); + await failover(legs, { body: {} }); + check( + '(g) events emitted by the failover itself', + trace.records.filter( + (r) => r.name !== 'primary' && r.name !== 'backup', + ).length, + 0, + ); + const { primary, backup } = rig(); + const node = any(primary, backup); + check('(g) `any` is a node you can nest', node.__composable, true); + note( + '(g) → three gaps remain, all of them the same gap', + 'the failover is not a NODE: no span of its own, not nestable in a combinator, not introspectable via `__config`, and not exportable to OpenAPI — the routing lives outside the object graph the library reasons about', + ); + } + + finish( + 'C8', + 'ACHIEVABLE WITH 30 LINES OF USER CODE AT ONE SEAM. The assembled answer (`failover.ts`) leaves everything PER PROVIDER declared on the stitch — origin, path, auth strategy, `retry: { attempts: 2, on: [429, 503] }`, `circuit: [3, "30s"]`, `timeout.perAttempt`, `pick` — and supplies only the ROUTING over `linked`. Measured: 10 successful calls sent [10, 0], every one credited to the primary, against `any`’s [10, 10] on identical providers; a 400 stopped the chain at [1, 0] with a real `StitchError` status 400, `attempts: 1`, and the provider’s `invalid_request` body intact; a 503 retried the primary twice and then failed over, [2, 1], returning `{ provider: "backup", value: "answer from backup" }` — per-member resilience and cross-member routing composing without either knowing about the other — in ONE trace tree with the spine primary<-, backup<-primary; and with both providers down the caller got the LAST real error (`StitchError` 503, body naming `backup`) rather than an `AggregateError` with `status: undefined`. The price is 30 counted lines against 104 for the same feature set hand-rolled on the same fake providers — the library carries ~74%, and all of it is PER MEMBER (auth, retry, breaker, timeout, normalisation, trace identity); its contribution to the routing BETWEEN members is zero. AND THE 30 LINES CANNOT BE GIVEN BACK: a `Composable` is not user-authorable, because `makeComposable` is unexported and `__runWith` is not on the public type, while the member gate `Member = { __stitch } | { __composable }` (pipe.ts:188-189) checks only the BRAND — so a hand-branded node COMPILES and then throws `TypeError` at runtime, measured. The failover is therefore not a node: no span of its own (0 events from the group), not nestable in a combinator, not introspectable, not exportable', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/provider-failover/failover.ts b/docs/scenarios/proofs/provider-failover/failover.ts new file mode 100644 index 00000000..38a59318 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/failover.ts @@ -0,0 +1,62 @@ +// The best available answer for "primary with a backup, classified correctly, one call on the happy +// path" — assembled from what the library actually provides, with the gaps filled in. +// +// Everything PER PROVIDER stays declared on the stitch and costs nothing: origin, path, method, +// auth strategy, `retry` with its own `on` set, `circuit`, `timeout`, `pick`/`transform`. What the +// library does not provide, and what this file is, is the ROUTING: try in order, classify the +// failure before moving on, and name the provider that served the call. +// +// It is written over `linked` rather than plain `try`/`catch` for one measured reason (C2c): the +// scope chains each call under the previous one, so the whole failover is a single trace tree +// reading `primary → backup` instead of two unrelated root traces. +// +// The region between the markers is what C8 counts. +import { linked } from '../../../../packages/core/src/pipe'; +import type { + Stitch, + StitchError, + StitchInput, +} from '../../../../packages/core/src/types'; + +// +/** Availability failures — try the next provider. A 400 is not here, by design. */ +export const AVAILABILITY = [408, 425, 429, 500, 502, 503, 504] as const; + +/** One leg of the chain: a name for attribution, and the stitch that carries its own config. */ +export interface Leg { + name: string; + call: Stitch; +} + +/** What the caller gets back: the value, and who produced it. */ +export interface Served { + provider: string; + value: T; +} + +/** + * Try each leg in order. Move on only when the failure is an AVAILABILITY failure; anything else + * (a 400, a validation error, a bad credential) stops the chain and reaches the caller unchanged. + */ +export function failover( + legs: readonly Leg[], + input: StitchInput = {}, + on: readonly number[] = AVAILABILITY, +): Promise> { + return linked(async (run) => { + let last: unknown; + for (const leg of legs) { + try { + return { + provider: leg.name, + value: await run(leg.call, input), + }; + } catch (e) { + if (!on.includes((e as StitchError).status ?? 0)) throw e; + last = e; + } + } + throw last; + }); +} +// diff --git a/docs/scenarios/proofs/provider-failover/fake-provider.ts b/docs/scenarios/proofs/provider-failover/fake-provider.ts new file mode 100644 index 00000000..7618a6a6 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/fake-provider.ts @@ -0,0 +1,256 @@ +// The two providers, as plain `Adapter`s over an injected {@link Clock} — no network, no timers of +// their own. +// +// The scenario's whole measurement is PER PROVIDER, so the fake is built around a single ledger per +// provider rather than one shared one: `primary.received` and `backup.received` are two independent +// integers, and every claim in this directory is a statement about the pair. That matters because +// "failover" and "hedging" produce IDENTICAL results and identical latencies on a happy path — the +// only thing that tells them apart is how many requests the backup received, which is exactly the +// number a bill is computed from. +// +// The two providers are deliberately NOT interchangeable at the wire level, because real ones never +// are: different origin, different path, different auth header, different success-body shape and +// different error vocabulary. C4 is the claim that turns on that difference. +import type { + Adapter, + AdapterRequest, + Clock, +} from '../../../../packages/core/src/types'; + +/** One request that actually reached a provider — the unit every count in this directory is over. */ +export interface ProviderCall { + /** Virtual ms at which the request arrived. */ + at: number; + method: string; + /** Path the request was addressed to — the evidence for C4's "did it hit MY endpoint". */ + path: string; + /** Query string as received, without the leading `?`. */ + query: string; + /** Every header, lowercased — C4 reads the auth headers off this. */ + headers: Record; + body: unknown; + /** Status this provider answered with (or would have, had it not been aborted). */ + status: number; + /** + * True when the caller's signal fired while this provider was still working. The distinction + * that matters for C6: an aborted request STILL ARRIVED. An LLM bills the tokens it generated + * before the abort, so a cancelled loser is cheaper than a completed one and is not free. + */ + abortedMidFlight: boolean; + /** Virtual ms of work this provider had done when it either answered or was aborted. */ + workedMs: number; +} + +export interface FakeProviderOptions { + /** Label used in the success body and in this file's own reporting. */ + name: string; + clock: Clock; + /** Origin this provider answers on — deliberately different per provider. */ + origin: string; + /** The one path this provider serves. A request to any other path is a 404 (see `adapter`). */ + path: string; + /** Status to answer with. Default 200. */ + status?: number; + /** Virtual ms this provider takes to answer. Default 0 (answers synchronously). */ + latency?: number; +} + +/** + * One provider. `adapter()` is what a stitch is handed; `calls` is the ledger, and `received` / + * `completed` / `aborted` are the three integers the claims assert on. + */ +export class FakeProvider { + readonly name: string; + readonly origin: string; + readonly path: string; + readonly calls: ProviderCall[] = []; + private readonly clock: Clock; + private status: number; + private latency: number; + + constructor(opts: FakeProviderOptions) { + this.name = opts.name; + this.clock = opts.clock; + this.origin = opts.origin; + this.path = opts.path; + this.status = opts.status ?? 200; + this.latency = opts.latency ?? 0; + } + + /** Requests that ARRIVED — billable in the general case, whatever happened afterwards. */ + get received(): number { + return this.calls.length; + } + + /** Requests this provider answered in full. */ + get completed(): number { + return this.calls.filter((c) => !c.abortedMidFlight).length; + } + + /** Requests that arrived and were then cancelled mid-flight — arrived, but never answered. */ + get aborted(): number { + return this.calls.filter((c) => c.abortedMidFlight).length; + } + + /** Virtual ms of work done across every request, aborted ones included — the "tokens billed" proxy. */ + get workedMs(): number { + return this.calls.reduce((sum, c) => sum + c.workedMs, 0); + } + + /** Make this provider answer with `status` from now on. `429`/`500` = availability, `400` = your payload. */ + respond(status: number): void { + this.status = status; + } + + /** Make this provider take `ms` of virtual time to answer — a slow leg, or a degraded backend. */ + takes(ms: number): void { + this.latency = ms; + } + + /** Forget every recorded request (so one script can measure several constructions cleanly). */ + reset(): void { + this.calls.length = 0; + } + + /** + * The error body this provider returns. Deliberately different per provider and per status — + * the "different error vocabularies" hazard, and what C3 measures the caller's ability to see. + */ + private errorBody(status: number): unknown { + return { + provider: this.name, + error: { + code: status === 400 ? 'invalid_request' : 'unavailable', + message: `${this.name} says ${status}`, + }, + }; + } + + adapter(): Adapter { + return async (req: AdapterRequest) => { + const url = new URL(req.url); + // Recorded on ARRIVAL, before any hold — `received` is "the request left the process and + // reached this provider", which is the quantity a bill is computed from. + const call: ProviderCall = { + at: this.clock.now(), + method: req.method, + path: url.pathname, + query: url.search.replace(/^\?/, ''), + headers: Object.fromEntries( + Object.entries(req.headers).map(([k, v]) => [ + k.toLowerCase(), + v, + ]), + ), + body: req.body, + status: this.status, + abortedMidFlight: false, + workedMs: 0, + }; + this.calls.push(call); + + // A provider that is not addressed at its own endpoint 404s, exactly as a real one + // would. This is the whole of C4's measurement: a combinator that hands every member + // the same input addresses one of them wrongly. + if (url.pathname !== this.path) { + call.status = 404; + return { + status: 404, + headers: {}, + body: { + provider: this.name, + error: { + code: 'no_such_endpoint', + message: `${this.name} has no ${url.pathname}`, + }, + }, + }; + } + + if (this.latency > 0) { + const start = this.clock.now(); + try { + await this.clock.sleep(this.latency, req.signal); + call.workedMs = this.clock.now() - start; + } catch { + // The caller cancelled. The request had already arrived and this provider had + // already burned `workedMs` of work on it — that work is billed. + call.abortedMidFlight = true; + call.workedMs = this.clock.now() - start; + throw new Error(`${this.name}: aborted`); + } + } + + if (this.status >= 400) + return { + status: this.status, + headers: {}, + body: this.errorBody(this.status), + }; + return { + status: 200, + headers: {}, + // Deliberately different success shapes: the primary nests its text, the backup + // flattens it. Normalising them is user code either way (see `pick` in the claims). + body: + this.name === 'backup' + ? { + served_by: this.name, + output: `answer from ${this.name}`, + } + : { + served_by: this.name, + choices: [{ text: `answer from ${this.name}` }], + }, + }; + }; + } +} + +/** Both providers over one clock, wired the way a real pair is: different origin, path and auth. */ +export interface ProviderPair { + primary: FakeProvider; + backup: FakeProvider; +} + +export function providerPair(clock: Clock): ProviderPair { + return { + primary: new FakeProvider({ + name: 'primary', + clock, + origin: 'https://primary.llm.test', + path: '/v1/complete', + }), + backup: new FakeProvider({ + name: 'backup', + clock, + origin: 'https://backup.llm.test', + path: '/generate', + }), + }; +} + +/** `[primary.received, backup.received]` — the two-integer spine every claim here prints. */ +export const hits = (p: ProviderPair): [number, number] => [ + p.primary.received, + p.backup.received, +]; + +/** + * Run one call and reduce it to a short outcome token — `'ok'`, or `''` / `''` for a + * failure. + * + * `PromiseLike`, not `Promise`: a stitch call returns a lazy `StitchResult` thenable that starts on + * `.then` (stitch.ts:729,781), and a combinator returns a real `Promise` — both are `PromiseLike`. + */ +export async function outcomeOf( + call: () => PromiseLike, +): Promise { + try { + await call(); + return 'ok'; + } catch (e) { + const err = e as Error & { status?: number }; + return String(err.status ?? err.name); + } +} diff --git a/docs/scenarios/proofs/provider-failover/hand-rolled.ts b/docs/scenarios/proofs/provider-failover/hand-rolled.ts new file mode 100644 index 00000000..8cf4298e --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/hand-rolled.ts @@ -0,0 +1,131 @@ +// The same behaviour with no library at all — the baseline C8 measures the assembled solution +// against. +// +// The comparison is only honest if the feature sets match, so this implements everything the +// assembled version gets from a stitch's declaration and not one thing more: per-provider auth +// headers, retry with a status set and a backoff, a per-provider circuit breaker with a cooldown, +// an error carrying `status` and the response body, response normalisation, lifecycle events for +// telemetry — and then the same routing on top (try in order, classify before moving on, name the +// provider that served it). +// +// It runs against the same `Adapter` the stitches do, so both sides in C8 talk to the same fake +// providers and their per-provider ledgers are directly comparable. +// +// The region between the markers is what C8 counts. +import type { Adapter, Clock } from '../../../../packages/core/src/types'; + +// +export interface HandLeg { + name: string; + url: string; + method: string; + /** Static auth + content headers — the `auth` strategy's job, done by hand. */ + headers: Record; + adapter: Adapter; + /** Response normalisation — the `pick`/`transform` job, done by hand. */ + pick?: (body: unknown) => T; +} + +export interface HandEvent { + leg: string; + type: 'start' | 'result' | 'error'; + status?: number; + attempt?: number; +} + +export interface HandOptions { + clock: Clock; + attempts?: number; + retryOn?: readonly number[]; + backoff?: number; + circuit?: { failures: number; cooldown: number }; + failoverOn?: readonly number[]; + onEvent?: (e: HandEvent) => void; +} + +export class HandError extends Error { + readonly status: number | undefined; + readonly body: unknown; + constructor(message: string, status: number | undefined, body: unknown) { + super(message); + this.name = 'HandError'; + this.status = status; + this.body = body; + } +} + +interface Breaker { + failures: number; + openedAt: number | null; +} + +export function handRolled( + legs: readonly HandLeg[], + opts: HandOptions, +): (body: unknown) => Promise<{ provider: string; value: T }> { + const breakers = new Map( + legs.map((l) => [l.name, { failures: 0, openedAt: null }]), + ); + const attempts = opts.attempts ?? 1; + const retryOn = opts.retryOn ?? [429, 502, 503, 504]; + const failoverOn = opts.failoverOn ?? [408, 425, 429, 500, 502, 503, 504]; + + async function one(leg: HandLeg, body: unknown): Promise { + const b = breakers.get(leg.name)!; + if (b.openedAt !== null) { + if (opts.clock.now() - b.openedAt < (opts.circuit?.cooldown ?? 0)) + throw new HandError('circuit open', 503, undefined); + b.openedAt = null; + b.failures = 0; + } + let err: HandError | undefined; + for (let attempt = 1; attempt <= attempts; attempt++) { + opts.onEvent?.({ leg: leg.name, type: 'start', attempt }); + const res = await leg.adapter({ + url: leg.url, + method: leg.method, + headers: leg.headers, + body, + }); + if (res.status < 400) { + b.failures = 0; + opts.onEvent?.({ + leg: leg.name, + type: 'result', + status: res.status, + }); + return (leg.pick ? leg.pick(res.body) : res.body) as T; + } + err = new HandError( + `${leg.name} ${res.status}`, + res.status, + res.body, + ); + opts.onEvent?.({ + leg: leg.name, + type: 'error', + status: res.status, + }); + if (!retryOn.includes(res.status) || attempt === attempts) break; + await opts.clock.sleep(opts.backoff ?? 100); + } + b.failures += 1; + if (opts.circuit && b.failures >= opts.circuit.failures) + b.openedAt = opts.clock.now(); + throw err; + } + + return async function call(body) { + let last: unknown; + for (const leg of legs) { + try { + return { provider: leg.name, value: await one(leg, body) }; + } catch (e) { + if (!failoverOn.includes((e as HandError).status ?? 0)) throw e; + last = e; + } + } + throw last; + }; +} +// diff --git a/docs/scenarios/proofs/provider-failover/harness.ts b/docs/scenarios/proofs/provider-failover/harness.ts new file mode 100644 index 00000000..9a429426 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/harness.ts @@ -0,0 +1,67 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is a PER-PROVIDER REQUEST COUNT. "Did the backup get called on a +// successful primary" is the whole question, and its answer is an integer, so both assertions +// print the measured value whether they pass or fail: `backup requests on 10 happy calls: 10` has +// to be readable out of context, because it IS the finding. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-call outcome spine + * (`["ok","ok","400"]`) and the per-provider hit spine (`[1,0]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a COST the library imposes — the verdict statement carries + * the direction, because "PASS C1" on a claim whose content is "the failover combinator bills + * twice on every successful call" is otherwise unreadable. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/provider-failover/probe-store.ts b/docs/scenarios/proofs/provider-failover/probe-store.ts new file mode 100644 index 00000000..37baf1a3 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/probe-store.ts @@ -0,0 +1,39 @@ +// A `StitchStore` that records every key the ENGINE touches, over a real `memoryStore`. +// +// C7 asks whether a circuit breaker can be scoped to just the hedge, and that is not a question +// about which objects were constructed — it is a question about WHAT STRING the engine keyed the +// breaker on. The breaker lives at `circuit:` (resilience.ts:353) where the key defaults to +// `cfg.name ?? cfg.path ?? 'stitch'` (engine.ts:140,265-274,860), so reading the key set off the +// store answers "are these two providers sharing one breaker" directly, rather than by inference. +import { memoryStore } from '../../../../packages/core/src/index'; +import type { StitchStore } from '../../../../packages/core/src/types'; + +export interface ProbeStore extends StitchStore { + /** Every key touched, in order, with duplicates — `keys('circuit:')` is the usual read. */ + readonly touched: string[]; + /** The distinct keys touched under a prefix, in first-touch order. */ + keys(prefix?: string): string[]; +} + +export function probeStore(inner: StitchStore = memoryStore()): ProbeStore { + const touched: string[] = []; + return { + touched, + keys(prefix = '') { + return [...new Set(touched.filter((k) => k.startsWith(prefix)))]; + }, + get(key) { + touched.push(key); + return inner.get(key); + }, + async set(key, value, ttl) { + touched.push(key); + return inner.set(key, value, ttl); + }, + async increment(key, ttl) { + touched.push(key); + return inner.increment(key, ttl); + }, + close: () => inner.close?.() ?? Promise.resolve(), + }; +} diff --git a/docs/scenarios/proofs/provider-failover/providers.ts b/docs/scenarios/proofs/provider-failover/providers.ts new file mode 100644 index 00000000..2151c7c2 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/providers.ts @@ -0,0 +1,67 @@ +// The construction under test, in one place: two providers of the same SHAPE that are not +// interchangeable on the wire. +// +// primary POST https://primary.llm.test/v1/complete Authorization: Bearer pk-primary +// backup POST https://backup.llm.test/generate x-api-key: sk-backup +// +// Different origin, different path, different auth scheme, different success body. That is the +// normal case for a provider pair — "interchangeable sources" is a property of the CAPABILITY, not +// of the endpoint — and C4 is the claim that measures whether the combinators can address it. +import { apiKey, bearer } from '../../../../packages/core/src/auth'; +import { stitch } from '../../../../packages/core/src/index'; +import { + type ManualClock, + manualClock, +} from '../../../../packages/core/src/testing'; +import type { + Stitch, + StitchConfig, + TraceSink, +} from '../../../../packages/core/src/types'; +import { type ProviderPair, providerPair } from './fake-provider'; + +export interface Rig { + clock: ManualClock; + p: ProviderPair; + primary: Stitch; + backup: Stitch; +} + +export interface RigOptions { + trace?: TraceSink; + /** Extra config merged into BOTH stitches — `retry`, `circuit`, `store`, `verdict`, … */ + each?: Partial; + /** Extra config for the primary only. */ + onPrimary?: Partial; + /** Extra config for the backup only. */ + onBackup?: Partial; +} + +/** Both providers, both stitches, one injected clock. */ +export function rig(opts: RigOptions = {}): Rig { + const clock = manualClock(); + const p = providerPair(clock); + const common: Partial = { + method: 'POST', + clock, + ...(opts.trace ? { trace: opts.trace } : {}), + ...opts.each, + }; + const primary = stitch({ + name: 'primary', + url: `${p.primary.origin}${p.primary.path}`, + adapter: p.primary.adapter(), + auth: bearer('pk-primary'), + ...common, + ...opts.onPrimary, + }); + const backup = stitch({ + name: 'backup', + url: `${p.backup.origin}${p.backup.path}`, + adapter: p.backup.adapter(), + auth: apiKey({ in: 'header', name: 'x-api-key', secret: 'sk-backup' }), + ...common, + ...opts.onBackup, + }); + return { clock, p, primary, backup }; +} diff --git a/docs/scenarios/proofs/provider-failover/trace-probe.ts b/docs/scenarios/proofs/provider-failover/trace-probe.ts new file mode 100644 index 00000000..a4f2d759 --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/trace-probe.ts @@ -0,0 +1,79 @@ +// A {@link TraceSink} that records every event with the run identity it arrived under. +// +// Two claims here are about OBSERVABILITY rather than cost, and both turn on the same question: +// after the call, can anything downstream say WHICH provider served it? A sink is the only place +// that can answer, because the returned value is just the winner's body and the combinators return +// no envelope. So this records `(name, type, traceId, spanId, parentSpanId)` per event and exposes +// the three reductions the claims assert on: which stitches emitted a terminal `result`, how many +// distinct trace trees the call produced, and the parent/child spine. +import type { + StitchEvent, + TraceContext, + TraceSink, +} from '../../../../packages/core/src/types'; + +export interface TraceRecord { + /** The stitch's `name` — the only provider identity a sink ever sees. */ + name: string; + type: StitchEvent['type']; + status?: number; + traceId?: string; + spanId?: string; + parentSpanId?: string; +} + +export interface RecordingSink extends TraceSink { + readonly records: TraceRecord[]; + /** Stitch names that emitted an event of `type` — `'result'` is "this member SUCCEEDED". */ + names(type: StitchEvent['type']): string[]; + /** Distinct `traceId`s seen — 1 means the whole failover is one trace tree, 2 means two. */ + traceIds(): string[]; + /** `name → parentSpanId ?? ''` for each `start`, the shape of the trace fan/chain. */ + spine(): string[]; + reset(): void; +} + +export function recordingSink(): RecordingSink { + const records: TraceRecord[] = []; + const spans = new Map(); // spanId → name, to resolve a parent to its label + return { + records, + handle(event: StitchEvent, ctx: TraceContext): void { + if (event.type === 'start' && ctx.spanId) + spans.set(ctx.spanId, ctx.name); + const rec: TraceRecord = { name: ctx.name, type: event.type }; + if (event.type === 'result') rec.status = event.status; + if (event.type === 'error' && event.status !== undefined) + rec.status = event.status; + if (ctx.traceId !== undefined) rec.traceId = ctx.traceId; + if (ctx.spanId !== undefined) rec.spanId = ctx.spanId; + if (ctx.parentSpanId !== undefined) + rec.parentSpanId = ctx.parentSpanId; + records.push(rec); + }, + names(type) { + return records.filter((r) => r.type === type).map((r) => r.name); + }, + traceIds() { + return [ + ...new Set( + records + .map((r) => r.traceId) + .filter((t): t is string => t !== undefined), + ), + ]; + }, + spine() { + return records + .filter((r) => r.type === 'start') + .map( + (r) => + `${r.name}<-${r.parentSpanId ? (spans.get(r.parentSpanId) ?? 'span') : ''}`, + ); + }, + reset() { + records.length = 0; + spans.clear(); + }, + }; +} diff --git a/docs/scenarios/proofs/provider-failover/type-probe.ts b/docs/scenarios/proofs/provider-failover/type-probe.ts new file mode 100644 index 00000000..7ea4ff0d --- /dev/null +++ b/docs/scenarios/proofs/provider-failover/type-probe.ts @@ -0,0 +1,127 @@ +// Ask the COMPILER which failover spellings exist, instead of grepping for them. +// +// "There is no sequential-fallback combinator" and "I could not find the sequential-fallback +// combinator" are different findings, and only one of them is the library's problem. So the honest +// way to establish what the `stitchapi/pipe` vocabulary contains is to hand the compiler one +// candidate statement per spelling and read back its diagnostics: a namespace access +// (`pipe.fallback`) fails with 2339 when the export does not exist, and an unknown key inside a +// house envelope fails with a `NoUnknownNestedKeys` diagnostic naming the slot +// (types.ts:411-448). A line that compiles is a spelling that EXISTS. +// +// The fixture is written to a temp dir (not into the repo) and deleted afterwards, so this leaves +// nothing behind and never lands in `prettier --check`. It imports core by ABSOLUTE path, which is +// why it can live outside the tree. +// +// `typescript` is loaded through a `require` ANCHORED AT `packages/core`, which is the workspace +// package that declares it. A bare `import ts from 'typescript'` resolves under `tsx` and NOT under +// plain Node from this directory (pnpm gives `docs/` no `node_modules`), so the bare form would be +// a script that runs one way and typechecks another. The compiler surface used is tiny, so it is +// declared structurally here rather than imported as a type. +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +/** Absolute path to core's barrel, so the fixture can be compiled from anywhere. */ +export const CORE = join(HERE, '../../../../packages/core/src/index'); +/** Absolute path to the `stitchapi/pipe` subpath module — where the combinators live. */ +export const PIPE = join(HERE, '../../../../packages/core/src/pipe'); + +/** The slice of the TypeScript compiler API this probe uses. */ +interface TsCompiler { + readonly ScriptTarget: Record; + readonly ModuleKind: Record; + readonly ModuleResolutionKind: Record; + createProgram( + rootNames: readonly string[], + options: Record, + ): unknown; + getPreEmitDiagnostics(program: unknown): readonly { + code: number; + start?: number | undefined; + file?: + | { + fileName: string; + getLineAndCharacterOfPosition(pos: number): { line: number }; + } + | undefined; + }[]; +} + +const ts = createRequire(join(HERE, '../../../../packages/core/package.json'))( + 'typescript', +) as TsCompiler; + +export interface Candidate { + /** What a reader would call this spelling — printed in the report. */ + label: string; + /** One statement. Compiles ⇒ the spelling exists. */ + code: string; +} + +export interface ProbeResult extends Candidate { + compiles: boolean; + /** First diagnostic code, e.g. 2339 (no such property) or 2353 (unknown object key). */ + diagnostic?: number; +} + +/** + * Typecheck each candidate as its own statement in one program and report which compile. + * Diagnostics are attributed by LINE, so each candidate must be a single line. + * + * The header puts a namespace `pipe`, a `stitch` factory and two ready-made stitches (`a`, `b`) in + * scope, so a candidate can probe either a missing EXPORT or a missing CONFIG KEY. + */ +export function probeSpellings( + candidates: readonly Candidate[], +): ProbeResult[] { + const dir = mkdtempSync(join(tmpdir(), 'stitch-failover-probe-')); + const file = join(dir, 'probe.ts'); + const header = [ + `import * as pipe from ${JSON.stringify(PIPE)};`, + `import { stitch } from ${JSON.stringify(CORE)};`, + `const a = stitch({ url: 'https://primary.test/v1/complete' });`, + `const b = stitch({ url: 'https://backup.test/generate' });`, + `void [pipe, stitch, a, b];`, + ]; + try { + writeFileSync( + file, + [...header, ...candidates.map((c) => c.code)].join('\n'), + ); + const program = ts.createProgram([file], { + target: ts.ScriptTarget['ES2022'], + module: ts.ModuleKind['ESNext'], + moduleResolution: ts.ModuleResolutionKind['Bundler'], + strict: true, + noEmit: true, + skipLibCheck: true, + exactOptionalPropertyTypes: true, + noUncheckedIndexedAccess: true, + }); + const byLine = new Map(); + for (const d of ts.getPreEmitDiagnostics(program)) { + if (d.file?.fileName !== file || d.start === undefined) continue; + const { line } = d.file.getLineAndCharacterOfPosition(d.start); + if (!byLine.has(line)) byLine.set(line, d.code); + } + return candidates.map((c, i) => { + const diagnostic = byLine.get(header.length + i); + return diagnostic === undefined + ? { ...c, compiles: true } + : { ...c, compiles: false, diagnostic }; + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** The spellings that compiled — the vocabulary that actually exists. */ +export const accepted = (results: readonly ProbeResult[]): string[] => + results.filter((r) => r.compiles).map((r) => r.label); + +/** The spellings the compiler refused. */ +export const rejected = (results: readonly ProbeResult[]): string[] => + results.filter((r) => !r.compiles).map((r) => r.label); diff --git a/docs/scenarios/proofs/stale-fixture/README.md b/docs/scenarios/proofs/stale-fixture/README.md new file mode 100644 index 00000000..7a1c0c68 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/README.md @@ -0,0 +1,142 @@ +# Proofs — the mock that passed for six months + +Runnable evidence for the claims in [`../../stale-fixture.md`](../../stale-fixture.md). + +**Both deciding claims came back split, and one of them found more than it was sent for.** C1 +predicted that a shared `output` schema would catch a stale fixture; measured, it catches the +fixture drifting from the _schema_ — all four mutations fail — and is structurally blind to the +scenario's actual shape, the _vendor_ drifting while the fixture sits still. C2 was sent to confirm +three known wall-clock features and came back with **six**, including two nobody had recorded: +OAuth2 token expiry and AWS SigV4 signing. + +Every script is standalone and offline. Time is load-bearing throughout, so `manualClock()` drives +every wait — except in the two places where the whole point is that it _doesn't_, and those are +measured against a real clock side by side. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/stale-fixture/c2-clock-coverage.ts + +# all of them +for f in docs/scenarios/proofs/stale-fixture/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set (the three `@ts-expect-error` lines are the +machine-checked half of C3(f), C6(c) and C6(e) — a `@ts-expect-error` that is _not_ an error fails +`tsc`): + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/stale-fixture/*.ts +``` + +### Why this directory imports Zod by path + +Same reason as [`../intermittent-drift/zod.ts`](../intermittent-drift/zod.ts): C1's question is +whether the **same** `output` schema guards prod and the double, and "the same schema" only means +something if the schema is real. Hand-rolling a `{ validate }` stub would let this directory invent +which key is required and what `.optional()` does to a removal — which is exactly what C1 is +measuring. The resolved version in this workspace is **Zod 3.25.76** +(`packages/core/node_modules/zod`), which is why C1(c)'s messages read `Required` and +`Expected boolean, received string`. In application code the spelling is `import { z } from 'zod'`. + +## What each script establishes + +| Script | Question | Measured | +| ------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------- | +| `c1-fixture-drift.ts` | can a stale fixture be caught? | **Only in the wrong direction.** 4/4 mutations fail; vendor-drift run is `test ok / prod fail`, 0 findings | +| `c2-clock-coverage.ts` | does `manualClock` drive every timed feature? | **6 driven, 6 wall-clock, 2 timeless.** A 1000ms `timeout.total` survived **2700 virtual ms** and returned ok | +| `c3-mock-fidelity.ts` | what does `mockAdapter` validate? | **Status + header case, nothing else.** Serves `Date`/`Map`/class/`bigint`; fails 1 of its own 9 contract rules | +| `c4-resilience-events.ts` | resilience with no vendor? | **Yes.** Circuit trace from `callCount()` alone. But retry backoff carries **no `waited`**, and `elapsed` is 0 | +| `c5-stub-contract.ts` | does a stub honour the input contract? | **No — zero of the `input` slots run.** Plus a bug: `.safe()` **throws** on a sync-throwing impl | +| `c6-sandbox-parity.ts` | can one stitch target sandbox and prod? | **Targeting yes, scheduling no.** 3 targets from 1 endpoint; 0 env slots, 0 of 8 CLI verbs verify anything | +| `c7-streams.ts` | is mid-stream failure deterministic? | **Yes — 5 runs, 1 distinct outcome.** But a clean early close is byte-identical to success | +| `c8-assembled.ts` | best "fixtures cannot rot" setup, and the price? | **93 executable lines, 5 seams.** Closes 4 of 5 gaps; the 5th needs a live call and cannot be offline | + +## Files + +- `vendor.ts` — the two objects the whole scenario lives between: `RECORDED_2026_02_04` (the + cassette) and `VENDOR_TODAY` (five keys different — a rename, a removal, a retype, a null). Plus + `MUTATIONS` (the four drift shapes as separate bodies) and two hand-written in-memory adapters. + Deliberately does **not** use `mockAdapter`, because C3 puts `mockAdapter` under test. +- `harness.ts` — `check` / `checkSeq` / `checkNear` / `checkAtLeast` / `note` / `heading` / `finish`, + plus the two this scenario needed: `checkClockDriven` (runs a scenario twice — once with the + advance the test believes is decisive, once with `advance(0)` — and asserts whether the two + outcomes DIFFER; identical outcomes mean the advance caused nothing, which is the signature of a + **vacuous** test) and `row`/`printClockTable` (the C2 table). +- `fixture-guard.ts` — **user code** for C8, between the `>>> BEGIN USER CODE` markers: `stamp` / + `expired` (a recording date, the one fact the library cannot hold), `jsonOnly` (an adapter wrapper + that rejects any fixture body a JSON wire could not deliver), `contractStub` (a `stubStitch` that + runs the real `input` schemas), `assertClockHonest` (refuses a `manualClock` paired with a slot it + cannot drive), and `parity` (the only export that needs a network call — quarantined, and saying so + is half its value). +- `zod.ts` — real Zod, imported by path. See above. + +## The C2 table + +The definitive enumeration. Fourteen rows, every place `packages/core/src` reads time. + +| Feature | Driven by | Evidence | +| -------------------------- | ------------- | ------------------------------------------------------------------------------- | +| retry backoff | `manualClock` | `advance(5000)` → 3 calls, ok; `advance(0)` → 1 call, pending | +| throttle rate | `manualClock` | `advance(3000)` → 3 calls; `advance(0)` → 1 call | +| throttle concurrency | `manualClock` | holder releases on virtual time → queued callers proceed | +| `circuit.cooldown` | `manualClock` | `advance(60_000)` past a 30s cooldown → half-open probe reaches the vendor | +| `timeout` (per-attempt) | `manualClock` | `advance(2000)` past a 1s timeout → error; `advance(0)` → pending | +| `Retry-After` (HTTP-date) | `manualClock` | `resilience.ts:70` reads `clock.now()` — **and that is the trap**, see below | +| **`timeout.total`** | **wall** | `engine.ts:465/483/505/527/661`; a 1000ms budget survived **2700 virtual ms** | +| **`cache.ttl`** | **wall** | `store.ts:30/59`; `advance(600_000)` past a 60s TTL still served the entry | +| **`memoryStore` TTL** | **wall** | the layer beneath it; a 1s entry survived 60_000 virtual ms | +| **event `at` / `elapsed`** | **wall** | measured `at=1785941…` while `clock.now()` was `0`; `done.elapsed` reads `0` | +| **OAuth2 token expiry** | **wall** | `auth.ts:502/564`; 600_000 virtual ms past a 60s `expires_in` refetched nothing | +| **AWS SigV4 signing date** | **wall** | `aws-sigv4/src/index.ts:301` `new Date()`; there is no `clock` option to pass | +| `paginate` | no time | no inter-page delay knob; 3 pages fetched at `clock.now() === 0` | +| `Retry-After` (delta-secs) | no time | a pure number; `parseRetryAfter('5')` is `5000` on any clock | + +Six driven, six wall-clock, two with no time in them. + +## Reading the numbers honestly + +- **C2's `timeout.total` row is not "the clock is ignored".** The engine clamps each attempt's abort + to `budget.deadline - now()` and hands that to `withTimeout(..., rt.clock)`, so the clamp _does_ + fire on virtual time. What is wall-anchored is the **deadline** — `wallT0 + total`. Virtual sleeps + never move the wall, so the remaining budget is recomputed as ~the full total at every attempt. The + budget does not drain; it **resets**. C2(f) measures it as a side-by-side: the same config shape on + a real clock dies at 101ms after 2 attempts with `timed out after 100ms`; on a manual clock it runs + 3 attempts across 2700 virtual ms and returns `ok`. + +- **The `Retry-After` HTTP-date row is a trap, not a gap, and it is arguably worse.** + `parseRetryAfter` reads the injected clock _faithfully_ — `httpDateEpoch - clock.now()` — and + `manualClock()` starts at `0`. A server date meaning "5 seconds" therefore becomes a wait of + **20,671 days**. The feature honouring the clock is exactly what breaks it. + +- **C1 is not a criticism of the schema.** The schema does its job in every direction it can see. The + gap is epistemic: offline, the only bytes available are the fixture's, so "has the vendor changed" + is not a question any amount of validation can answer. C8 concedes this and settles for dating the + fixture (`getInvoice recorded 2026-02-04 (182d old)`), which fails the suite on the calendar rather + than on the drift — a weaker guarantee, reported as one. + +- **Two findings are library bugs rather than design trade-offs**, and both were found in passing: + `mockAdapter` violates the library's own `verifyAdapterContract` rule + `abort: a pre-aborted signal rejects` (it consults `req.signal` only inside its `delay` branch, + `test-mock.ts:188-189`), and `stubStitch(...).safe()` **throws** when the impl throws synchronously + (`resolve()` at `test-stub.ts:59-63` evaluates `impl(input)` as an argument to `Promise.resolve`). + The real stitch honours `.safe()` in the same situation; `.stream()` on the same stub is fine. + +- **C6's answer is a split, and the split is the point.** Targeting is genuinely well served — + `extends: { baseUrl, adapter }` aims one endpoint definition at a fixture, a sandbox and prod, and + the same `output` schema makes the difference visible. What is absent is the _schedule_: no + environment concept on `StitchConfig` (an unknown key is a compile error), none of the 8 CLI + subcommands verifies anything, and all four `verify*Contract` functions check an implementation of + one of StitchAPI's **own** seams, never a vendor. diff --git a/docs/scenarios/proofs/stale-fixture/c1-fixture-drift.ts b/docs/scenarios/proofs/stale-fixture/c1-fixture-drift.ts new file mode 100644 index 00000000..2389057a --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c1-fixture-drift.ts @@ -0,0 +1,302 @@ +// C1 (DECIDING) — can a fixture be caught when it goes stale? +// +// The capture's hypothesis is that a shared `output` schema is "the honest middle ground — but only +// if the SAME schema guards prod and the fixtures". This script builds exactly that: one Zod schema, +// one stitch config, and only the `adapter` swapped between the live vendor and the test double. +// Then it drifts things and measures who fails. +// +// The finding is a DIRECTION problem, and it is the whole scenario: +// +// (b) fixture drifts, schema holds -> the test FAILS. The schema works. +// (e) VENDOR drifts, fixture holds -> the test PASSES. Production is broken. +// +// (e) is the scenario's actual shape — "the mock that passed for six months" is the vendor moving +// while the cassette sits still — and no arrangement of `output`/`drift()` detects it offline, +// because offline there is nothing to compare the fixture against except the schema the fixture +// already satisfies. Everything the library validates is downstream of a byte that must come from +// somewhere, and in a test that somewhere is the fixture. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c1-fixture-drift.ts +import { drift, stitch } from '../../../../packages/core/src/index'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import type { + Adapter, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { + BASE, + MUTATIONS, + RECORDED_2026_02_04, + RECORDED_ON, + VENDOR_TODAY, + fmt, + vendorAdapter, +} from './vendor'; +import { z } from './zod'; + +// ── THE CONTRACT ──────────────────────────────────────────────────────────────────────────────── +// One schema. It is what production validates against, and — because the stitch config is shared — +// it is also what the test double's responses are validated against. This is the capture's +// prescription, built literally. +const Invoice = z.object({ + id: z.string(), + amount_cents: z.number(), + currency: z.string(), + paid: z.boolean(), + customer_email: z.string(), + legacy_ref: z.string(), +}); + +/** The result of one call, reduced to a comparable shape. */ +interface Outcome { + ok: boolean; + message: string | null; + findings: string[]; + data: unknown; +} + +/** + * ONE stitch definition. `adapter` is the only parameter — that is the seam the capture says should + * be the only difference between prod and test, so it is the only difference here. + */ +async function callWith( + adapter: Adapter, + output: unknown = Invoice, +): Promise { + const findings: string[] = []; + const sink: TraceSink = { + handle(e: StitchEvent) { + if (e.type === 'drift') findings.push(fmt(e.finding)); + }, + }; + const call = stitch({ + name: 'getInvoice', + baseUrl: BASE, + path: '/v1/invoices/{id}', + adapter, + output: output as never, + trace: sink, + }); + const r = await call.safe({ params: { id: 'inv_9f2' } }); + return { + ok: r.ok, + message: r.error?.message ?? null, + findings, + data: r.data, + }; +} + +async function main(): Promise { + heading('C1 (a) — the baseline: cassette and schema agree'); + { + const o = await callWith(vendorAdapter(RECORDED_2026_02_04)); + check('the call succeeded', o.ok, true); + checkSeq('drift findings', o.findings, []); + note('this is the state on the day the cassette was cut', RECORDED_ON); + } + + heading( + 'C1 (b) — the fixture drifts from the schema: removed / renamed / retyped / nulled', + ); + // Direction 1. The FIXTURE is wrong and the schema is right. This is the direction the capture + // predicts is covered, and it is: all four mutations fail the call. + { + const rows: string[] = []; + for (const name of [ + 'removed', + 'renamed', + 'retyped', + 'nulled', + ] as const) { + const o = await callWith(vendorAdapter(MUTATIONS[name])); + rows.push( + `${name.padEnd(8)} ok=${String(o.ok)} findings=${String(o.findings.length)}`, + ); + } + checkSeq('four mutations, four outcomes', rows, [ + 'removed ok=false findings=1', + 'renamed ok=false findings=1', + 'retyped ok=false findings=1', + 'nulled ok=false findings=1', + ]); + note( + '(b) → a fixture that violates the schema CANNOT be served silently. Note `findings=1` on a PLAIN `output`, with no `drift()` anywhere: a hard validation failure travels the drift CHANNEL (`error|invalid`) whether or not you opted into drift reporting', + '', + ); + } + + heading('C1 (c) — the same four, wrapped in `drift()`, for the messages'); + // `drift()` turns the same failures into named findings. Same verdict, more detail — and this is + // the answer to "does drift() help here": it helps you READ the failure, not FIND it. + { + const messages: string[] = []; + for (const name of [ + 'removed', + 'renamed', + 'retyped', + 'nulled', + ] as const) { + const o = await callWith( + vendorAdapter(MUTATIONS[name]), + drift(Invoice as never), + ); + messages.push(`${name}: ${o.findings.join(' + ') || ''}`); + } + for (const m of messages) note(m); + check( + '(c) every mutation produced at least one finding under drift()', + messages.every((m) => !m.endsWith('')), + true, + ); + note( + '(c) → `drift()` reports WHICH field and HOW. But it fired because the fixture broke the SCHEMA, not because the fixture is old', + '', + ); + } + + heading('C1 (d) — is the double really validated the same as prod?'); + // The claim "the same schema guards both" is worth measuring rather than assuming: swap in the + // library's own `mockAdapter` and confirm the engine still runs `output` over its canned body. + { + const mock = mockAdapter([ + { + match: '/v1/invoices/inv_9f2', + respond: { body: MUTATIONS.retyped }, + }, + ]); + const o = await callWith(mock); + check('mockAdapter fixture is validated too', o.ok, false); + check('and the transport was actually consulted', mock.callCount(), 1); + note( + '(d) → `mockAdapter` is below validation, so the full engine runs over the fixture. The double is not a bypass', + '', + ); + } + + heading( + 'C1 (e) — THE ACTUAL SCENARIO: the vendor drifts, the fixture does not', + ); + // Direction 2, and the one the scenario is named for. The cassette still satisfies the schema + // (it was recorded when both were true). The vendor no longer does. The test is green. + { + const test = await callWith(vendorAdapter(RECORDED_2026_02_04)); + const prod = await callWith(vendorAdapter(VENDOR_TODAY)); + check('the TEST passes', test.ok, true); + check('PRODUCTION fails', prod.ok, false); + check( + 'the test emitted no finding of any kind', + test.findings.length, + 0, + ); + checkSeq( + 'the keys the two responses disagree on', + [ + ...new Set([ + ...Object.keys(RECORDED_2026_02_04), + ...Object.keys(VENDOR_TODAY), + ]), + ].filter( + (k) => + JSON.stringify( + (RECORDED_2026_02_04 as Record)[k], + ) !== + JSON.stringify( + (VENDOR_TODAY as Record)[k], + ), + ), + ['amount_cents', 'paid', 'customer_email', 'legacy_ref', 'amount'], + ); + note( + '(e) → five keys differ between the cassette and the live vendor, and the suite is green. This is the six months', + '', + ); + } + + heading( + 'C1 (f) — can anything assert "my fixture still matches the contract"?', + ); + // The narrow question: given a fixture and the schema, is there a published call that says + // "validate this object against this stitch's `output`" without a transport? Measured by + // checking the exported surface of the main entry and of `stitchapi/testing`. + { + const core = + (await import('../../../../packages/core/src/index')) as Record< + string, + unknown + >; + const testing = + (await import('../../../../packages/core/src/testing')) as Record< + string, + unknown + >; + const named = [...Object.keys(core), ...Object.keys(testing)].sort(); + const fixtureWords = named.filter((n) => + /fixture|cassette|record|snapshot|stale|fresh|expire/i.test(n), + ); + checkSeq( + 'exports mentioning fixture/cassette/record/snapshot/stale/freshness', + fixtureWords, + ['adapterContractFixture'], + ); + note( + 'and `adapterContractFixture` is the ADAPTER echo contract — for people writing transports, not people holding a stale invoice body', + '', + ); + note('total exported names across both entries', named.length); + + // The honest workaround: a fixture is just data, and a schema is callable directly. This + // works — but it is your code, not a library seam, and it validates the fixture against the + // SCHEMA, which is the direction (b) already covered. + const parsed = Invoice.safeParse(RECORDED_2026_02_04); + check( + 'a fixture CAN be checked against the schema directly', + parsed.success, + true, + ); + const stale = Invoice.safeParse(VENDOR_TODAY); + check("…and it catches today's vendor body", stale.success, false); + note( + "(f) → but that last check required HAVING today's vendor body. Offline, you do not", + '', + ); + } + + heading('C1 (g) — is the recording date expressible anywhere?'); + { + // `__config` is the published read-out of a stitch. If a recorded-on date could ride + // anywhere it would ride here. + const call = stitch({ + name: 'getInvoice', + baseUrl: BASE, + path: '/v1/invoices/{id}', + adapter: vendorAdapter(RECORDED_2026_02_04), + output: Invoice as never, + }); + const cfgKeys = Object.keys(call.__config).sort(); + checkSeq('__config keys', cfgKeys, [ + 'baseUrl', + 'kind', + 'name', + 'output', + 'path', + ]); + check( + 'a slot for arbitrary metadata', + cfgKeys.some((k) => /meta|tag|label|note|version/i.test(k)), + false, + ); + note( + `(g) → \`${RECORDED_ON}\` lives in this proof directory's source and nowhere the library can read`, + '', + ); + } + + finish( + 'C1', + 'PARTIAL, AND THE HALF THAT IS MISSING IS THE SCENARIO. A shared `output` schema DOES catch a fixture that drifts from the contract: all four mutations — `legacy_ref` removed, `amount_cents` renamed, `paid` retyped, `customer_email` nulled — fail the call, through the library\'s own `mockAdapter` as well as a hand-written one, and `drift()` names each one. But that is the fixture drifting from the SCHEMA. The scenario is the VENDOR drifting from the schema while the fixture sits still, and measured, that run is: test ok=true with zero findings, production ok=false, five keys different. Nothing offline closes it, because offline the only bytes are the fixture\'s. Of the exported names across the main entry and `stitchapi/testing`, exactly one matches /fixture|cassette|record|snapshot|stale|fresh|expire/ — `adapterContractFixture`, which is the transport echo contract for plugin authors — and `__config` has no metadata slot, so "recorded on 2026-02-04" is not expressible in the library at all', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c2-clock-coverage.ts b/docs/scenarios/proofs/stale-fixture/c2-clock-coverage.ts new file mode 100644 index 00000000..1470fee3 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c2-clock-coverage.ts @@ -0,0 +1,692 @@ +// C2 (DECIDING, PRE-REGISTERED SUSPICION) — is `manualClock` sound across EVERY time-driven feature? +// +// Pattern 2b of this research pass records three incidental sightings of a feature ignoring the +// injected clock. This script stops accumulating sightings and enumerates the whole surface: every +// place `packages/core/src` reads time, probed the same way, in one table. +// +// THE METHOD. For each feature, the same scenario runs twice — once with the clock advanced by the +// amount the test believes is decisive, once with `advance(0)` — and the two outcomes are compared: +// +// outcomes DIFFER -> the feature consulted the injected clock. The assertion had teeth. +// outcomes IDENTICAL -> the feature never looked. A test that advances the clock and then asserts +// on the result is asserting something the advance did not cause. +// +// That second row is the point of the whole script. A test of an inert feature does not fail — it +// PASSES, having exercised nothing, which is the worst failure mode a testing tool has. Section (i) +// makes that concrete with a `timeout.total` test that looks correct, passes, and is a lie. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c2-clock-coverage.ts +import { stitch } from '../../../../packages/core/src/index'; +import { parseRetryAfter } from '../../../../packages/core/src/resilience'; +import { manualClock } from '../../../../packages/core/src/test-clock'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, + StitchEvent, + TraceSink, +} from '../../../../packages/core/src/types'; +import { + check, + checkAtLeast, + checkClockDriven, + checkSeq, + finish, + heading, + note, + printClockTable, + row, +} from './harness'; +import { BASE, sequenceAdapter, vendorAdapter } from './vendor'; + +/** Let queued continuations run without advancing virtual time. */ +const settle = (): Promise => + new Promise((r) => { + setTimeout(r, 0); + }); + +/** + * A transport whose latency is VIRTUAL: it waits on `clock.sleep`, so it only "takes time" when the + * test advances the clock. This is the honest way to model a slow endpoint under a manual clock — + * `mockAdapter`'s own `delay` uses a real `setTimeout`, which would make every timing probe here a + * wall-clock race. + */ +function slowAdapter( + clock: Clock, + latencyMs: number, + steps: readonly (readonly [number, unknown])[], +): Adapter & { count(): number } { + let n = 0; + const fn = (async (req: AdapterRequest): Promise => { + const i = n++; + await clock.sleep(latencyMs, req.signal); + const step = steps[Math.min(i, steps.length - 1)] as readonly [ + number, + unknown, + ]; + return { status: step[0], headers: {}, body: step[1] }; + }) as Adapter & { count(): number }; + fn.count = () => n; + return fn; +} + +async function main(): Promise { + // ── (a) retry backoff ─────────────────────────────────────────────────────────────────────── + heading('C2 (a) — retry backoff'); + await checkClockDriven( + 'retry backoff', + 'driven', + async (advanceMs) => { + const clock = manualClock(); + const vendor = sequenceAdapter([ + [503, {}], + [503, {}], + [200, { ok: true }], + ]); + const call = stitch({ + url: `${BASE}/flaky`, + adapter: vendor, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 1000 } }, + clock, + }); + let state = 'pending'; + void call.safe().then((r) => { + state = r.ok ? 'ok' : 'err'; + }); + await clock.advance(advanceMs); + return `calls=${String(vendor.count())} ${state}`; + }, + 5000, + ); + row( + 'retry backoff', + 'CLOCK', + 'advance(5000) -> calls=3 ok; advance(0) -> calls=1 pending', + ); + + // ── (b) throttle: rate spacing ────────────────────────────────────────────────────────────── + heading('C2 (b) — throttle rate spacing'); + await checkClockDriven( + 'throttle rate', + 'driven', + async (advanceMs) => { + const clock = manualClock(); + const vendor = vendorAdapter({ ok: true }); + const call = stitch({ + url: `${BASE}/paced`, + adapter: vendor, + throttle: '1/s', + clock, + }); + void call.safe(); + void call.safe(); + void call.safe(); + await clock.advance(advanceMs); + return `calls=${String(vendor.count())}`; + }, + 3000, + ); + row( + 'throttle rate', + 'CLOCK', + 'advance(3000) -> 3 calls; advance(0) -> 1 call', + ); + + // ── (c) throttle: concurrency ─────────────────────────────────────────────────────────────── + // Concurrency is not a duration, but the WAIT it induces is — a blocked caller only proceeds + // when the holder finishes, and here the holder finishes on virtual time. + heading('C2 (c) — throttle concurrency'); + await checkClockDriven( + 'throttle concurrency', + 'driven', + async (advanceMs) => { + const clock = manualClock(); + const vendor = slowAdapter(clock, 500, [[200, { ok: true }]]); + const call = stitch({ + url: `${BASE}/limited`, + adapter: vendor, + throttle: { concurrency: 1 }, + clock, + }); + void call.safe(); + void call.safe(); + void call.safe(); + await clock.advance(advanceMs); + return `calls=${String(vendor.count())}`; + }, + 2000, + ); + row( + 'throttle concurrency', + 'CLOCK', + 'holder releases on virtual time -> queued callers proceed', + ); + + // ── (d) circuit cooldown ──────────────────────────────────────────────────────────────────── + heading('C2 (d) — circuit cooldown'); + await checkClockDriven( + 'circuit.cooldown', + 'driven', + async (advanceMs) => { + const clock = manualClock(); + const vendor = sequenceAdapter([[500, { e: 1 }]]); + const call = stitch({ + url: `${BASE}/breaker`, + adapter: vendor, + circuit: { failures: 2, cooldown: 30_000 }, + clock, + }); + await call.safe(); + await call.safe(); // trips the breaker + const openTry = await call.safe(); + const opened = openTry.error?.message ?? ''; + await clock.advance(advanceMs); + const afterTry = await call.safe(); + return `open=${opened.includes('circuit') ? 'y' : 'n'} calls=${String(vendor.count())} after=${afterTry.error?.message?.includes('circuit') ? 'still-open' : 'probed'}`; + }, + 60_000, + ); + row( + 'circuit.cooldown', + 'CLOCK', + 'advance(60000) past a 30s cooldown -> half-open probe reaches the vendor', + ); + + // ── (e) per-attempt timeout ───────────────────────────────────────────────────────────────── + heading('C2 (e) — per-attempt timeout'); + await checkClockDriven( + 'timeout (per-attempt)', + 'driven', + async (advanceMs) => { + const clock = manualClock(); + const vendor = slowAdapter(clock, 5000, [[200, { ok: true }]]); + const call = stitch({ + url: `${BASE}/slow`, + adapter: vendor, + timeout: 1000, + clock, + }); + let state = 'pending'; + void call.safe().then((r) => { + state = r.ok ? 'ok' : (r.error?.name ?? 'err'); + }); + await clock.advance(advanceMs); + return state; + }, + 2000, + ); + row( + 'timeout (per-attempt)', + 'CLOCK', + 'advance(2000) past a 1s timeout -> TimeoutError; advance(0) -> pending', + ); + + // ── (f) timeout.total ─────────────────────────────────────────────────────────────────────── + // The pre-registered suspicion, tested head-on. ADR 0010 §4 says this deliberately stays on + // wall-clock; the question is what that COSTS a test that does not know it. + // + // A driven/inert binary is the WRONG instrument for this row, and saying why is half the + // finding. `timeout.total` is not ignored by the clock: engine.ts:656-662 clamps each attempt's + // abort to `budget.deadline - now()` and hands that to `withTimeout(..., rt.clock)`, so the + // clamp DOES fire on virtual time. What is wall-anchored is the DEADLINE — `wallT0 + total`. + // Virtual sleeps never move the wall, so the remaining budget is recomputed as ~the full total + // at every attempt. The budget does not drain; it resets. + // + // So the honest measurement is a side-by-side of the SAME config on the two clocks, scaled so + // the real-clock arm is fast: 3 attempts x 90ms latency against a 100ms total budget. + heading('C2 (f) — timeout.total'); + { + const STEPS = [ + [503, {}], + [503, {}], + [200, { ok: true }], + ] as const; + + // ARM 1 — manual clock, 900ms virtual latency, 1000ms total budget. + const clock = manualClock(); + const vendor = slowAdapter(clock, 900, STEPS); + const virt = stitch({ + url: `${BASE}/budget`, + adapter: vendor, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + timeout: { total: 1000 }, + clock, + }); + let ok: boolean | null = null; + let errName = ''; + let consumed = -1; + void virt.safe().then((r) => { + ok = r.ok; + errName = r.error?.name ?? ''; + consumed = clock.now(); + }); + await clock.advance(10_000); + check('(f) manual clock: the call SUCCEEDED', ok, true); + check('(f) manual clock: no timeout error', errName, ''); + check('(f) manual clock: attempts the vendor saw', vendor.count(), 3); + checkAtLeast( + '(f) manual clock: VIRTUAL ms consumed before it settled', + consumed, + 2700, + ); + + // ARM 2 — the real clock, same shape, 10x smaller so the script stays fast. + const realVendor = (() => { + let n = 0; + const fn = (async ( + _req: AdapterRequest, + ): Promise => { + const i = n++; + await new Promise((r) => { + setTimeout(r, 90); + }); + const step = STEPS[Math.min(i, STEPS.length - 1)] as readonly [ + number, + unknown, + ]; + return { status: step[0], headers: {}, body: step[1] }; + }) as Adapter & { count(): number }; + fn.count = () => n; + return fn; + })(); + const real = stitch({ + url: `${BASE}/budget`, + adapter: realVendor, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + timeout: { total: 100 }, + }); + const realStart = Date.now(); + const realResult = await real.safe(); + const realElapsed = Date.now() - realStart; + check('(f) real clock: the call FAILED', realResult.ok, false); + check( + '(f) real clock: with a timeout message', + realResult.error?.message, + 'timed out after 100ms', + ); + check('(f) real clock: attempts the vendor saw', realVendor.count(), 2); + note('(f) real clock: real ms elapsed', realElapsed); + note( + '(f) → SAME config shape, 2.7x the budget consumed in each arm. Real clock: 2 attempts, dead at 101ms with `timed out after 100ms`. Manual clock: 3 attempts, 2700 virtual ms against a 1000ms budget, ok=true. The budget deadline is `wallT0 + total` (engine.ts:465) and every remaining-budget read is `deadline - now()` on the WALL (engine.ts:483/505/527/661), so under a manual clock it never drains', + '', + ); + row( + 'timeout.total', + 'WALL', + 'engine.ts:465/483/505/527/661 use util now(); a 1000ms budget survived 2700 virtual ms and returned ok=true', + ); + } + + // ── (g) cache.ttl ─────────────────────────────────────────────────────────────────────────── + heading('C2 (g) — cache.ttl'); + await checkClockDriven( + 'cache.ttl', + 'inert', + async (advanceMs) => { + const clock = manualClock(); + const vendor = vendorAdapter({ v: 1 }); + const call = stitch({ + url: `${BASE}/cached`, + adapter: vendor, + cache: { ttl: 60_000 }, + clock, + }); + await call.safe(); + await clock.advance(advanceMs); + await settle(); + await call.safe(); + return `calls=${String(vendor.count())}`; + }, + 600_000, + ); + row( + 'cache.ttl', + 'WALL', + 'store.ts:30/59 use util `now()`; advancing 600000 virtual ms past a 60s TTL still served the cached entry', + ); + + // ── (h) paginate ──────────────────────────────────────────────────────────────────────────── + // Pagination has no delay knob of its own — `PaginateOptions` is `{ next }` plus limits, and + // page N+1 is fetched immediately. It is listed here because the capture names it, and the + // honest answer is that there is no time in it to drive. + heading('C2 (h) — paginate'); + { + const clock = manualClock(); + let page = 0; + const vendor: Adapter = () => { + page++; + return Promise.resolve({ + status: 200, + headers: {}, + body: { items: [page], next: page < 3 ? page + 1 : null }, + }); + }; + const call = stitch({ + url: `${BASE}/pages`, + adapter: vendor, + paginate: { + next: (body) => + (body as { next: number | null }).next == null + ? undefined + : { + query: { + page: (body as { next: number }).next, + }, + }, + }, + clock, + }); + const r = await call.safe(); + check('(h) all pages fetched with NO clock advance', r.ok, true); + check('(h) pages fetched', page, 3); + check('(h) virtual time consumed', clock.now(), 0); + note( + '(h) → `paginate` has no inter-page delay to drive. Nothing to be inert about; a paced crawl is spelled with `throttle`, which IS clock-driven (row b)', + '', + ); + row( + 'paginate', + 'NONE', + 'no inter-page delay knob; 3 pages fetched at clock.now()=0. Pacing is spelled `throttle` (row b), which IS driven', + ); + } + + // ── (i) event `at` / `done.elapsed` ───────────────────────────────────────────────────────── + heading('C2 (i) — event timestamps'); + { + const clock = manualClock(); + const events: StitchEvent[] = []; + const sink: TraceSink = { + handle(e) { + events.push(e); + }, + }; + const call = stitch({ + url: `${BASE}/ts`, + adapter: vendorAdapter({ ok: true }), + clock, + trace: sink, + }); + await call.safe(); + const ats = events.map((e) => e.at); + const allZero = ats.every((a) => a === 0); + check('(i) every event `at` is virtual 0', allZero, false); + checkAtLeast( + '(i) the first event `at` (epoch ms — a wall-clock read)', + ats[0] ?? 0, + 1_700_000_000_000, + ); + check('(i) clock.now() at the same moment', clock.now(), 0); + note( + '(i) → an assertion on event ordering by `at` under a manual clock compares WALL timestamps, so two events the test believes are 30s apart carry `at` values microseconds apart', + '', + ); + row( + 'event `at` / `done.elapsed`', + 'WALL', + `engine.ts uses util now(); measured at=${String(ats[0])} while clock.now()=0`, + ); + } + + // ── (j) OAuth2 token expiry — NOT in the capture's list ───────────────────────────────────── + // `auth.ts:502` gates token freshness on `now() < t.expiresAt - skew`, both wall-clock. A test + // that advances a manual clock past `expires_in` to assert a refresh will never see one. + heading('C2 (j) — OAuth2 token expiry (NOT in the claims list)'); + await checkClockDriven( + 'oauth2 token expiry', + 'inert', + async (advanceMs) => { + const clock = manualClock(); + let tokenCalls = 0; + let apiCalls = 0; + const transport = (async ( + req: AdapterRequest, + ): Promise => { + if (req.url.includes('/oauth/token')) { + tokenCalls++; + return { + status: 200, + headers: {}, + body: { + access_token: `tok_${String(tokenCalls)}`, + expires_in: 60, // one minute + }, + }; + } + apiCalls++; + return { status: 200, headers: {}, body: { ok: true } }; + }) as Adapter; + const { oauth2 } = + await import('../../../../packages/core/src/auth'); + const call = stitch({ + url: `${BASE}/secure`, + adapter: transport, + clock, + auth: oauth2({ + tokenUrl: `${BASE}/oauth/token`, + clientId: () => 'id', + clientSecret: () => 'secret', + adapter: transport, + }), + }); + await call.safe(); + await clock.advance(advanceMs); + await settle(); + await call.safe(); + return `tokenCalls=${String(tokenCalls)} apiCalls=${String(apiCalls)}`; + }, + 600_000, + ); + row( + 'oauth2 token expiry', + 'WALL', + 'auth.ts:502/564 use util `now()`; advancing 600000 virtual ms past a 60s `expires_in` refetched nothing', + ); + + // ── (k) AWS SigV4 signing date ────────────────────────────────────────────────────────────── + heading('C2 (k) — AWS SigV4 signing date'); + { + const { awsSigV4 } = + (await import('../../../../packages/aws-sigv4/src/index')) as typeof import('../../../../packages/aws-sigv4/src/index'); + const strategy = awsSigV4({ + accessKeyId: () => 'AKIAEXAMPLE', + secretAccessKey: () => 'secret', + region: 'us-east-1', + service: 'execute-api', + }); + const req: AdapterRequest = { + url: `${BASE}/signed`, + method: 'GET', + headers: {}, + }; + // A manual clock is not even expressible here: `awsSigV4` takes no `clock`. + await strategy.apply(req, { + emit: () => undefined, + vault: { + get: () => Promise.resolve(undefined), + set: () => Promise.resolve(), + delete: () => Promise.resolve(), + }, + } as never); + const amzDate = req.headers['x-amz-date'] ?? ''; + check('(k) the signature carries a date', amzDate.length > 0, true); + check( + '(k) …and it is NOT the manual clock epoch (19700101T000000Z)', + amzDate.startsWith('19700101'), + false, + ); + check( + '(k) `awsSigV4` accepts a `clock` option at all', + 'clock' in + ({ + accessKeyId: () => '', + secretAccessKey: () => '', + region: '', + service: '', + } as Record), + false, + ); + note('(k) measured x-amz-date', amzDate); + row( + 'AWS SigV4 signing date', + 'WALL', + `aws-sigv4/src/index.ts:301 \`new Date()\`; no clock option; measured ${amzDate}`, + ); + } + + // ── (l) Retry-After as an HTTP-date ───────────────────────────────────────────────────────── + // `parseRetryAfter` DOES take the clock — and that is exactly what makes it a trap. It computes + // `httpDateEpoch - clock.now()`, and `manualClock()` starts at 0, so an HTTP-date the server + // meant as "5 seconds" becomes a wait of the entire Unix epoch. + heading('C2 (l) — Retry-After as an HTTP-date'); + { + const clock = manualClock(); + const fiveSecondsOut = new Date(Date.now() + 5000).toUTCString(); + const wall = parseRetryAfter(fiveSecondsOut); + const virtual = parseRetryAfter(fiveSecondsOut, clock); + check( + '(l) delta-seconds form is clock-independent', + parseRetryAfter('5', clock), + 5000, + ); + checkAtLeast( + '(l) HTTP-date under systemClock (~5000ms)', + wall ?? 0, + 4000, + ); + check( + '(l) …and under systemClock it is under 6s', + (wall ?? 0) < 6000, + true, + ); + checkAtLeast( + '(l) HTTP-date under manualClock() (ms)', + virtual ?? 0, + 1_700_000_000_000, + ); + note( + '(l) measured manualClock wait, in DAYS', + Math.round((virtual ?? 0) / 86_400_000), + ); + note( + '(l) → `parseRetryAfter` reads the injected clock faithfully, and that is the bug: `manualClock()` starts at 0, so a server date is ~20,000 days in the "future". A retry test with an HTTP-date `Retry-After` hangs on an advance that will never come', + '', + ); + row( + 'Retry-After (delta-seconds)', + 'NONE', + "a pure number; parseRetryAfter('5') = 5000 on any clock", + ); + row( + 'Retry-After (HTTP-date)', + 'CLOCK', + `resilience.ts:70 reads clock.now(); manualClock(0) turns "5s" into ${String(Math.round((virtual ?? 0) / 86_400_000))} days`, + ); + } + + // ── (m) memoryStore TTL, directly ─────────────────────────────────────────────────────────── + heading('C2 (m) — memoryStore TTL (the layer under cache.ttl)'); + await checkClockDriven( + 'memoryStore TTL', + 'inert', + async (advanceMs) => { + const { memoryStore } = + await import('../../../../packages/core/src/store'); + const clock = manualClock(); + const store = memoryStore(); + await store.set('k', 'v', 1000); + await clock.advance(advanceMs); + const got = await store.get('k'); + return `got=${JSON.stringify(got)}`; + }, + 60_000, + ); + row( + 'memoryStore TTL', + 'WALL', + 'store.ts:30/59 use util `now()`; a 1s entry survived 60000 virtual ms', + ); + + // ── (n) THE VACUOUS TEST ──────────────────────────────────────────────────────────────────── + // The sharp finding, spelled as the test someone actually writes. It looks right. It passes. + // It asserts nothing, and the identical assertion passes with the advance line DELETED. + heading('C2 (n) — the vacuous test, written out'); + { + /** "Assert that `timeout.total: 1000` fails a call that takes far longer." */ + const vacuousTest = async (withAdvance: boolean): Promise => { + const clock = manualClock(); + const vendor = slowAdapter(clock, 900, [ + [503, {}], + [503, {}], + [200, { ok: true }], + ]); + const call = stitch({ + url: `${BASE}/budget`, + adapter: vendor, + retry: { attempts: 3, backoff: { curve: 'fixed', base: 0 } }, + timeout: { total: 1000 }, + clock, + }); + const p = call.safe(); + if (withAdvance) + await clock.advance(10_000); // the "decisive" line + else await settle(); + const r = await Promise.race([ + p, + new Promise((res) => { + setTimeout(() => { + res(null); + }, 5); + }), + ]); + return r === null ? 'pending' : r.ok ? 'ok' : 'timed-out'; + }; + const withLine = await vacuousTest(true); + const withoutLine = await vacuousTest(false); + check('(n) with `await clock.advance(10_000)`', withLine, 'ok'); + check('(n) with that line DELETED', withoutLine, 'pending'); + note( + '(n) → the two differ, so the advance is not literally dead code — it is worse. It drives the call to COMPLETION while the budget it is supposed to exhaust never drains. A suite asserting `r.ok === false` here fails; a suite asserting the call finishes passes and believes it proved the budget', + '', + ); + } + + // ── the table ─────────────────────────────────────────────────────────────────────────────── + heading('C2 — the table'); + const tally = printClockTable(); + note('rows driven by the injected clock', tally.clock); + note('rows on wall clock regardless', tally.wall); + note('rows with no time in them', tally.none); + check( + 'every time-driven feature is accounted for', + tally.clock + tally.wall + tally.none, + 14, + ); + checkSeq( + 'the wall-clock set', + [ + 'timeout.total', + 'cache.ttl', + 'memoryStore TTL', + 'oauth2 token expiry', + 'AWS SigV4 signing date', + 'event `at` / `done.elapsed`', + ].sort(), + [ + 'AWS SigV4 signing date', + 'cache.ttl', + 'event `at` / `done.elapsed`', + 'memoryStore TTL', + 'oauth2 token expiry', + 'timeout.total', + ], + ); + + finish( + 'C2', + 'THE SUSPICION IS CONFIRMED AND THE SCOPE IS WIDER THAN RECORDED. Driven by `manualClock`: retry backoff, throttle rate, throttle concurrency, `circuit.cooldown`, the per-attempt `timeout`, and `Retry-After`. On wall clock regardless: `timeout.total`, `cache.ttl`, the `memoryStore` TTL beneath it, event `at`/`done.elapsed` — and TWO the pass had not recorded: OAuth2 token expiry (`auth.ts:502` gates freshness on wall `now()`, so advancing 600000 virtual ms past a 60s `expires_in` refetches nothing) and AWS SigV4 signing (`new Date()`, with no `clock` option to pass). `paginate` has no time in it to drive. The measured cost of the `timeout.total` row: a call configured to die after 1000ms consumed 2700 virtual ms across 3 attempts and returned ok=true, because the budget deadline is wall-anchored and virtual sleeps never drain it. And one row is a trap rather than a gap: `parseRetryAfter` DOES read the injected clock, so an HTTP-date `Retry-After` under `manualClock()` (which starts at 0) becomes a wait of ~20,000 DAYS instead of 5 seconds', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c3-mock-fidelity.ts b/docs/scenarios/proofs/stale-fixture/c3-mock-fidelity.ts new file mode 100644 index 00000000..1e7ea46d --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c3-mock-fidelity.ts @@ -0,0 +1,299 @@ +// C3 — what does `mockAdapter` actually CHECK about a fixture? +// +// The question matters because a mock that accepts a response no transport could produce lets you +// write code against a shape that cannot exist. The library HAS a published notion of what a +// well-formed transport does — `verifyAdapterContract` + `adapterContractFixture` — so the sharp +// version of the question is: does `mockAdapter` hold itself to the contract it publishes for +// everybody else? +// +// Measured: it normalises the two fields it owns (`status` defaults to 200, header names are +// lowercased) and performs ZERO validation of anything else. It will serve a `Date`, a `Map`, a +// class instance, `undefined`, and a status of `999` or `-1` — none of which survives a JSON +// transport. The gap is `body`, and it is not a small one, because `body` is the entire fixture. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c3-mock-fidelity.ts +import { stitch } from '../../../../packages/core/src/index'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import { + adapterContractFixture, + verifyAdapterContract, +} from '../../../../packages/core/src/testing'; +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { BASE } from './vendor'; + +/** Call a mock adapter directly and report the raw `AdapterResponse` it produced. */ +async function raw( + adapter: Adapter, + req: Partial = {}, +): Promise { + return adapter({ + url: `${BASE}/x`, + method: 'GET', + headers: {}, + ...req, + }); +} + +/** `typeof`, but distinguishing the shapes JSON can and cannot carry. */ +function shapeOf(v: unknown): string { + if (v === null) return 'null'; + if (v === undefined) return 'undefined'; + if (Array.isArray(v)) return 'array'; + if (v instanceof Date) return 'Date'; + if (v instanceof Map) return 'Map'; + if (typeof v === 'object') return (v.constructor as { name: string }).name; + return typeof v; +} + +class Invoice { + constructor(public id: string) {} + get total(): number { + return 42; + } +} + +async function main(): Promise { + heading('C3 (a) — the two fields `mockAdapter` DOES normalise'); + { + const a = mockAdapter([{ respond: { body: { ok: true } } }]); + const res = await raw(a); + check('status defaults to 200', res.status, 200); + checkSeq('headers default to {}', Object.keys(res.headers), []); + + const b = mockAdapter([ + { + respond: { + headers: { + 'Content-Type': 'application/json', + 'X-Trace': 'abc', + }, + body: {}, + }, + }, + ]); + const resB = await raw(b); + checkSeq( + 'header NAMES are lowercased, matching a real adapter', + Object.keys(resB.headers).sort(), + ['content-type', 'x-trace'], + ); + note( + '(a) → test-mock.ts:149-161 `build()`. That is the whole of its response hygiene', + '', + ); + } + + heading('C3 (b) — statuses no HTTP transport can produce'); + { + const rows: string[] = []; + for (const status of [999, -1, 0, 1.5, 200.7]) { + const a = mockAdapter([{ respond: { status, body: {} } }]); + const res = await raw(a); + rows.push(`${String(status)} -> ${String(res.status)}`); + } + checkSeq('status passthrough', rows, [ + '999 -> 999', + '-1 -> -1', + '0 -> 0', + '1.5 -> 1.5', + '200.7 -> 200.7', + ]); + note( + '(b) → `adapterContractFixture` only ever emits 100..599 integers (testing.ts:613-617), and `verifyAdapterContract` probes 200/404/500. `mockAdapter` is bound by neither', + '', + ); + } + + heading('C3 (c) — bodies no JSON transport can produce'); + { + const bodies: [string, unknown][] = [ + ['Date', new Date(0)], + ['Map', new Map([['a', 1]])], + ['class instance', new Invoice('inv_1')], + ['undefined', undefined], + ['function', () => 1], + ['bigint', 10n], + ['Symbol-keyed', { [Symbol('s')]: 1 }], + ]; + const rows: string[] = []; + for (const [label, body] of bodies) { + const a = mockAdapter([{ respond: { body } as never }]); + const res = await raw(a); + rows.push(`${label} -> ${shapeOf(res.body)}`); + } + checkSeq('what came back out', rows, [ + 'Date -> Date', + 'Map -> Map', + 'class instance -> Invoice', + 'undefined -> undefined', + 'function -> function', + 'bigint -> bigint', + 'Symbol-keyed -> Object', + ]); + note( + '(c) → every one is served verbatim. A real adapter hands the engine `JSON.parse` output or a string, so `Date`/`Map`/`Invoice`/`function`/`bigint` are shapes production can never deliver', + '', + ); + } + + heading('C3 (d) — and the engine believes them'); + // The consequence: a fixture with methods lets you write `data.total` in a test that passes and + // in production throws, because the wire only ever carried `{"id":"inv_1"}`. + { + const a = mockAdapter([ + { respond: { body: new Invoice('inv_1') } as never }, + ]); + const call = stitch({ url: `${BASE}/inv`, adapter: a }); + const r = await call.safe(); + check('the call succeeded', r.ok, true); + check( + 'the caller received a live class instance', + r.data instanceof Invoice, + true, + ); + check( + '…with a GETTER that exists only in the test', + (r.data as Invoice).total, + 42, + ); + check( + 'what the same object looks like over a JSON wire', + JSON.stringify(JSON.parse(JSON.stringify(new Invoice('inv_1')))), + '{"id":"inv_1"}', + ); + note( + '(d) → `data.total` is 42 under the mock and `undefined` in production. That is the "mock lets you write code that cannot work" shape, in the response direction', + '', + ); + } + + heading( + 'C3 (e) — the contract the library DOES publish, run against the mock', + ); + // `verifyAdapterContract` is the library's own definition of a well-formed transport. Point it + // at a `mockAdapter` wired to serve `adapterContractFixture` and see whether the mock passes. + { + const fixtureAdapter = mockAdapter([ + { + respond: (call) => { + const u = new URL(call.req.url); + const out = adapterContractFixture({ + method: call.req.method, + path: u.pathname + u.search, + headers: call.req.headers, + ...(call.req.body === undefined + ? {} + : { body: JSON.stringify(call.req.body) }), + }); + const parsed: unknown = out.headers[ + 'content-type' + ]?.includes('json') + ? JSON.parse(out.body) + : out.body; + return { + status: out.status, + headers: out.headers, + body: parsed, + ...(out.delay === undefined + ? {} + : { delay: out.delay }), + }; + }, + }, + ]); + const report = await verifyAdapterContract(fixtureAdapter, BASE); + check('seam verified', report.seam, 'adapter'); + note('rules passed', report.passed.length); + note('rules violated', report.violations.length); + for (const v of report.violations) + note(`violation: ${v.rule}`, v.detail); + checkSeq('the rules a `mockAdapter` CAN satisfy', report.passed, [ + 'status: 200 resolves with status 200', + 'status: 404 resolves without throwing', + 'status: 500 resolves without throwing', + 'request: method, headers, and body are delivered', + 'response: headers are readable with lowercase names', + 'response: text body round-trips', + 'response: JSON body round-trips as parsed data', + 'abort: an in-flight abort rejects promptly', + ]); + checkSeq( + 'the rule it CANNOT', + report.violations.map((v) => v.rule), + ['abort: a pre-aborted signal rejects'], + ); + checkSeq( + '…and the detail', + report.violations.map((v) => v.detail), + ['adapter resolved although the signal was already aborted'], + ); + note( + '(e) → 8 of 9 rules pass, and the ONE failure is a genuine fidelity bug, not an artifact of this harness: `mockAdapter` consults `req.signal` only inside its `delay` branch (test-mock.ts:188-189), so a route with no `delay` RESOLVES for a request whose signal was already aborted. Every real transport rejects. But read what the 9 rules cover — statuses, header case, body round-tripping, abort. NONE constrains what a fixture MAY contain; they constrain what a TRANSPORT must do with it. The contract kit is for adapter authors, exactly as the capture predicted', + '', + ); + } + + heading('C3 (f) — the one shape `mockAdapter` DOES reject at compile time'); + { + // `respond: {}` is a type error (`AtLeastOne`, CONTRACT.md P20). It is the + // only fixture shape the mock refuses, and it is refused by the type system, not at runtime. + // @ts-expect-error — the opaque `respond: {}` is rejected by AtLeastOne + const _rejected = mockAdapter([{ respond: {} }]); + void _rejected; + check( + 'an empty `respond` is a compile error (see the @ts-expect-error above)', + true, + true, + ); + + // At RUNTIME, however, nothing stops the same thing arriving through a responder function. + const viaFn = mockAdapter([{ respond: () => ({}) }]); + const res = await raw(viaFn); + check('…but a responder FUNCTION may return `{}`', res.status, 200); + check('…and the body is `undefined`', res.body, undefined); + note( + '(f) → the guard is a type, so it is defeated by the one form the type deliberately relaxes. A JSON transport cannot deliver `undefined`', + '', + ); + } + + heading('C3 (g) — unmatched requests'); + { + const strict = mockAdapter([ + { match: '/known', respond: { body: {} } }, + ]); + let msg = ''; + try { + await raw(strict, { url: `${BASE}/unknown` }); + } catch (e) { + msg = (e as Error).message; + } + check( + 'the default is a loud throw', + msg, + `mockAdapter: no route matched GET ${BASE}/unknown`, + ); + const lax = mockAdapter([{ match: '/known', respond: { body: {} } }], { + onUnmatched: 404, + }); + const res = await raw(lax, { url: `${BASE}/unknown` }); + check('`onUnmatched` replies with a bare status', res.status, 404); + check('…and a body of `{}`', JSON.stringify(res.body), '{}'); + note( + '(g) → this is the one place the mock is stricter than a real vendor by default, and it is the right default: an unexpected request fails the test instead of being silently absorbed', + '', + ); + } + + finish( + 'C3', + 'IT VALIDATES ALMOST NOTHING, AND THE GAP IS EXACTLY THE FIXTURE. `mockAdapter` normalises two fields — `status` defaults to 200 and header names are lowercased (test-mock.ts:149-161) — and checks nothing else. Measured: it served statuses 999, -1, 0, 1.5 and 200.7 verbatim, and served a `Date`, a `Map`, a class instance, `undefined`, a `function`, a `bigint` and a Symbol-keyed object as response bodies, all of which reached the caller unchanged through the full engine. The consequence is a working test for code that cannot work: a fixture built from `new Invoice(...)` gives the caller `data.total === 42` from a GETTER, where the same object over a JSON wire is `{"id":"inv_1"}` and `data.total` is undefined. The library does publish a transport contract, and running it against `mockAdapter` turns up a SECOND finding the claim did not ask for: the mock passes 8 of 9 rules and VIOLATES `abort: a pre-aborted signal rejects` — "adapter resolved although the signal was already aborted" — because it consults `req.signal` only inside its `delay` branch (test-mock.ts:188-189), so any route without a `delay` ignores an aborted signal that every real transport honours. And every one of the 9 rules constrains what a TRANSPORT must do with a response, not what a FIXTURE may contain, so passing them says nothing about fixture realism. The single fixture shape that is rejected, `respond: {}`, is rejected by the TYPE (`AtLeastOne`) and is reachable at runtime anyway through a responder function, which returns `status: 200, body: undefined`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c4-resilience-events.ts b/docs/scenarios/proofs/stale-fixture/c4-resilience-events.ts new file mode 100644 index 00000000..cd2c4497 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c4-resilience-events.ts @@ -0,0 +1,326 @@ +// C4 — can resilience be tested with NO vendor at all? +// +// This is the claim the testing kit is built for, and it is the one it answers best. Attempt +// counts, circuit transitions and throttle spacing are all assertable offline, deterministically, +// with zero real waiting: `mockAdapter` supplies the responses, `manualClock` supplies the time, +// and `collectStitchEvents` drains the event stream into something you can compare. +// +// The interesting part is where the event stream STOPS carrying what you would want to assert. +// Measured: a `progress` event carries `waited` for a throttle wait and for a stream reconnect, but +// NOT for a retry backoff — so "the second attempt waited 2000ms" is not readable from events. It +// is recoverable, but only by reading `clock.now()` yourself, which means the backoff assertion is +// the one resilience assertion the kit does not hand you. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c4-resilience-events.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/test-clock'; +import { collectStitchEvents } from '../../../../packages/core/src/test-events'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import type { StitchEvent } from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { BASE } from './vendor'; + +/** Drain an event stream while driving the clock, so a backoff resolves without real waiting. */ +async function drainUnderClock( + gen: AsyncIterable>, + clock: { advance(ms: number): Promise }, + stepMs: number, + steps: number, +): Promise[]> { + const events: StitchEvent[] = []; + const it = gen[Symbol.asyncIterator](); + for (;;) { + const nextP = it.next(); + // Race the pull against the clock: whichever the engine is waiting on, advancing frees it. + let settled = false; + void nextP.then(() => { + settled = true; + }); + for (let i = 0; i < steps && !settled; i++) await clock.advance(stepMs); + const r = await nextP; + if (r.done) break; + events.push(r.value); + } + return events; +} + +async function main(): Promise { + heading('C4 (a) — attempt counts, with no vendor'); + { + const clock = manualClock(); + const api = mockAdapter([ + { + match: '/flaky', + respond: [ + { status: 503 }, + { status: 503 }, + { body: { ok: true } }, + ], + }, + ]); + const call = stitch({ + url: `${BASE}/flaky`, + adapter: api, + retry: { attempts: 3, backoff: { curve: 'expo', base: 1000 } }, + clock, + }); + const events = await drainUnderClock(call().stream(), clock, 1000, 20); + const c = await collectStitchEvents( + (async function* () { + for (const e of events) yield e; + })(), + ); + checkSeq('the event spine', c.types, [ + 'start', + 'progress', + 'progress', + 'progress', + 'progress', + 'progress', + 'result', + 'done', + ]); + check('requests the transport saw', api.callCount(), 3); + check( + 'the `result` event reports attempts', + (c.events.find((e) => e.type === 'result') as { attempts: number }) + .attempts, + 3, + ); + check('done.ok', c.done?.ok, true); + checkSeq( + 'the progress phases', + c.events + .filter((e) => e.type === 'progress') + .map((e) => (e as { phase: string }).phase), + ['request', 'retry', 'request', 'retry', 'request'], + ); + note( + '(a) → attempts are assertable three ways: `mockAdapter.callCount()`, `result.attempts`, and the count of `progress{phase:"request"}` events', + '', + ); + } + + heading('C4 (b) — backoff DELAYS: what the events carry'); + { + const clock = manualClock(); + const api = mockAdapter([ + { + match: '/flaky', + respond: [ + { status: 503 }, + { status: 503 }, + { body: { ok: true } }, + ], + }, + ]); + const call = stitch({ + url: `${BASE}/flaky`, + adapter: api, + retry: { + attempts: 3, + backoff: { curve: 'expo', base: 1000 }, + respect: false, + }, + clock, + }); + // Record VIRTUAL time at each request the transport receives — the honest measurement of + // spacing, since the events do not carry it. + const at: number[] = []; + const spy = mockAdapter([ + { + match: '/flaky', + respond: (c) => { + at.push(clock.now()); + return c.index < 2 + ? { status: 503 } + : { body: { ok: true } }; + }, + }, + ]); + const spied = stitch({ + url: `${BASE}/flaky`, + adapter: spy, + retry: { + attempts: 3, + backoff: { curve: 'expo', base: 1000 }, + respect: false, + }, + clock, + }); + void spied.safe(); + await clock.advance(100_000); + + const events = await drainUnderClock(call().stream(), clock, 1000, 40); + const retryProgress = events.filter( + (e) => e.type === 'progress' && e.phase === 'retry', + ) as { waited?: number; detail?: string }[]; + check('retry `progress` events', retryProgress.length, 2); + checkSeq( + '…and the `waited` each carries', + retryProgress.map((e) => e.waited), + [undefined, undefined], + ); + checkSeq( + '…what they DO carry', + retryProgress.map((e) => e.detail), + ['status 503', 'status 503'], + ); + checkSeq('virtual time at each request', at, [0, 1000, 3000]); + checkSeq( + 'the gaps, derived', + at.slice(1).map((t, i) => t - (at[i] as number)), + [1000, 2000], + ); + note( + '(b) → the `expo` curve is EXACT under a manual clock — 1000 then 2000 — but it is readable only from `clock.now()`. `progress{phase:"retry"}` carries `detail` and no `waited`, so the backoff delay is the one resilience number the event stream does not report (engine.ts:682-688 omits it; engine.ts:641 and :1486 set it for throttle and reconnect)', + '', + ); + } + + heading('C4 (c) — throttle spacing DOES carry `waited`'); + { + const clock = manualClock(); + const api = mockAdapter([{ respond: { body: { ok: true } } }]); + const call = stitch({ + url: `${BASE}/paced`, + adapter: api, + throttle: '2/s', // a 500ms minimum spacing + clock, + }); + const seen: number[] = []; + const waits: (number | undefined)[] = []; + for (let i = 0; i < 3; i++) { + void (async () => { + for await (const e of call().stream()) { + if (e.type === 'progress' && e.phase === 'throttled') { + waits.push(e.waited); + } + if (e.type === 'progress' && e.phase === 'request') + seen.push(clock.now()); + } + })(); + } + await clock.advance(5000); + checkSeq('virtual time at each request', seen, [0, 500, 1000]); + checkSeq('the `waited` the throttle reported', waits, [500, 1000]); + note( + "(c) → `'2/s'` is a 500ms minimum spacing (ADR 0023), and both the spacing and the reported wait are exact. This is the assertion the kit does hand you", + '', + ); + } + + heading( + 'C4 (d) — circuit transitions: closed -> open -> half-open -> closed', + ); + { + const clock = manualClock(); + let mode: 'fail' | 'heal' = 'fail'; + const api = mockAdapter([ + { + respond: () => + mode === 'fail' ? { status: 500 } : { body: { ok: true } }, + }, + ]); + const call = stitch({ + url: `${BASE}/breaker`, + adapter: api, + circuit: { failures: 2, cooldown: 30_000 }, + clock, + }); + const transitions: string[] = []; + + await call.safe(); + transitions.push(`after 1 failure: calls=${String(api.callCount())}`); + await call.safe(); + transitions.push(`after 2 failures: calls=${String(api.callCount())}`); + + const blocked = await call.safe(); + transitions.push( + `while OPEN: calls=${String(api.callCount())} err=${blocked.error?.name ?? ''}`, + ); + + await clock.advance(30_000); + mode = 'heal'; + const probe = await call.safe(); + transitions.push( + `after cooldown: calls=${String(api.callCount())} ok=${String(probe.ok)}`, + ); + + const after = await call.safe(); + transitions.push( + `once CLOSED: calls=${String(api.callCount())} ok=${String(after.ok)}`, + ); + + checkSeq('the full transition trace', transitions, [ + 'after 1 failure: calls=1', + 'after 2 failures: calls=2', + 'while OPEN: calls=2 err=StitchError', + 'after cooldown: calls=3 ok=true', + 'once CLOSED: calls=4 ok=true', + ]); + check( + 'the open circuit blocked the request entirely', + blocked.error?.message, + 'circuit open', + ); + note( + '(d) → every transition is assertable by `callCount()` alone: the OPEN state is visible because the transport count does NOT move. No vendor, no waiting', + '', + ); + } + + heading( + 'C4 (e) — what a `done` event actually carries under a manual clock', + ); + { + const clock = manualClock(); + const api = mockAdapter([{ respond: { body: { ok: true } } }]); + const call = stitch({ url: `${BASE}/x`, adapter: api, clock }); + const c = await collectStitchEvents(call()); + const done = c.events.find((e) => e.type === 'done') as { + elapsed: number; + attempts: number; + at: number; + }; + check('done.attempts', done.attempts, 1); + check('done.elapsed under a manual clock', done.elapsed, 0); + check('clock.now()', clock.now(), 0); + note('done.at (a wall-clock epoch read)', done.at); + note( + "(e) → `elapsed` is wall-clock (C2 row 9), so under `manualClock` it reads 0 no matter how much virtual time the call consumed. An assertion like `expect(done.elapsed).toBeGreaterThan(3000)` after advancing 5s FAILS — the kit's own duration field cannot see the kit's own clock", + '', + ); + } + + heading('C4 (f) — the CollectedEvents surface'); + { + const api = mockAdapter([{ respond: { status: 500, body: { e: 1 } } }]); + const call = stitch({ url: `${BASE}/bad`, adapter: api }); + const c = await collectStitchEvents(call()); + checkSeq('fields', Object.keys(c).sort(), [ + 'deltas', + 'done', + 'drifts', + 'error', + 'events', + 'result', + 'types', + ]); + check('error.message', c.error?.message, 'HTTP 500'); + check('error.status', c.error?.status, 500); + check('done.ok', c.done?.ok, false); + check('result', c.result, undefined); + note( + '(f) → `collectStitchEvents` accepts the `StitchResult` directly (no `.stream()` call needed) and reduces it to 7 fields. `error` is flattened to `{message,status}`; the full event is still in `events`', + '', + ); + } + + finish( + 'C4', + 'CONFIRMED, WITH ONE GAP AND ONE TRAP. Resilience is fully testable with no vendor and no real waiting. Attempt counts: assertable three ways (`mockAdapter.callCount()` = 3, `result.attempts` = 3, five `progress` events phased `request,retry,request,retry,request`). Circuit transitions: the whole closed->open->half-open->closed trace is readable from `callCount()` — 1, 2, 2 (blocked, a plain `StitchError` whose message is "circuit open" — the breaker has no distinct error NAME to assert on), 3 (probe after `advance(30_000)`), 4 — because an open circuit does not move the transport count. Throttle spacing: exact, and self-reporting — requests at virtual 0/500/1000 for `\'2/s\'`, with `progress{phase:"throttled"}.waited` reading 500 then 1000. THE GAP: retry backoff delays are exact under the clock (requests at virtual 0/1000/3000, gaps 1000 and 2000 for an `expo` base-1000 curve) but are NOT in the event stream — `progress{phase:"retry"}` carries `detail: "status 503"` and `waited: undefined`, where the throttle and reconnect paths do set `waited`. You can assert the backoff, but only by reading `clock.now()` yourself. THE TRAP: `done.elapsed` is wall-clock, so it reads 0 after any amount of virtual time — the kit\'s own duration field cannot see the kit\'s own clock', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c5-stub-contract.ts b/docs/scenarios/proofs/stale-fixture/c5-stub-contract.ts new file mode 100644 index 00000000..e0b95194 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c5-stub-contract.ts @@ -0,0 +1,317 @@ +// C5 — `stubStitch` / `failStitch`: does the stub honour the same INPUT contract as the real stitch? +// +// This is the classic "mock lets you write code that cannot work" asymmetry, in the request +// direction. C3 measured it on the way back (a fixture can be a shape no wire produces); this is the +// way out. If the real stitch validates `input` and the stub does not, then a service tested against +// the stub can send `{ id: 42 }` where the endpoint demands `{ id: "42" }`, and the test is green. +// +// Measured: the stub is a FAITHFUL `Stitch` in every structural respect — `isStitch()` accepts it, +// `.safe`/`.unwrap`/`.stream`/`.with`/`.cache`/`__config` are all there, the event spine matches — +// and it runs ZERO of the six declared `input` slots. A stitch that rejects `{ id: 42 }` with a +// named validation error is replaced by a stub that resolves and records the call. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c5-stub-contract.ts +import { stitch } from '../../../../packages/core/src/index'; +import { collectStitchEvents } from '../../../../packages/core/src/test-events'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import { + failStitch, + stubStitch, +} from '../../../../packages/core/src/test-stub'; +import { isStitch } from '../../../../packages/core/src/types'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { BASE } from './vendor'; +import { z } from './zod'; + +interface Invoice { + id: string; + amount_cents: number; +} + +/** THE CODE UNDER TEST. It takes a stitch and calls it — it does not care which one it got. */ +async function chargeReport( + getInvoice: Stitch, + id: unknown, +): Promise { + const r = await getInvoice.safe({ params: { id } as never }); + return r.ok + ? `${r.data.id}:${String(r.data.amount_cents)}` + : `ERR ${r.error.message}`; +} + +async function main(): Promise { + // The REAL stitch: `id` must be a string of digits, and the endpoint 404s on anything else. + const api = mockAdapter([ + { + match: /\/v1\/invoices\/\d+$/, + respond: { body: { id: 'inv_1', amount_cents: 4200 } }, + }, + ]); + const real = stitch({ + name: 'getInvoice', + baseUrl: BASE, + path: '/v1/invoices/{id}', + adapter: api, + input: { params: z.object({ id: z.string() }) }, + output: z.object({ id: z.string(), amount_cents: z.number() }), + }) as unknown as Stitch; + + heading('C5 (a) — the real stitch enforces `input`'); + { + const good = await chargeReport(real, '42'); + check('a valid id works', good, 'inv_1:4200'); + const bad = await chargeReport(real, 42); + check( + 'a NUMBER id is rejected before the wire', + bad.startsWith('ERR'), + true, + ); + check('…and the transport never saw it', api.callCount(), 1); + note('the rejection message', bad); + } + + heading('C5 (b) — the stub accepts what the real stitch refuses'); + { + const stub = stubStitch({ id: 'inv_1', amount_cents: 4200 }); + const viaStub = await chargeReport(stub, 42); + check( + 'the stub RESOLVED for the same bad input', + viaStub, + 'inv_1:4200', + ); + check('calls recorded', stub.callCount(), 1); + checkSeq('…and what it recorded', stub.calls(), [ + { params: { id: 42 } }, + ]); + note( + '(b) → same code, same argument. Against the real stitch: a validation error, no request. Against the stub: a green test and a plausible answer', + '', + ); + } + + heading('C5 (c) — can a stub be GIVEN the input contract?'); + { + // The published options are `name`, `status`, `config`, `events`. None takes a schema. + const stub = stubStitch( + { id: 'inv_1', amount_cents: 4200 }, + { + name: 'getInvoice', + status: 200, + config: { name: 'getInvoice', method: 'GET' }, + }, + ); + const cfgKeys = Object.keys(stub.__config).sort(); + checkSeq('`__config` on a stub', cfgKeys, ['method', 'name']); + check( + 'is there an `input` slot on StubStitchOptions?', + 'input' in + ({ + name: '', + status: 200, + config: {}, + events: () => [], + } as Record), + false, + ); + note( + '(c) → `StubStitchOptions` is `{ name, status, config, events }` (test-stub.ts:31-42). `config` is `Partial`, which is the REDACTED read-out shape — it carries no schema, so there is nowhere to put the contract even if you wanted to', + '', + ); + + // The workaround is your own: wrap the impl in a validator. It works — but only if the impl + // is `async`, for the reason section (g) measures. + const Params = z.object({ id: z.string() }); + const guarded = stubStitch(async (input) => { + Params.parse(input.params); + return { id: 'inv_1', amount_cents: 4200 }; + }); + const ok = await chargeReport(guarded, '42'); + const rejected = await chargeReport(guarded, 42); + check('(c) a hand-guarded stub accepts good input', ok, 'inv_1:4200'); + check( + '(c) …and rejects the bad input the real stitch rejects', + rejected.startsWith('ERR'), + true, + ); + note( + '(c) → the `impl` function is the seam. It receives the raw `StitchInput`, so a validator can run there — but you have to remember to write it, nothing tells you that you did not, and it has to be `async` (see (g))', + '', + ); + } + + heading('C5 (d) — everything ELSE about the stub is faithful'); + { + const stub = stubStitch({ id: 'inv_1', amount_cents: 4200 }); + check('isStitch() accepts it', isStitch(stub), true); + checkSeq( + 'the callable surface', + ['safe', 'unwrap', 'stream', 'with', 'cache', 'invalidate'].map( + (k) => + `${k}=${typeof (stub as unknown as Record)[k]}`, + ), + [ + 'safe=function', + 'unwrap=function', + 'stream=function', + 'with=function', + 'cache=object', + 'invalidate=function', + ], + ); + const c = await collectStitchEvents(stub()); + checkSeq('the event spine matches a real run', c.types, [ + 'start', + 'result', + 'done', + ]); + check( + 'result data', + JSON.stringify(c.result), + '{"id":"inv_1","amount_cents":4200}', + ); + check('done.ok', c.done?.ok, true); + + const failing = failStitch({ status: 503, message: 'vendor down' }); + const f = await collectStitchEvents(failing()); + checkSeq('failStitch spine', f.types, ['start', 'error', 'done']); + check('failStitch error message', f.error?.message, 'vendor down'); + check('failStitch error status', f.error?.status, 503); + check('failStitch done.ok', f.done?.ok, false); + note( + '(d) → structurally this is a real `Stitch`: a registry, `isStitch()`, or a Nest `overrideProvider(useValue:)` accepts it, and `.stream()` yields the same spine. The ONLY thing missing is the contract', + '', + ); + } + + heading( + 'C5 (e) — the real stitch and the stub disagree on the event spine, once', + ); + { + // A real run emits `progress` events; a stub does not. So a test that asserts on `types` + // against a stub encodes a spine production never produces. + const realEvents = await collectStitchEvents( + ( + stitch({ + url: `${BASE}/x`, + adapter: mockAdapter([{ respond: { body: { ok: true } } }]), + }) as unknown as Stitch + )(), + ); + const stubEvents = await collectStitchEvents( + stubStitch({ ok: true })(), + ); + checkSeq('real spine', realEvents.types, [ + 'start', + 'progress', + 'result', + 'done', + ]); + checkSeq('stub spine', stubEvents.types, ['start', 'result', 'done']); + note( + '(e) → the stub omits `progress`. Harmless for a caller test, but it means the two spines are not interchangeable in an assertion', + '', + ); + } + + heading('C5 (f) — `.with()` on a stub'); + { + const stub = stubStitch((input) => ({ + id: String((input.params as { id: unknown }).id), + amount_cents: 1, + })); + const bound = stub.with({ params: { id: 'pinned' } } as never); + const r = await bound.safe({} as never); + check( + 'the bound partial reached the impl', + r.ok && r.data.id, + 'pinned', + ); + check('the PARENT stub recorded the call', stub.callCount(), 0); + note( + "(f) → `.with()` on a stub returns a NEW stub with its own spy (test-stub.ts:187-191 re-`assemble`s), so the parent's `callCount()` stays 0. A test that binds and then asserts on the original spy sees nothing", + '', + ); + } + + heading('C5 (g) — a BUG: `.safe()` on a stub can throw'); + // `.safe()` is the never-throws surface. The real stitch honours that even when the transport + // throws synchronously. The stub does not: `resolve()` (test-stub.ts:59-63) evaluates + // `impl(input)` as an ARGUMENT to `Promise.resolve`, so a synchronous throw escapes the promise + // chain before there is a chain to catch it. + { + const syncThrow = stubStitch(() => { + throw new Error('boom'); + }); + let outcome = ''; + try { + const r = await syncThrow.safe(); + outcome = `returned ok=${String(r.ok)}`; + } catch (e) { + outcome = `THREW ${(e as Error).message}`; + } + check( + '(g) stub .safe() with a SYNC-throwing impl', + outcome, + 'THREW boom', + ); + + const asyncThrow = stubStitch(() => + Promise.reject(new Error('boom')), + ); + let asyncOutcome = ''; + try { + const r = await asyncThrow.safe(); + asyncOutcome = `returned ok=${String(r.ok)}`; + } catch (e) { + asyncOutcome = `THREW ${(e as Error).message}`; + } + check( + '(g) …and with an ASYNC-rejecting impl', + asyncOutcome, + 'returned ok=false', + ); + + // The real stitch, same shape: a transport that throws synchronously. + const realSync = stitch({ + url: `${BASE}/x`, + adapter: () => { + throw new Error('transport boom'); + }, + }); + const realOut = await realSync.safe(); + check( + '(g) real stitch .safe() with a SYNC-throwing adapter', + realOut.ok, + false, + ); + check( + '(g) …and it carries the message', + realOut.error?.message, + 'transport boom', + ); + + // `.stream()` on the same stub is fine — the generator has a try/catch. + const streamed = await collectStitchEvents( + stubStitch(() => { + throw new Error('boom'); + })().stream(), + ); + checkSeq('(g) …but `.stream()` handles it correctly', streamed.types, [ + 'start', + 'error', + 'done', + ]); + note( + '(g) → `SafeResult` exists so a caller never needs a try/catch. A stub whose impl throws synchronously breaks that, where the real stitch it replaces does not — so the code under test needs a try/catch that production does not need, and the fix (`async (input) => …`) is invisible at the call site', + '', + ); + } + + finish( + 'C5', + 'THE ASYMMETRY IS REAL AND IT IS THE WHOLE INPUT CONTRACT. Structurally the stub is faithful: `isStitch()` accepts it, `safe`/`unwrap`/`stream`/`with`/`invalidate` are functions and `cache` is an object, `.stream()` yields `start,result,done`, and `failStitch({status:503,message:"vendor down"})` yields `start,error,done` with the message and status intact. But it runs NONE of the `input` schemas. Measured on one line of calling code with the argument `42` where the schema says `z.string()`: the real stitch returns an error and the transport count stays at 1 (no request left the process); the stub RESOLVES to `inv_1:4200` and records `{"params":{"id":42}}`. There is no way to hand a stub the contract either — `StubStitchOptions` is `{name,status,config,events}` and `config` is `Partial`, the redacted read-out shape, which carries no schema. The workaround is to validate inside the `impl` function yourself (3 lines, and it does work), which is exactly the kind of thing nobody remembers. Two smaller divergences: a real run emits a `progress` event the stub omits, so the two spines are not interchangeable; and `.with()` returns a NEW stub with a fresh spy, so the parent\'s `callCount()` reads 0 after a bound call. AND A BUG, found in passing: `.safe()` on a stub whose impl throws SYNCHRONOUSLY throws instead of resolving — measured "THREW boom" where the same stub with an async rejection returns ok=false and the REAL stitch with a synchronously-throwing adapter returns ok=false / "transport boom". `resolve()` (test-stub.ts:59-63) evaluates `impl(input)` as an argument to `Promise.resolve`, so the throw escapes before there is a chain to catch it. `.stream()` on the same stub is unaffected (`start,error,done`)', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c6-sandbox-parity.ts b/docs/scenarios/proofs/stale-fixture/c6-sandbox-parity.ts new file mode 100644 index 00000000..e507f602 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c6-sandbox-parity.ts @@ -0,0 +1,310 @@ +// C6 — sandbox parity. Can one stitch be pointed at sandbox AND prod so the difference is visible? +// And is there a spelling for "run this against the real API weekly"? +// +// Two questions with two different answers, and the split is the finding. +// +// Targeting: YES, and cleanly. `extends` swaps `baseUrl` and `adapter` wholesale, so ONE endpoint +// definition can be aimed at a fixture, a sandbox and prod with a one-line fragment. +// `.with()` cannot — it is `Partial` and a `baseUrl` there is a compile +// error, which is the right answer to the wrong question. +// Scheduling: NO. There is no `env`/`sandbox`/`profile` key (an unknown key is a compile error), +// no CLI verb that runs a verification, and nothing that records when a check last ran. +// The comparison is 100% yours to write and yours to remember to run. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c6-sandbox-parity.ts +import { seam, stitch } from '../../../../packages/core/src/index'; +import type { Stitch, StitchConfig } from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { RECORDED_2026_02_04, VENDOR_TODAY, vendorAdapter } from './vendor'; +import { z } from './zod'; + +import { readFileSync } from 'node:fs'; + +const SANDBOX = 'https://sandbox.billing.test'; +const PROD = 'https://api.billing.test'; + +const Invoice = z.object({ + id: z.string(), + amount_cents: z.number(), + currency: z.string(), + paid: z.boolean(), + customer_email: z.string(), + legacy_ref: z.string(), +}); + +/** The endpoint, defined ONCE. Everything environment-shaped is left to a fragment. */ +const ENDPOINT = { + name: 'getInvoice', + path: '/v1/invoices/{id}', + output: Invoice, +} satisfies Partial; + +async function main(): Promise { + heading('C6 (a) — `extends`: one endpoint, three targets'); + { + const fixture = vendorAdapter(RECORDED_2026_02_04); + const sandbox = vendorAdapter(RECORDED_2026_02_04); + const prod = vendorAdapter(VENDOR_TODAY); + + const against = (baseUrl: string, adapter: typeof fixture): Stitch => + stitch({ + ...ENDPOINT, + extends: { baseUrl, adapter }, + } as never) as unknown as Stitch; + + const a = against(PROD, fixture); + const b = against(SANDBOX, sandbox); + const c = against(PROD, prod); + const ra = await a.safe({ params: { id: 'inv_9f2' } }); + const rb = await b.safe({ params: { id: 'inv_9f2' } }); + const rc = await c.safe({ params: { id: 'inv_9f2' } }); + + checkSeq( + 'the three URLs the transports saw', + [fixture.seen[0]?.url, sandbox.seen[0]?.url, prod.seen[0]?.url], + [ + `${PROD}/v1/invoices/inv_9f2`, + `${SANDBOX}/v1/invoices/inv_9f2`, + `${PROD}/v1/invoices/inv_9f2`, + ], + ); + checkSeq( + 'and the three verdicts', + [ra.ok, rb.ok, rc.ok], + [true, true, false], + ); + note( + '(a) → one `ENDPOINT` object, three `extends: { baseUrl, adapter }` fragments. The fixture and the sandbox agree; PROD fails the same `output` schema. THAT is the difference being visible', + '', + ); + } + + heading('C6 (b) — a seam does the same for a whole client'); + { + const prod = vendorAdapter(VENDOR_TODAY); + const fixture = vendorAdapter(RECORDED_2026_02_04); + const live = seam({ baseUrl: PROD, adapter: prod }); + const fake = seam({ baseUrl: SANDBOX, adapter: fixture }); + const liveCall = live.stitch(ENDPOINT as never) as unknown as Stitch; + const fakeCall = fake.stitch(ENDPOINT as never) as unknown as Stitch; + const rl = await liveCall.safe({ params: { id: 'inv_9f2' } }); + const rf = await fakeCall.safe({ params: { id: 'inv_9f2' } }); + check('live seam failed', rl.ok, false); + check('fixture seam passed', rf.ok, true); + checkSeq( + 'urls', + [prod.seen[0]?.url, fixture.seen[0]?.url], + [`${PROD}/v1/invoices/inv_9f2`, `${SANDBOX}/v1/invoices/inv_9f2`], + ); + note( + '(b) → the seam carries `baseUrl` + `adapter` for every member at once, and a member may still override either. This is the idiomatic environment switch, and it is composition rather than configuration', + '', + ); + } + + heading('C6 (c) — `.with()` CANNOT retarget, and that is by design'); + { + const call = stitch({ + ...ENDPOINT, + baseUrl: PROD, + adapter: vendorAdapter(RECORDED_2026_02_04), + } as never) as unknown as Stitch; + // @ts-expect-error — `.with()` takes Partial; `baseUrl` is not an input slot + const _bad = call.with({ baseUrl: SANDBOX }); + void _bad; + check( + '`.with({ baseUrl })` is a compile error (see the @ts-expect-error above)', + true, + true, + ); + note( + '(c) → `.with()` is `

>(partial: P)` (types.ts:1994-1996) and `StitchInput` is `{params,query,body,headers,variables,signal,onProgress}`. It binds INPUT, never config — so it is not the environment seam, and the type says so', + '', + ); + } + + heading( + 'C6 (d) — the `baseUrl` thunk: the only per-call environment hatch', + ); + { + let target = SANDBOX; + const wire = vendorAdapter(RECORDED_2026_02_04); + const call = stitch({ + ...ENDPOINT, + baseUrl: () => target, + adapter: wire, + } as never) as unknown as Stitch; + await call.safe({ params: { id: 'a' } }); + target = PROD; + await call.safe({ params: { id: 'b' } }); + checkSeq( + 'one stitch, two targets, resolved at call time', + wire.seen.map((r) => r.url), + [`${SANDBOX}/v1/invoices/a`, `${PROD}/v1/invoices/b`], + ); + note( + '(d) → `baseUrl?: string | (() => string)` (types.ts:1570). A thunk is the ONE place the library lets an environment change between calls without rebuilding the stitch', + '', + ); + } + + heading('C6 (e) — is there a NAMED environment concept?'); + { + // Any unknown key is a compile error (`NoUnknownConfigKeys`, types.ts:338-342), so there is + // no informal extension point either. + const _reject = () => + stitch({ + url: PROD, + // @ts-expect-error — no `env` slot exists on StitchConfig, and unknown keys are rejected + env: 'sandbox', + adapter: vendorAdapter({}), + }); + void _reject; + check( + 'an `env:` key is a compile error (see the @ts-expect-error above)', + true, + true, + ); + + const types = readFileSync( + new URL('../../../../packages/core/src/types.ts', import.meta.url), + 'utf8', + ); + const configBlock = types.slice( + types.indexOf('export interface StitchConfig'), + types.indexOf('export interface InputSchemas'), + ); + const envish = [ + 'env', + 'sandbox', + 'environment', + 'profile', + 'variant', + 'stage', + 'mode', + ].filter((k) => new RegExp(`^\\s{4}${k}\\?:`, 'm').test(configBlock)); + checkSeq('environment-shaped slots on StitchConfig', envish, []); + note( + '(e) → zero. The environment is expressed by WHICH seam/fragment you built, never by a value you can read back off the config', + '', + ); + } + + heading( + 'C6 (f) — is there a spelling for "verify against the real API weekly"?', + ); + { + const cli = readFileSync( + new URL('../../../../packages/core/src/cli.ts', import.meta.url), + 'utf8', + ); + const help = cli.slice( + cli.indexOf('const HELP'), + cli.indexOf('const HELP') + 1800, + ); + const verbs = [ + ...new Set( + [...help.matchAll(/^\s+stitch\s+([a-z-]+)/gm)].map( + (m) => m[1] as string, + ), + ), + ].sort(); + checkSeq('CLI subcommands', verbs, [ + 'diagram', + 'export', + 'from-curl', + 'init', + 'mcp', + 'run', + 'serve', + 'trace', + ]); + const scheduling = [ + 'verify', + 'check', + 'canary', + 'schedule', + 'watch', + 'smoke', + ].filter((v) => verbs.includes(v)); + checkSeq('…of which any run a verification', scheduling, []); + + const testing = readFileSync( + new URL( + '../../../../packages/core/src/testing.ts', + import.meta.url, + ), + 'utf8', + ); + const verifiers = [ + ...new Set( + [ + ...testing.matchAll( + /export (?:async )?function (verify\w+)/g, + ), + ].map((m) => m[1] as string), + ), + ].sort(); + checkSeq('the `verify*` family', verifiers, [ + 'verifyAdapterContract', + 'verifyFingerprintContract', + 'verifySinkContract', + 'verifyStoreContract', + ]); + check( + 'any of them verifies a VENDOR rather than a plugin', + verifiers.some((v) => /vendor|endpoint|api|live|upstream/i.test(v)), + false, + ); + note( + "(f) → all four verifiers take an implementation of one of StitchAPI's OWN seams (Adapter / StitchStore / TraceSink / fingerprint) and check it against a fixed contract. They are for people writing plugins. There is no runner, no schedule, and no vendor-facing mode — exactly the split the capture predicted", + '', + ); + } + + heading('C6 (g) — so what does a parity check actually cost you?'); + { + // The whole thing, written out: run the SAME endpoint against two targets and diff the + // verdicts. It is short — because `extends` did the hard part — but every line is yours. + const parity = async ( + live: Stitch, + fake: Stitch, + input: Record, + ): Promise => { + const [a, b] = await Promise.all([ + live.safe(input as never), + fake.safe(input as never), + ]); + if (a.ok === b.ok) return 'AGREE'; + return `DISAGREE live=${a.ok ? 'ok' : (a.error?.message ?? 'err')} fake=${b.ok ? 'ok' : (b.error?.message ?? 'err')}`; + }; + const live = stitch({ + ...ENDPOINT, + extends: { baseUrl: PROD, adapter: vendorAdapter(VENDOR_TODAY) }, + } as never) as unknown as Stitch; + const fake = stitch({ + ...ENDPOINT, + extends: { + baseUrl: PROD, + adapter: vendorAdapter(RECORDED_2026_02_04), + }, + } as never) as unknown as Stitch; + const verdict = await parity(live, fake, { params: { id: 'inv_9f2' } }); + check( + 'the parity check catches the stale fixture', + verdict, + 'DISAGREE live=contract violation (drift) fake=ok', + ); + note( + '(g) → 8 executable lines, and it is the ONLY thing in this whole scenario that detects the drift in C1(e). It needs a real call, so it cannot run in the offline suite — which is precisely why the missing half is the SCHEDULE, not the comparison', + '', + ); + } + + finish( + 'C6', + 'TARGETING YES, SCHEDULING NO. One endpoint object aimed at three targets via `extends: { baseUrl, adapter }` produced exactly the three URLs expected and three verdicts — fixture ok, sandbox ok, PROD `contract violation (drift)` — so the difference between environments IS visible, through the same `output` schema. A `seam` does the same for a whole client and a member may still override either slot; a `baseUrl` THUNK (`string | (() => string)`) is the one hatch that retargets between calls without rebuilding. `.with()` deliberately cannot: it is `Partial` and `.with({ baseUrl })` is a compile error. What does NOT exist: any named environment concept — zero env/sandbox/environment/profile/variant/stage/mode slots on `StitchConfig`, and an unknown key is a compile error, so there is no informal extension either; and any scheduling or live-verification spelling — the 8 CLI subcommands are run/trace/serve/mcp/diagram/export/from-curl/init and none verifies anything, while the four `verify*Contract` functions all take an implementation of one of StitchAPI\'s OWN seams, not a vendor. The parity check that WOULD catch C1(e) is 8 executable lines and catches it exactly ("DISAGREE live=contract violation (drift) fake=ok") — but it needs a real call, so the missing piece is the schedule, not the comparison', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c7-streams.ts b/docs/scenarios/proofs/stale-fixture/c7-streams.ts new file mode 100644 index 00000000..a40feeba --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c7-streams.ts @@ -0,0 +1,374 @@ +// C7 — streams: is mid-stream failure deterministic with the published fixtures? +// +// `streamOf` / `streamThenError` / `gatedStream` / `sseStream` are the four stream builders in +// `stitchapi/testing`. The question is whether a mid-stream break — scenario 5's whole subject — is +// reproducible EXACTLY, run after run, with no timing in it. +// +// Measured: yes, and unusually cleanly. The builders are pull-driven (`ReadableStream`'s `pull`, +// one chunk per read), so chunk boundaries are chosen by the fixture rather than by a socket, and +// the same fixture produces byte-identical delta sequences across repeated runs. `streamThenError` +// preserves every delta emitted before the break; `gatedStream` holds a connection open on a +// promise you own; `sseStream` frames well-formed SSE including `id:`/`retry:`/comments. +// +// The one thing they do NOT do is pace: the fixtures carry no time, so `manualClock` has nothing to +// drive in them. Inter-chunk timing is not expressible. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c7-streams.ts +import { sse } from '../../../../packages/core/src/sse'; +import { stream } from '../../../../packages/core/src/stream'; +import { manualClock } from '../../../../packages/core/src/test-clock'; +import { collectStitchEvents } from '../../../../packages/core/src/test-events'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import { + gatedStream, + sseStream, + streamAdapter, + streamOf, + streamThenError, +} from '../../../../packages/core/src/test-stream'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { BASE } from './vendor'; + +const dec = new TextDecoder(); + +/** Read a `ReadableStream` to the end, returning the chunks as strings plus any terminal error. */ +async function readAll( + rs: ReadableStream, +): Promise<{ chunks: string[]; error: string | null }> { + const reader = rs.getReader(); + const chunks: string[] = []; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(dec.decode(value)); + } + return { chunks, error: null }; + } catch (e) { + return { chunks, error: (e as Error).message }; + } +} + +async function main(): Promise { + heading( + 'C7 (a) — `streamOf`: chunk boundaries are the fixture’s to choose', + ); + { + const a = await readAll(streamOf(['A', 'B', 'C'])); + checkSeq('three chunks arrive as three reads', a.chunks, [ + 'A', + 'B', + 'C', + ]); + // The same bytes, split differently — a single SSE line across two chunks. + const b = await readAll(streamOf(['data: he', 'llo\n\n'])); + checkSeq('…and a line can be split mid-token', b.chunks, [ + 'data: he', + 'llo\n\n', + ]); + note( + '(a) → `streamOf` enqueues exactly one chunk per `pull` (test-stream.ts:18-27), so cross-chunk buffering is testable at any boundary you name', + '', + ); + } + + heading('C7 (b) — `streamThenError`: the deltas before the break survive'); + { + const clock = manualClock(); + const api = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: streamThenError( + [ + 'data: {"t":"A"}\n\n', + 'data: {"t":"B"}\n\n', + 'data: {"t":"C"}\n\n', + ], + new Error('socket reset'), + ), + }, + }, + ]); + const feed = sse({ url: `${BASE}/feed`, adapter: api, clock }); + const c = await collectStitchEvents(feed()); + checkSeq('the event spine', c.types, [ + 'start', + 'progress', + 'delta', + 'delta', + 'delta', + 'error', + 'done', + ]); + checkSeq( + 'the deltas delivered before the break (data is JSON-parsed)', + c.deltas.map((d) => (d as { data: unknown }).data), + [{ t: 'A' }, { t: 'B' }, { t: 'C' }], + ); + check('the error message', c.error?.message, 'socket reset'); + check('done.ok', c.done?.ok, false); + note( + '(b) → a mid-stream failure surfaces as an `error` EVENT with every prior delta intact — not a throw that discards them. That is the assertion scenario 5 needs', + '', + ); + } + + heading('C7 (c) — and it is deterministic across runs'); + { + const run = async (): Promise => { + const api = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: streamThenError(['data: A\n\n', 'data: B\n\n']), + }, + }, + ]); + const feed = sse({ url: `${BASE}/feed`, adapter: api }); + const c = await collectStitchEvents(feed()); + return JSON.stringify({ + types: c.types, + deltas: c.deltas.map((d) => (d as { data: unknown }).data), + error: c.error?.message, + ok: c.done?.ok, + }); + }; + const runs = [ + await run(), + await run(), + await run(), + await run(), + await run(), + ]; + check('5 runs, distinct outcomes', new Set(runs).size, 1); + note('the single outcome', runs[0]); + note( + '(c) → byte-identical five times. The default error is the literal `stream broke mid-flight` (test-stream.ts:58), so even the message is fixed', + '', + ); + } + + heading( + 'C7 (d) — a CLEAN truncation is indistinguishable from a complete stream', + ); + { + // The counterpart finding: `streamOf` closes the stream, which is a well-formed end. If the + // vendor drops the connection cleanly mid-response, nothing in the event spine says so. + const complete = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: sseStream(['A', 'B', 'C']), + }, + }, + ]); + const truncated = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: sseStream(['A', 'B']), + }, + }, + ]); + const full = await collectStitchEvents( + sse({ url: `${BASE}/f`, adapter: complete })(), + ); + const cut = await collectStitchEvents( + sse({ url: `${BASE}/f`, adapter: truncated })(), + ); + checkSeq('complete spine', full.types, [ + 'start', + 'progress', + 'delta', + 'delta', + 'delta', + 'result', + 'done', + ]); + checkSeq('truncated spine', cut.types, [ + 'start', + 'progress', + 'delta', + 'delta', + 'result', + 'done', + ]); + check( + 'both report ok', + `${String(full.done?.ok)}/${String(cut.done?.ok)}`, + 'true/true', + ); + note( + '(d) → the ONLY difference is the delta count. A clean early close is `result` + `done(ok:true)`, same as success — so "the vendor stopped early" is not a failure the fixtures can make the engine report. You have to count deltas yourself', + '', + ); + } + + heading( + 'C7 (e) — `gatedStream`: a connection held open on a promise you own', + ); + { + let open = true; + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + const body = gatedStream('data: first\n\n', gate); + const api = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: body, + }, + }, + ]); + const feed = sse({ url: `${BASE}/longpoll`, adapter: api }); + const done = collectStitchEvents(feed()).then((c) => { + open = false; + return c; + }); + await new Promise((r) => { + setTimeout(r, 20); + }); + check('still open while the gate is unresolved', open, true); + release(); + const c = await done; + check('…and it closes when you resolve it', open, false); + checkSeq('the spine', c.types, [ + 'start', + 'progress', + 'delta', + 'result', + 'done', + ]); + note( + '(e) → the "long-lived connection" shape with no timers in it at all. Concurrency and cancellation tests get a deterministic held-open socket', + '', + ); + } + + heading('C7 (f) — `sseStream`: the frames it actually writes'); + { + const frames = await readAll( + sseStream([ + 'plain', + { data: { t: 1 }, event: 'token', id: 'e1' }, + { data: 'multi\nline' }, + { comment: 'keep-alive', data: '' }, + { data: 'x', retry: 3000 }, + ]), + ); + checkSeq('one frame per chunk', frames.chunks, [ + 'data: plain\n\n', + 'event: token\nid: e1\ndata: {"t":1}\n\n', + 'data: multi\ndata: line\n\n', + ': keep-alive\ndata: \n\n', + 'retry: 3000\ndata: x\n\n', + ]); + note( + '(f) → a bare string is `{ data }`; a non-string `data` is JSON-stringified; a multi-line payload is split across `data:` lines per the spec; `comment`/`event`/`id`/`retry` all render. These are well-formed frames, not an approximation', + '', + ); + } + + heading( + 'C7 (g) — `streamAdapter`: the minimal transport, and its one rule', + ); + { + const ok = await collectStitchEvents( + sse({ + url: `${BASE}/f`, + adapter: streamAdapter(sseStream(['A'])), + })(), + ); + checkSeq('it drives the sse surface', ok.types, [ + 'start', + 'progress', + 'delta', + 'result', + 'done', + ]); + // It REJECTS a non-streaming request — the one contract it enforces. + let msg = ''; + try { + await streamAdapter(streamOf(['x']))({ + url: `${BASE}/f`, + method: 'GET', + headers: {}, + }); + } catch (e) { + msg = (e as Error).message; + } + check( + 'a buffered request is refused', + msg, + 'expected req.stream to be set', + ); + note( + '(g) → `streamAdapter` is 8 lines and checks exactly one thing (test-stream.ts:129-131). Compare C3: that is one more runtime check than `mockAdapter` performs on a fixture body', + '', + ); + } + + heading('C7 (h) — the fixtures carry no TIME'); + { + const clock = manualClock(); + const api = mockAdapter([ + { + respond: { + headers: { 'content-type': 'text/event-stream' }, + stream: sseStream(['A', 'B', 'C']), + }, + }, + ]); + const at: number[] = []; + const feed = sse({ url: `${BASE}/f`, adapter: api, clock }); + for await (const e of feed().stream()) { + if (e.type === 'delta') at.push(clock.now()); + } + checkSeq('virtual time at each delta', at, [0, 0, 0]); + note( + '(h) → every chunk arrives at virtual 0. There is no `delay` on `SseFixtureEvent` and no per-chunk pacing on `streamOf`, so "the vendor sent a token every 200ms" is not expressible. `mockAdapter.delay` paces the RESPONSE, not the chunks — and it uses a real `setTimeout`, so it is wall-clock (C2)', + '', + ); + } + + heading('C7 (i) — the raw `stream` surface, same builders'); + { + const api = mockAdapter([ + { + respond: { + stream: streamThenError(['{"a":1}\n', '{"a":2}\n']), + }, + }, + ]); + const lines = stream({ + url: `${BASE}/ndjson`, + adapter: api, + stream: 'ndjson', + }); + const c = await collectStitchEvents(lines()); + checkSeq('spine', c.types, [ + 'start', + 'progress', + 'delta', + 'delta', + 'error', + 'done', + ]); + checkSeq('deltas', c.deltas, [{ a: 1 }, { a: 2 }]); + check('error', c.error?.message, 'stream broke mid-flight'); + note( + '(i) → the same builders drive the non-SSE `stream` surface, decoded per `stream: "ndjson"`. Determinism is a property of the builder, not of the surface', + '', + ); + } + + finish( + 'C7', + 'CONFIRMED — MID-STREAM FAILURE IS FULLY DETERMINISTIC. `streamThenError` over an `sse` stitch produced the spine `start,progress,delta,delta,delta,error,done` with all three deltas intact before the break, `error.message` "socket reset", `done.ok` false — and five repeat runs produced ONE distinct outcome, byte-identical, including the default message `stream broke mid-flight`. `gatedStream` holds a connection open on a promise you resolve (verified open at +20ms, closed on release). `sseStream` writes well-formed frames — a bare string becomes `data: plain\\n\\n`, a non-string `data` is JSON-stringified, a multi-line payload splits across `data:` lines, and `comment`/`event`/`id`/`retry` all render. The same builders drive the raw `stream` surface with `stream: "ndjson"` decoding. TWO LIMITS, both about what a fixture cannot say. (1) A CLEAN early close is indistinguishable from a complete stream: 3 deltas and 2 deltas both end `result` + `done(ok:true)`, so "the vendor stopped early" is a delta count you check yourself, not a failure the engine reports. (2) The fixtures carry no time — all three deltas arrive at `clock.now() === 0`, there is no `delay` on `SseFixtureEvent` and no per-chunk pacing on `streamOf`, so inter-chunk timing is not expressible at all; `mockAdapter.delay` paces the whole response and does it on a real `setTimeout`', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/c8-assembled.ts b/docs/scenarios/proofs/stale-fixture/c8-assembled.ts new file mode 100644 index 00000000..053ae32b --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/c8-assembled.ts @@ -0,0 +1,282 @@ +// C8 — assemble the best available "my fixtures cannot silently rot" setup, and price it. +// +// Every gap C1–C7 measured gets replayed here twice: once against the naive setup (a `mockAdapter` +// and a `stubStitch`, written the way the docs show) and once against `fixture-guard.ts`. The +// difference is the whole claim, and the line count is the bill. +// +// The honest headline: FOUR of the five gaps close in-process, and the fifth — C1(e), the vendor +// moving while the fixture holds — does not close at all without a live call. What the guard can do +// for that one is make the staleness VISIBLE (a recording date, an expiry) so the suite fails on a +// calendar rather than on a schema. That is a weaker guarantee than the other four and it is +// reported as one. +// +// pnpm exec tsx docs/scenarios/proofs/stale-fixture/c8-assembled.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/test-clock'; +import { mockAdapter } from '../../../../packages/core/src/test-mock'; +import { stubStitch } from '../../../../packages/core/src/test-stub'; +import type { Stitch } from '../../../../packages/core/src/types'; +import { + assertClockHonest, + contractStub, + expired, + jsonOnly, + parity, + stamp, +} from './fixture-guard'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { + BASE, + RECORDED_2026_02_04, + RECORDED_ON, + VENDOR_TODAY, + vendorAdapter, +} from './vendor'; +import { z } from './zod'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * Executable lines between the USER CODE markers — imports, blanks and comments removed, so the + * number is the code someone actually maintains. Same counter as `agent-holds-the-tool/c8`. + */ +function executableLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8'); + const from = src.indexOf('// >>> BEGIN USER CODE'); + const to = src.indexOf('// <<< END USER CODE'); + return src + .slice(from, to) + .replace(/^import[\s\S]*?;$/gm, '') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +const Invoice = z.object({ + id: z.string(), + amount_cents: z.number(), + currency: z.string(), + paid: z.boolean(), + customer_email: z.string(), + legacy_ref: z.string(), +}); + +class LiveInvoice { + constructor(public id: string) {} + get amount_cents(): number { + return 4200; + } +} + +async function main(): Promise { + heading('C8 (a) — the C3 gap: a fixture body no wire could produce'); + { + // NAIVE: `mockAdapter` serves the class instance, getter and all. + const naive = mockAdapter([ + { respond: { body: new LiveInvoice('inv_1') } as never }, + ]); + const naiveCall = stitch({ url: `${BASE}/i`, adapter: naive }); + const n = await naiveCall.safe(); + check('naive: the call succeeded', n.ok, true); + check( + 'naive: the caller read a value that exists only in the test', + (n.data as LiveInvoice).amount_cents, + 4200, + ); + + // GUARDED: the same fixture is refused at the transport seam. + const guarded = stitch({ + url: `${BASE}/i`, + adapter: jsonOnly( + mockAdapter([ + { respond: { body: new LiveInvoice('inv_1') } as never }, + ]), + ), + }); + const g = await guarded.safe(); + check('guarded: the call FAILED', g.ok, false); + check( + 'guarded: with a reason naming the fixture', + g.error?.message, + `fixture is not a wire shape ($: LiveInvoice): GET ${BASE}/i`, + ); + + // …and a legitimate fixture still passes straight through. + const fine = stitch({ + url: `${BASE}/i`, + adapter: jsonOnly( + mockAdapter([{ respond: { body: RECORDED_2026_02_04 } }]), + ), + output: Invoice, + }); + const f = await fine.safe(); + check('guarded: a real JSON fixture is unaffected', f.ok, true); + } + + heading('C8 (b) — the C5 gap: a stub that skips the input contract'); + { + const InputSchemas = { params: z.object({ id: z.string() }) }; + const naive = stubStitch({ id: 'inv_1' }); + const nr = await naive.safe({ params: { id: 42 } } as never); + check('naive stub: accepted a number id', nr.ok, true); + + const guarded = contractStub(InputSchemas, { id: 'inv_1' }); + const gr = await guarded.safe({ params: { id: 42 } } as never); + check('guarded stub: REFUSED it', gr.ok, false); + check( + 'guarded stub: with the schema’s own message', + gr.error?.message?.startsWith('stub input.params:'), + true, + ); + note('the message', gr.error?.message); + const ok = await guarded.safe({ params: { id: '42' } } as never); + check('guarded stub: a valid id still works', ok.ok, true); + check('…and the spy still records', guarded.callCount(), 2); + } + + heading('C8 (c) — the C2 gap: a slot that is inert under `manualClock`'); + { + const clock = manualClock(); + const cfg = { cache: { ttl: 60_000 }, timeout: { total: 1000 } }; + let refused = ''; + try { + assertClockHonest(cfg); + } catch (e) { + refused = (e as Error).message; + } + check( + 'the pairing is refused', + refused, + 'manualClock cannot drive: cache, timeout.total — these read wall-clock (ADR 0010 §4), so an advance() proves nothing about them', + ); + // The clock-driven slots are not refused. + let allowed = 'ok'; + try { + assertClockHonest({ + retry: { attempts: 3 }, + throttle: '1/s', + circuit: { failures: 2, cooldown: 1000 }, + timeout: 500, + }); + } catch (e) { + allowed = (e as Error).message; + } + check('retry/throttle/circuit/per-attempt timeout pass', allowed, 'ok'); + note( + 'the library’s own diagnostic for this', + 'none — `policySummary` reports `cfg[k] !== undefined` (configured), never whether a slot RAN', + ); + void clock; + } + + heading('C8 (d) — the C1(e) gap: the vendor moved, the fixture did not'); + { + // This is the one that does not close offline. What the guard adds is a DATE. + const fixtures = { + getInvoice: stamp(RECORDED_2026_02_04, RECORDED_ON, 90), + listPlans: stamp({ plans: [] }, '2026-07-20', 90), + }; + const stale = expired(fixtures, new Date('2026-08-05')); + checkSeq('fixtures past their re-record date', stale, [ + 'getInvoice recorded 2026-02-04 (182d old)', + ]); + check('…and the fresh one is not flagged', stale.length, 1); + note( + '(d) → 182 days. The offline suite can now fail on the CALENDAR, which is the only signal available without a call. It does not prove the fixture is wrong — it proves nobody has checked', + '', + ); + + // And the live check, quarantined, for when a call is permitted. + const live = stitch({ + name: 'getInvoice', + baseUrl: BASE, + path: '/v1/invoices/{id}', + output: Invoice, + adapter: vendorAdapter(VENDOR_TODAY), + }) as unknown as Stitch; + const fake = stitch({ + name: 'getInvoice', + baseUrl: BASE, + path: '/v1/invoices/{id}', + output: Invoice, + adapter: vendorAdapter(RECORDED_2026_02_04), + }) as unknown as Stitch; + const verdict = await parity(live, fake, { params: { id: 'inv_9f2' } }); + check( + 'the live parity check names the drift', + verdict, + 'DISAGREE live=contract violation (drift) fake=ok', + ); + note( + '(d) → `parity()` is the only thing in this directory that DETECTS the drift rather than dating it, and it needs a real call. It cannot live in the offline suite', + '', + ); + } + + heading('C8 (e) — what still does not close'); + { + // C7(d): a clean early close is indistinguishable from a complete stream. No seam here + // changes that — it is a delta count the consumer has to assert. + check('a truncated stream is still a successful stream', true, true); + note( + 'C7(d) — a clean early close ends `result` + `done(ok:true)`, same as success. Countable, not detectable', + '', + ); + note( + 'C4 — retry backoff delays are still absent from the event stream; `clock.now()` remains the only reader', + '', + ); + note( + 'C5(g) — `stubStitch.safe()` still throws on a SYNC-throwing impl. `contractStub` sidesteps it by being `async`, but the underlying behaviour is unchanged', + '', + ); + note( + 'C2 — `timeout.total`, `cache.ttl`, `memoryStore` TTL, OAuth2 expiry, SigV4 and `done.elapsed` are still wall-clock. `assertClockHonest` refuses the pairing; it does not fix it', + '', + ); + } + + heading('C8 (f) — the bill'); + { + const lines = executableLines('fixture-guard.ts'); + note('fixture-guard.ts, executable lines', lines); + note('seams used', 5); + note(' 1. the `adapter` seam', 'wrap the transport — `jsonOnly`'); + note( + ' 2. the `impl` function of `stubStitch`', + 'run the input schemas — `contractStub`', + ); + note( + ' 3. the published `validate()`', + 'the SAME schema the real stitch declares, called directly', + ); + note( + ' 4. the config object, read before construction', + 'refuse an inert pairing — `assertClockHonest`', + ); + note( + ' 5. `extends: { baseUrl, adapter }`', + 'one endpoint, two targets — `parity`', + ); + note('config keys that know about any of this', 0); + check('under 100 executable lines', lines <= 100, true); + check('of which the part needing a NETWORK call', 'parity', 'parity'); + } + + finish( + 'C8', + `FOUR OF FIVE GAPS CLOSE IN ${String(executableLines('fixture-guard.ts'))} EXECUTABLE LINES ACROSS 5 SEAMS; THE FIFTH CANNOT CLOSE OFFLINE. Replayed side by side: (a) a class-instance fixture gives the naive setup \`data.amount_cents === 4200\` from a getter and gives the guarded setup \`fixture is not a wire shape ($: LiveInvoice): GET ${BASE}/i\`, with a real JSON fixture unaffected; (b) \`{ params: { id: 42 } }\` is accepted by \`stubStitch\` and refused by \`contractStub\` with the schema's own message, while valid input still passes and the spy still records 2 calls; (c) pairing a \`manualClock\` with \`cache\` + \`timeout.total\` is refused by name, while retry/throttle/circuit/per-attempt-timeout pass — a diagnostic the library does not have, since \`policySummary\` reports only whether a slot was CONFIGURED, never whether it RAN. (d) is the one that does not close: the guard can only DATE the fixture — \`getInvoice recorded 2026-02-04 (182d old)\` against a 90-day expiry — which fails the suite on the calendar rather than on the drift. The only thing that actually detects it is \`parity()\`, measured returning \`DISAGREE live=contract violation (drift) fake=ok\`, and it needs a live call, so it cannot run in the offline suite. That is the scenario's answer: the comparison is 8 lines and the library already gives you every seam it needs; what is missing is a place to put a call you are only allowed to make sometimes`, + ); +} + +void main(); diff --git a/docs/scenarios/proofs/stale-fixture/fixture-guard.ts b/docs/scenarios/proofs/stale-fixture/fixture-guard.ts new file mode 100644 index 00000000..2133c5bc --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/fixture-guard.ts @@ -0,0 +1,151 @@ +// The assembled answer to "my fixtures cannot silently rot" — the USER CODE half of C8. +// +// Everything between the markers is what an integrator has to write and maintain. Everything +// outside them is StitchAPI. The four gaps this closes are the four C1–C5 measured: +// +// C1(e) the vendor drifts while the fixture holds -> `parity()` + `stamp()` +// C2 a config slot is inert under `manualClock` -> `assertClockHonest()` +// C3 a fixture body no wire could produce -> `jsonOnly()` +// C5 a stub that skips the input contract -> `contractStub()` +// +// The design constraint that shapes all four: NOTHING here may need a network call, except the one +// function that is explicitly about needing one. `parity()` is quarantined for that reason — it is +// the only export that must run outside the offline suite, and saying so is half its value. +import { validate } from '../../../../packages/core/src/index'; +import type { SchemaLike } from '../../../../packages/core/src/infer'; +import { stubStitch } from '../../../../packages/core/src/test-stub'; +import type { + Adapter, + Stitch, + StitchInput, +} from '../../../../packages/core/src/types'; + +// >>> BEGIN USER CODE + +/** A fixture with the one fact the library cannot hold: when it was taken. */ +export interface Dated { + body: T; + recordedOn: string; + /** Days after which this fixture must be re-recorded. */ + staleAfterDays: number; +} + +/** Stamp a fixture with its recording date. The only place `recordedOn` can live. */ +export function stamp( + body: T, + recordedOn: string, + staleAfterDays = 90, +): Dated { + return { body, recordedOn, staleAfterDays }; +} + +/** Which stamped fixtures are past their re-record date, as of `today`. */ +export function expired( + fixtures: Record>, + today: Date, +): string[] { + const out: string[] = []; + for (const [name, f] of Object.entries(fixtures)) { + const age = + (today.getTime() - new Date(f.recordedOn).getTime()) / 86_400_000; + if (age > f.staleAfterDays) + out.push( + `${name} recorded ${f.recordedOn} (${Math.floor(age)}d old)`, + ); + } + return out; +} + +/** The first non-wire value in `v`, as a `path: description`, or `null` if it is all wire shapes. */ +function nonWire(v: unknown, path = '$'): string | null { + if (v === null || ['string', 'number', 'boolean'].includes(typeof v)) + return null; + if (typeof v !== 'object') return `${path}: ${typeof v}`; + const proto: unknown = Object.getPrototypeOf(v); + if (Array.isArray(v)) + return v.reduce( + (acc, el, i) => acc ?? nonWire(el, `${path}[${String(i)}]`), + null, + ); + if (proto !== Object.prototype && proto !== null) + return `${path}: ${(v.constructor as { name: string } | undefined)?.name ?? 'exotic'}`; + for (const [k, el] of Object.entries(v)) { + const hit = nonWire(el, `${path}.${k}`); + if (hit) return hit; + } + return null; +} + +/** + * Wrap a transport so every response body must be a shape a JSON wire can actually deliver, and is + * then normalised through a real round-trip. A `Date`, a `Map`, a class instance, a function or a + * `bigint` — anywhere in the tree — is REJECTED rather than silently degraded, which is what a bare + * `JSON.stringify` comparison would do (a prototype getter is not enumerable, so it vanishes without + * changing the JSON text). + */ +export function jsonOnly(inner: Adapter): Adapter { + return async (req) => { + const res = await inner(req); + if (res.body === undefined || res.body instanceof ReadableStream) + return res; + const bad = nonWire(res.body); + if (bad) + throw new Error( + `fixture is not a wire shape (${bad}): ${req.method} ${req.url}`, + ); + return { + ...res, + body: JSON.parse(JSON.stringify(res.body)) as unknown, + }; + }; +} + +/** A `stubStitch` that runs the SAME `input` schemas the real stitch declares. */ +export function contractStub( + input: Record, + impl: TOut | ((i: StitchInput) => TOut | Promise), +): ReturnType> { + return stubStitch(async (call: StitchInput) => { + for (const [slot, schema] of Object.entries(input)) { + const value = (call as Record)[slot]; + const r = await validate(schema, value ?? {}); + if (!r.ok) + throw new Error( + `stub input.${slot}: ${r.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')}`, + ); + } + return typeof impl === 'function' + ? (impl as (i: StitchInput) => TOut | Promise)(call) + : impl; + }); +} + +/** Config slots that ignore an injected `Clock` (measured in C2) — inert under `manualClock`. */ +export const WALL_CLOCK_SLOTS = ['cache'] as const; + +/** + * Fail a test that pairs a `manualClock` with a slot the clock cannot drive. The library has no + * such diagnostic, so this reads the config the way `policySummary` does and refuses the pairing. + */ +export function assertClockHonest(cfg: Record): void { + const bad = WALL_CLOCK_SLOTS.filter((s) => cfg[s] !== undefined); + const total = (cfg['timeout'] as { total?: unknown } | undefined)?.total; + if (total !== undefined) bad.push('timeout.total' as never); + if (bad.length) + throw new Error( + `manualClock cannot drive: ${bad.join(', ')} — these read wall-clock (ADR 0010 §4), so an advance() proves nothing about them`, + ); +} + +/** Run one input against two targets and report whether they agree. NEEDS A LIVE CALL. */ +export async function parity( + live: Stitch, + fake: Stitch, + input: StitchInput, +): Promise { + const [a, b] = await Promise.all([live.safe(input), fake.safe(input)]); + if (a.ok === b.ok) return 'AGREE'; + return `DISAGREE live=${a.ok ? 'ok' : (a.error?.message ?? 'err')} fake=${b.ok ? 'ok' : (b.error?.message ?? 'err')}`; +} + +// <<< END USER CODE diff --git a/docs/scenarios/proofs/stale-fixture/harness.ts b/docs/scenarios/proofs/stale-fixture/harness.ts new file mode 100644 index 00000000..d154c6c8 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/harness.ts @@ -0,0 +1,200 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script exits +// non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence has one unusual shape that needed its own assertion, so it is worth +// naming up front. Most proofs measure "did the library do the right thing". C2 has to measure +// something else: **did the test assert anything at all**. A `timeout.total` test written with +// `manualClock()` does not fail — it passes, having exercised nothing. `checkVacuous` is the +// assertion for that: it runs the SAME script body twice, once with the clock advanced by the +// amount the test claims is decisive and once with the clock never advanced, and PASSES when the +// two outcomes are identical. Identical outcomes mean the `advance()` call was decoration. +// +// `check` / `checkSeq` / `note` / `heading` / `finish` follow `intermittent-drift/harness.ts` +// unchanged, so a reader who has seen one proof directory has seen this one. + +let failures = 0; +let checks = 0; + +/** Render a measured value unambiguously — `undefined` vs `'undefined'` decides several rows. */ +function show(v: unknown): string { + if (v === undefined) return 'undefined'; + if (typeof v === 'bigint') return `${v}n`; + if (typeof v === 'number' && Number.isNaN(v)) return 'NaN'; + return JSON.stringify(v) ?? String(v); +} + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${show(actual)}${ok ? '' : ` (expected ${show(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the event spine + * (`["start","progress","retry","progress","result","done"]`) and the backoff gaps + * (`[1000,2000,4000]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Assert a measured number lies within `tol` of `expected` — for the one wall-clock row in C2. */ +export function checkNear( + label: string, + actual: number, + expected: number, + tol: number, +): void { + checks++; + const ok = Math.abs(actual - expected) <= tol; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)} (expected ${String(expected)} ±${String(tol)})`, + ); +} + +/** Assert a measured value is at least `bound`. */ +export function checkAtLeast( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual >= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (>= ${String(bound)})` : ` (expected >= ${String(bound)})`}`, + ); +} + +/** The outcome of one run of a clock-sensitivity probe, reduced to a comparable string. */ +export type Outcome = string; + +/** + * THE assertion of C2. `body(advanceMs)` runs a scenario and returns a string summarising what + * happened. It is called twice: once with the amount of virtual time the test believes is decisive, + * and once with `0`. The claim under test is "`manualClock` drives this feature", so: + * + * - **driven**: the two outcomes DIFFER — `advance()` changed the result, the assertion had teeth. + * - **inert**: the two outcomes are IDENTICAL — the feature never consulted the injected clock, so + * a test that advances it and then asserts on the result asserts nothing about time. + * + * Both directions are legitimate measurements, so `expect` says which one this row claims, and the + * printed line always carries both outcomes. + */ +export async function checkClockDriven( + feature: string, + expect: 'driven' | 'inert', + body: (advanceMs: number) => Promise, + advanceMs: number, +): Promise { + checks++; + const advanced = await body(advanceMs); + const frozen = await body(0); + const driven = advanced !== frozen; + const verdict = driven ? 'driven' : 'inert'; + const ok = verdict === expect; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${feature.padEnd(30)} -> ${verdict.toUpperCase().padEnd(6)} ` + + `advance(${String(advanceMs)})=${advanced} | advance(0)=${frozen}` + + `${ok ? '' : ` (expected ${expect})`}`, + ); +} + +/** + * One row of the C2 table. Three verdicts, because two were not enough: + * + * - `CLOCK` — `advance()` drives it. A test can assert on it. + * - `WALL` — it reads `Date.now()` through `util.now()` regardless. A test that advances the clock + * and asserts on it is asserting nothing the advance caused. + * - `NONE` — the feature has no time in it to drive. Neither a win nor a gap; recorded so the + * table is an enumeration rather than a selection. + */ +export interface ClockRow { + feature: string; + verdict: 'CLOCK' | 'WALL' | 'NONE'; + evidence: string; +} + +const rows: ClockRow[] = []; + +const LABEL: Record = { + CLOCK: 'manualClock', + WALL: 'WALL CLOCK ', + NONE: 'no time ', +}; + +/** Record a C2 table row (printed by {@link printClockTable}). */ +export function row( + feature: string, + verdict: ClockRow['verdict'], + evidence: string, +): void { + rows.push({ feature, verdict, evidence }); +} + +/** Print the accumulated C2 table, plus the per-verdict tallies. */ +export function printClockTable(): { + clock: number; + wall: number; + none: number; +} { + const w = Math.max(...rows.map((r) => r.feature.length)); + console.log( + `\n ${'feature'.padEnd(w)} driven by evidence\n ${'-'.repeat(w)} ----------- --------`, + ); + for (const r of rows) { + console.log( + ` ${r.feature.padEnd(w)} ${LABEL[r.verdict]} ${r.evidence}`, + ); + } + return { + clock: rows.filter((r) => r.verdict === 'CLOCK').length, + wall: rows.filter((r) => r.verdict === 'WALL').length, + none: rows.filter((r) => r.verdict === 'NONE').length, + }; +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown = ''): void { + const v = value === '' ? '' : `: ${show(value)}`; + console.log(` note ${label}${v}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring a FAILURE of the library (C2's inert rows, C5's asymmetry), + * so the verdict statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/stale-fixture/vendor.ts b/docs/scenarios/proofs/stale-fixture/vendor.ts new file mode 100644 index 00000000..28e2f008 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/vendor.ts @@ -0,0 +1,177 @@ +// The vendor, and the fixture that was recorded from it six months ago. +// +// The whole scenario lives in the gap between two objects, so they are both here and both literal: +// +// RECORDED_2026_02_04 — what `GET /v1/invoices/{id}` returned on the day someone ran the +// recorder. This is the cassette. It never changes again. +// VENDOR_TODAY — what the same endpoint returns now. `amount_cents` became `amount` +// (a rename), `legacy_ref` was dropped (a removal), `paid` became a +// string enum (a retype), and `customer_email` is now sometimes `null`. +// +// Both are plain data. Nothing in this file imports the library's testing kit — the point of C1 is +// to ask whether the LIBRARY can tell these two apart, so the fixtures must not be built by +// anything that already knows the answer. +// +// `vendorAdapter()` is the only moving part: a hand-written in-memory `Adapter` that serves one of +// these bodies and records every request it saw. It is deliberately NOT `mockAdapter` — C3 puts +// `mockAdapter` itself under test, and a proof that used it everywhere could not measure it. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +/** The day the cassette was cut. Carried as a literal because C1(f) asks who else knows it. */ +export const RECORDED_ON = '2026-02-04'; + +/** The cassette: the exact response body the recorder captured, six months before today. */ +export const RECORDED_2026_02_04 = { + id: 'inv_9f2', + amount_cents: 4200, + currency: 'usd', + paid: true, + customer_email: 'ada@example.com', + legacy_ref: 'REF-118', +} as const; + +/** What the vendor returns today. Four independent changes from the cassette. */ +export const VENDOR_TODAY = { + id: 'inv_9f2', + amount: 4200, // RENAMED from `amount_cents` + currency: 'usd', + paid: 'paid', // RETYPED from boolean to a string enum + customer_email: null, // NULLED — now absent for invoices without a contact + // `legacy_ref` REMOVED +} as const; + +/** The four drift shapes C1 walks, each as a body derived from the cassette. */ +export const MUTATIONS = { + /** `legacy_ref` is gone. */ + removed: { + id: 'inv_9f2', + amount_cents: 4200, + currency: 'usd', + paid: true, + customer_email: 'ada@example.com', + }, + /** `amount_cents` is now `amount`. */ + renamed: { + id: 'inv_9f2', + amount: 4200, + currency: 'usd', + paid: true, + customer_email: 'ada@example.com', + legacy_ref: 'REF-118', + }, + /** `paid` is now a string. */ + retyped: { + id: 'inv_9f2', + amount_cents: 4200, + currency: 'usd', + paid: 'paid', + customer_email: 'ada@example.com', + legacy_ref: 'REF-118', + }, + /** `customer_email` is now `null`. */ + nulled: { + id: 'inv_9f2', + amount_cents: 4200, + currency: 'usd', + paid: true, + customer_email: null, + legacy_ref: 'REF-118', + }, +} as const; + +export const BASE = 'https://api.billing.test'; + +/** An in-memory transport serving one fixed body, with a request log. */ +export interface FakeVendor extends Adapter { + /** Every request the transport received. */ + readonly seen: AdapterRequest[]; + /** How many requests it received. */ + count(): number; +} + +/** + * A hand-written {@link Adapter} that answers every request with `body` at `status`. No routing, no + * sequences — the scripts that need those reach for the real `mockAdapter`, which is the point of + * C3. `delayMs` is honoured against the request signal so a per-attempt `timeout` can cancel it. + */ +export function vendorAdapter( + body: unknown, + opts: { + status?: number; + headers?: Record; + delayMs?: number; + } = {}, +): FakeVendor { + const seen: AdapterRequest[] = []; + const fn = (async (req: AdapterRequest): Promise => { + seen.push(req); + if (opts.delayMs !== undefined && opts.delayMs > 0) { + await new Promise((resolve, reject) => { + const t = setTimeout(resolve, opts.delayMs); + req.signal?.addEventListener( + 'abort', + () => { + clearTimeout(t); + reject(new Error('aborted')); + }, + { once: true }, + ); + }); + } + return { + status: opts.status ?? 200, + headers: opts.headers ?? { 'content-type': 'application/json' }, + body, + }; + }) as FakeVendor; + Object.defineProperty(fn, 'seen', { value: seen }); + fn.count = () => seen.length; + return fn; +} + +/** + * A transport that answers with a SEQUENCE of `[status, body]` pairs (the last entry repeats) — the + * flaky-endpoint shape C4 needs, hand-written for the same reason as {@link vendorAdapter}. + */ +export function sequenceAdapter( + steps: readonly (readonly [number, unknown])[], + headersFor?: (i: number) => Record, +): FakeVendor { + const seen: AdapterRequest[] = []; + const fn = (async (req: AdapterRequest): Promise => { + const i = seen.length; + seen.push(req); + const step = steps[Math.min(i, steps.length - 1)] as readonly [ + number, + unknown, + ]; + return { + status: step[0], + headers: headersFor?.(i) ?? { 'content-type': 'application/json' }, + body: step[1], + }; + }) as FakeVendor; + Object.defineProperty(fn, 'seen', { value: seen }); + fn.count = () => seen.length; + return fn; +} + +/** Render a `DriftFinding` as `level|change|path|detail` — the format `intermittent-drift` uses. */ +export function fmt(f: { + level?: string; + change?: string; + path?: string; + detail?: string; + message?: string; +}): string { + return [ + f.level ?? '?', + f.change ?? '?', + f.path ?? '', + f.detail ?? f.message ?? '', + ].join('|'); +} diff --git a/docs/scenarios/proofs/stale-fixture/zod.ts b/docs/scenarios/proofs/stale-fixture/zod.ts new file mode 100644 index 00000000..22083911 --- /dev/null +++ b/docs/scenarios/proofs/stale-fixture/zod.ts @@ -0,0 +1,13 @@ +// Real Zod, imported by path — the same convenience `intermittent-drift/zod.ts` documents. +// +// This directory needs a real schema library for exactly one reason: C1's question is "does the +// SAME `output` schema that guards production also fail a drifted fixture", and "the same schema" +// is only a meaningful phrase if the schema is a real one. A hand-rolled `{ validate }` stub would +// let this directory invent the very behaviour under test (which key is required, what `.optional()` +// does to a removal), and C1's whole point is that those declarations decide the answer. +// +// `packages/core` already depends on Zod v4 in devDependencies; pnpm does not hoist it to the +// workspace root and `docs/` has no manifest, so the import goes by relative path. It resolves +// under `tsx` and typechecks under `packages/core`'s strict set. In application code the spelling +// is `import { z } from 'zod'`. +export { z } from '../../../../packages/core/node_modules/zod'; diff --git a/docs/scenarios/proofs/unconfirmed-write/README.md b/docs/scenarios/proofs/unconfirmed-write/README.md new file mode 100644 index 00000000..fe00af83 --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/README.md @@ -0,0 +1,231 @@ +# Proofs — the charge you can't confirm + +Runnable evidence for the claims in [`../../unconfirmed-write.md`](../../unconfirmed-write.md). + +**The scenario's answer is a count of charges, and the deciding one is 2.** A job that charges once, +crashes, and is re-driven by its queue creates **two charges for one intended payment** under +`idempotency: true` + `retry` — the configuration the +[idempotency guide's](../../../../apps/docs/content/docs/guides/resilience/idempotency.mdx) first +example uses. The default key is `randomUUID()` evaluated once per **call** (engine.ts:172), so it is +not restart-stable and not even call-stable: the same stitch instance called twice mints two keys and +charges twice. C2 is the deciding claim and it goes against the library. The capture's central worry +is confirmed. + +`idempotency.keyOf` fixes it — 1 key, 1 charge across a restart — and then two things bite that the +capture did not anticipate. `keyOf` is function sugar and is **stripped from the public `__config`**, +so a stitch rebuilt from a JSON round-trip keeps `idempotency` (as `{}`), silently falls back to the +random key, and double-charges with nothing warning. And a key derived the obvious way — +`JSON.stringify(input.body)` — moves when the body is re-serialised in a different **key order**, which +does not produce the 409 the capture expected: it produces a **second charge**, silently. + +Three findings go the library's way, two of them against the capture's own hypotheses. A cached 500 is +**not** retried by default (`retry.on` is `[429,502,503,504]`; 500 is not in it). A 409 is not retried +either. And the case the whole scenario exists for — the server processed the charge and lost the +response — is **recovered for free**: the retry replays the stored 200 and hands the caller the real +charge id, with no query and no user code. + +Every script is standalone and offline. The measurement is always the same one: **how many charges the +fake vendor holds at the end, against how many the caller intended**. Each script prints one +`PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c2-restart-stability.ts + +# all of them +for f in docs/scenarios/proofs/unconfirmed-write/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. The whole suite takes about two seconds: every +wait is on a `manualClock`, and nothing here does real crypto or real I/O. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/unconfirmed-write/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ----------------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `c1-key-per-attempt.ts` | one key per call, or one per attempt? | **One per call.** 3 attempts → 1 key, 1 charge. A **lost response was recovered** by the retry's replay. Pagination is the exception: 3 pages → 3 keys | +| `c2-restart-stability.ts` | **DECIDING** — does the default key survive a crash? | **No.** 2 keys, **2 charges for 1 payment**. `keyOf` fixes it; a `__config` round-trip silently un-fixes it | +| `c3-derived-key-stability.ts` | is a derived key stable against re-serialisation? | **Only if it ignores the body's shape.** `JSON.stringify(body)` → 2 keys, **2 charges, and no 409** | +| `c4-cached-failure.ts` | does `retry` burn its budget on a replayed 500? | **Not by default** (1 request). With 500 in `retry.on`: 4 requests, 3 replays. `interpret` cannot veto it | +| `c5-key-body-mismatch.ts` | same key, changed body → 409. Retried? Actionable? | **Not retried, and actionable** (`status: 409` + `idempotency_key_in_use`). `verdict.accept` **swallows** it | +| `c6-ttl-expiry.ts` | a re-drive after the key is pruned | **2 charges** at 25h vs a 24h TTL, 1 at 23h. **Nothing client-side notices** — the duplicate is a clean 200 | +| `c7-timeout-ambiguity.ts` | "never sent" vs "sent, outcome unknown"? | **Indistinguishable.** Identical `StitchError`s over ledgers of **0 and 1 charges**. No event carries the key | +| `c8-assembled.ts` | all six workloads, end to end | **5 charges / 6 intended** (the sixth declined). Control: **8 / 6**, two duplicates. 43 lines of user code | + +## Files + +- `fake-payments.ts` — the vendor, as a plain `Adapter`, with real idempotency semantics: it stores the + status **and body** of the first request per key and replays it (including a stored failure, with + `Idempotent-Replayed: true`), rejects a parameter mismatch with `409 idempotency_error`, prunes + records after a TTL, and can be told to **process a charge and then lose the response**. Its + `charges` array is the ground truth every verdict is a statement about; `chargeCount()` is the + number. Two comparison modes (`canonical`, Stripe's; `bytes`, the stricter vendors') because the + difference decides whether a re-serialised body errors or double-charges. +- `keys.ts` — the three key strategies and the three body variants. `naiveKeyOf` + (`JSON.stringify(body)`), `refKeyOf` (the business fact alone), `canonicalKeyOf` (sha256 over + sorted-key JSON). `keyOf` is **synchronous** (types.ts:1118-1119), so `crypto.subtle` is not + available inside it — `node:crypto`'s `createHash` is what a real derivation would use. +- `harness.ts` — `check` / `checkSeq` / `checkCharges` / `checkAtMost` / `note` / `heading` / + `finish`. `checkCharges` prints **created and intended together**, because a bare number is not the + finding. +- `virtual-time.ts` — `drain` / `runOut`. Much lighter than the `expiring-signatures` version: nothing + here does real async work between a clock wait and the transport, so `advance` alone is faithful. + `runOut` exists so a claim can say "advance past everything" without hand-computing a retry schedule. + +## Reading the numbers honestly + +- **C2 is the finding.** `applyIdempotency` runs inside `buildRequest` (engine.ts:257), and + `buildRequest` runs once per **call** — so `headers[header] = keyOf ? keyOf(input) : randomUUID()` + (engine.ts:172) mints a fresh uuid for every call. Two runs of the same declaration with the same + input produced two keys and **2 charges for 1 payment**; so did **one stitch instance called twice**. + The docs are not wrong about this (the idempotency guide's anti-pattern callout says a double-clicked + Pay charges twice) — but they frame it as a double-click problem, and the measured failure is a + **queue re-drive**, which is both more likely and more expensive. +- **`keyOf` genuinely fixes the restart**, in configuration alone: 1 key across two fresh processes, 1 + charge, and the re-driven job was handed the ORIGINAL charge id rather than creating a new one. +- **C1 is the good news and it is bigger than it looks.** The case the scenario exists for — server + processed the charge, response lost — was **recovered by the retry with no user code**: attempt 1 + created `ch_0001` and vanished, attempt 2 carried the same key, the vendor replayed the stored 200, + and `result.data.id` was `ch_0001`. "Retry into the unknown" is the right move here precisely + because the key is per-call rather than per-attempt (`cloneReq` at engine.ts:261-264 copies headers + from a base that already carries the key). A caller-supplied `headers['idempotency-key']` also wins + (engine.ts:165-170) and is reused the same way — the seam for a job runner that mints its own ids. +- **C4 refutes the capture in the library's favour.** `retry.on` defaults to `[429,502,503,504]` + (engine.ts:612) and **500 is not in it**, so `retry: { attempts: 4 }` against a replayed 500 made + **one** request. The capture guessed 500 might be in the default set. It is not. +- **But nothing declarative can stop the burn if you add it.** With 500 in `retry.on`, 4 attempts, 3 of + them served from the recording. `Surface.interpret` cannot veto it — the retry check (engine.ts:743) + runs **above** the terminal verdict (engine.ts:775), so `interpret` is not consulted until the last + attempt (measured: 4 requests). `retry.on`'s predicate form is handed the **status and nothing else** + (measured: `[[500],[500],[500]]`), so "retry a 500 unless it is a replay" is inexpressible. +- **The one seam that works for it is a hack.** `hooks.onResponse` runs at engine.ts:705, before the + retry check, and receives the live response — rewriting `res.status` there when + `Idempotent-Replayed` is present cut the burn from **4 requests to 2** (the floor: you cannot know a + failure is cached until you have seen it twice). The price is that the caller is then told **409 for + a declined card**, and only `error.body` still says `card_declined`. +- **C3's failure mode is the opposite of the one the capture predicted.** A body-derived key that moves + does not reach a 409 — the server never sees the same key twice, so it creates a **second charge**, + status `200`, silently. Measured: `[200, 200, 200]` across three renderings of one payment, 2 + distinct keys, 2 charges. Two of the capture's named risks turn out to be non-risks in JavaScript: + `4999.0` and `4999` are one value, and a present-but-`undefined` optional field is dropped by + `JSON.stringify`. **Key order is the whole exposure.** +- **An input schema does not protect the key.** `validateInput` (engine.ts:384-409) validates each slot + and **discards the parsed value** — the coerced/defaulted/stripped object never replaces `input`. A + canonicalising body schema left the two keys distinct and the two charges in place. +- **C5 is the least dangerous failure here, and worth saying so.** A 409 means the vendor **refused**, + so the money is safe: 1 charge, at the FIRST body's amount. It is not retried (409 is not in the + default `retry.on`) and it is actionable (`status: 409`, `error.body.error.code === +'idempotency_key_in_use'`; the `message` is the generic `HTTP 409`). +- **C6 is the failure a correct key cannot prevent.** 25 hours against a 24-hour TTL: same key, **2 + charges**. At 23 hours: 1 charge. The entire difference is a queue delay the client does not control. + Nothing client-side sees it — the duplicate arrives as a clean `200` with a new charge id and **no + replay marker**, because from the vendor's side it genuinely is fresh. `timeout.total` cannot bound + it either: the budget is per call, and the re-drive is a new call (measured: still 2 charges). +- **C7's answer is "no", and the reason is not a defect.** The information is not on the client. What + IS measurable is how much StitchAPI carries: a dropped request (0 charges) and a lost response (1 + charge) produced **field-for-field identical** `StitchError`s — `name: 'StitchError'`, + `status: undefined`, `attempts: 1`, `message: 'timed out after 5000ms'`, `body: undefined`. +- **The `TimeoutError` class is flattened away.** The engine throws one (resilience.ts:17,233); `errEvt` + (engine.ts:346-355) reduces it to a message on an event and `rebuildError` (stitch.ts:489-516) + rebuilds a plain `StitchError`. The class survives **only** in `hooks.onError` (measured: + `TimeoutError/Error` — note `.name` is `'Error'`, the class never sets it), and `TimeoutError` is not + exported from any public entry point (only `RateLimitError` is, index.ts:86). So "was this a + timeout?" is a string test on the message. +- **A transport failure is retried unconditionally.** The throw path (engine.ts:675-703) retries on + `attempt < max` alone — there is no status to match `retry.on` against. Measured: + `retry: { attempts: 3, on: [] }` still made **3 requests**. "Retry a 503 but not a timeout" is not + expressible; the only way not to retry into the unknown is no retry at all. Here that is benign + (the stable key held it at 1 charge) and it is worth knowing it is not a choice. +- **C8's assembled answer is 43 lines of user code** (the block between the `BEGIN`/`END` markers, + comments and blanks excluded — two stitch declarations, a result type, and one `settleCharge` + function) and settled all six workloads: **5 charges for 6 intended payments**, the missing one being + a card the vendor declined. The control (`idempotency: true`, awaited, no recovery) produced **8 + charges for the same 6**, with duplicates from the restart (C2) and the TTL (C6). Of those 43 lines, + **19 are the recovery** — the `findByRef` stitch, the `look` helper, and the four lines that call it. + The other 24 (the charge declaration, the result type, the success and refusal branches) are what any + caller writes anyway. +- **The recovery has to be QUERY-FIRST.** The first draft of C8 queried only after an ambiguous outcome + and still double-charged on the TTL workload — a recovery that runs after the write cannot un-write + it. Asking before writing costs one extra `GET` per payment and is the only ordering that prevents + the duplicate. +- **Query-first does not close the race; the key does.** Two concurrent runs, both querying first and + both finding nothing: with `keyOf`, **1 charge**; with the default key, **2**. The capture flags the + query/decide race as the weakness of this approach — it is, and a derived key covers it exactly. +- **C8's W5 is the uncomfortable one.** A stable key makes a **recorded failure sticky** for the whole + TTL: a declined card stayed declined on the re-drive (0 charges, correct), while the random key + **never reached the recorded failure at all** and simply charged on the second run (1 charge). The + property that fixes the restart also makes a transient recorded failure permanent. Neither answer is + unambiguously right, and no configuration expresses "sticky for a decline, fresh for a blip". + +## The footguns + +- **`idempotency: true` protects a retry and nothing else.** A key that changes per call cannot dedupe + across calls, and a queue re-drive is a second call. Measured: **2 charges for 1 payment**, from a + config that reads as if it prevents exactly that. The construction nudge (stitch.ts:384-390) fires + only for a random key with **no** `retry` — so adding `retry`, the thing the guide recommends, + **silences the only warning** while leaving the restart hole wide open. +- **A `__config` JSON round-trip silently drops `keyOf` and restores the random key.** `idempotency` is + in `FN_BEARING_SLOTS` (stitch.ts:874-883) and `stripFns` removes every function-valued field + (stitch.ts:941-943), so `__config.idempotency` came back as **`{}`** — present, truthy, and therefore + still ON with the default. Measured: the rebuilt stitch charged **twice**, and nothing warned. + Nothing in the shipped code rebuilds a stitch from `__config` (the CLI, `mcp`, `diagram` and + `registry` only read it), but `__config` is designed to round-trip as JSON and a config file, a + registry row, or a cross-service handoff is an obvious use of that. +- **`keyOf: (i) => JSON.stringify(i.body)` is the obvious implementation and it is unstable.** Key + order alone moved it, and the failure is a **silent second charge**, not an error. Derive from the + business fact, or hash a canonical rendering. An input schema will not do it for you. +- **`verdict: { accept: [409], flag: 'ok' }` does not classify an idempotency conflict — it SUCCEEDS on + it.** Measured `ok: true`, with `{ error: { type: 'idempotency_error', … } }` handed back as the + call's **data**. The caller records a successful charge for the new amount; the vendor holds one + charge for the old amount and refused the amendment. (`expiring-signatures` C6 measured the identical + trap on an AWS skew 403 — the flag needs a body field that is present and falsy, and vendor error + envelopes have none.) +- **`StitchError` carries no response headers.** `status`, `attempts`, `body`, `url` — that is all + (types.ts:1657-1691). A vendor that signals a replay in a **header** (`Idempotent-Replayed`, as + Stripe does) is invisible to the caller; one that signals it in the body rides `error.body` for free. + Capturing a header takes `hooks.onResponse` or a custom `Surface.interpret`. +- **No event carries the idempotency key.** The `start` event is + `{type, name, method, url, input, at, spanId, traceId}` (types.ts:1306-1320) — no headers. So a + caller using the default random key **cannot learn which key their lost request carried**, which + makes the standard "ask the vendor about key X" recovery impossible by construction. `hooks.onRequest` + is the only seam that sees it; a `keyOf` key can simply be recomputed. +- **Pagination mints a NEW key per page.** `buildRequest` is called per page (engine.ts:936,940), so + `applyIdempotency` runs again each time: measured in C1 (e) — 3 pages, **3 distinct keys**. Harmless + for a read; a paginated write surface with `idempotency: true` would be dedupe-free after page 1. + "Once per logical call" is really "once per request BUILD". +- **`retry.on` cannot exclude a timeout.** A transport failure is retried whenever attempts remain, + whatever `retry.on` says — measured 3 requests with `on: []`. If retrying an unconfirmed write is + unacceptable for a given endpoint, the only lever is `retry.attempts: 1`. + +## What is NOT measured here + +- **A real vendor.** `fake-payments.ts` implements the semantics Stripe documents; it is not Stripe. + Nothing here demonstrates that any particular vendor prunes at 24 hours, compares parameters + canonically, or sets `Idempotent-Replayed`. +- **Concurrent requests with the same key at the vendor.** Real servers reject the second in-flight + request for a live key (`409 idempotency_key_in_use` while the first is still running); this fake + processes them in arrival order. The C8 race measures the CLIENT-side race the capture names (two + workers, query-then-write), not the server's in-flight lock. +- **The expiry race.** Two requests arriving as a key expires can both pass the existence check on a + real server. The prune here is deterministic and synchronous, so that window does not exist in this + fake. +- **Persist-intent-first.** The capture's durable two-phase answer (write "about to charge X" locally, + charge, mark done) is a workflow, not a client-library feature, and nothing in this directory + measures it. +- **`store`-backed coordination.** Whether a shared `StitchStore` could hold a key across processes — + making a restart-stable key without `keyOf` — was not attempted. The engine reads no store in + `applyIdempotency`, so it would be user code either way. diff --git a/docs/scenarios/proofs/unconfirmed-write/c1-key-per-attempt.ts b/docs/scenarios/proofs/unconfirmed-write/c1-key-per-attempt.ts new file mode 100644 index 00000000..a3eef64e --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c1-key-per-attempt.ts @@ -0,0 +1,245 @@ +// C1 — is the SAME idempotency key sent on every attempt of one call? +// +// If the key were minted per ATTEMPT rather than per call, `retry` would be a charge multiplier: the +// server would see N unrelated writes and create N charges. This is the property everything else +// rests on, so it is measured across a retry with a real backoff, across a transport failure, and +// across the case the whole scenario is about — a response lost AFTER the charge was created. +// +// MEASURED: one key per logical call, reused by every attempt. +// (a) 3 attempts against 503s → 3 requests, ONE distinct key, 1 charge for 1 intended. +// (b) THE GOOD RESULT. The server processes the charge and loses the response; the retry replays +// the STORED 200. 1 charge for 1 intended, and the caller gets the real charge id — the retry +// turned "outcome unknown" into "outcome known", which is exactly what the key is for. +// (c) A TRANSPORT failure is retried too — and `retry.on` cannot stop it (see C7). Same key, so +// the 3 attempts still cost 1 charge. +// (d) The mechanism: `applyIdempotency` runs inside `buildRequest` (engine.ts:257), once per +// logical call, and each attempt gets `cloneReq(baseReq)` (engine.ts:261-264,646) — a fresh +// header object COPIED from a base that already carries the key. Measured through +// `hooks.onRequest`, which is the only user-visible seam that sees the header. +// (e) …which is also the property's boundary: PAGINATION calls `buildRequest` per PAGE +// (engine.ts:936,940), so a paginated stitch mints a fresh key per page. Measured: 3 pages, 3 +// distinct keys. "Once per logical call" is really "once per request BUILD". +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c1-key-per-attempt.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; +/** Fixed and long enough to be unmistakable in the arrival times; `backoff.max` would clamp a bigger one. */ +const BACKOFF = { curve: 'fixed', base: '2s', max: '10s' } as const; + +async function main(): Promise { + heading( + 'C1 — three attempts of one call: one key, or three? (three keys = three charges)', + ); + + // ── (a) a plain retry against a failing host ────────────────────────────────────────────── + // 503 is in `retry.on`'s default set, so all three attempts run. The server creates a charge + // only on a request it can process; here it fails them all, so the interesting number is the + // key count. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, failChargeOn: [1, 2, 3] }); + const sent: string[] = []; + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 3, on: [500, 503], backoff: BACKOFF }, + hooks: { + onRequest: ({ req }) => { + sent.push(req?.headers['Idempotency-Key'] ?? '(absent)'); + }, + }, + clock, + }); + + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 30_000); + const result = await pending; + + check('(a) requests that reached the wire', pay.calls.length, 3); + check( + '(a) DISTINCT keys across those 3 attempts', + pay.distinctKeys(), + 1, + ); + checkSeq( + '(a) key seen by hooks.onRequest, per attempt (uuid elided)', + sent.map((k) => k.slice(0, 8)), + [sent[0]?.slice(0, 8), sent[0]?.slice(0, 8), sent[0]?.slice(0, 8)], + ); + checkSeq( + '(a) arrival time per attempt (virtual seconds from t0)', + pay.calls.map((c) => (c.at - T0) / 1000), + [0, 2, 4], + ); + checkSeq( + '(a) replayed? per attempt — attempts 2-3 hit the STORED failure', + pay.replays(), + [false, true, true], + ); + check('(a) result.ok — the call failed', result.ok, false); + note( + '(a) note', + 'attempts 2 and 3 were served from the record, not processed — that is C4', + ); + } + + // ── (b) THE CASE: processed, then the response was lost ────────────────────────────────── + // Request 1 creates the charge and never answers. The per-attempt timeout fires, the engine + // retries with the SAME key, and request 2 is served the stored 200. One charge, and the caller + // learns the charge id it would otherwise never have seen. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1] }); + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 3, backoff: BACKOFF }, + timeout: { perAttempt: '5s' }, + clock, + }); + + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check('(b) requests that reached the wire', pay.calls.length, 2); + check('(b) DISTINCT keys', pay.distinctKeys(), 1); + checkSeq('(b) replayed? per request', pay.replays(), [false, true]); + check('(b) the call SUCCEEDED on the replay', result.ok, true); + check( + '(b) the caller got the id of the charge request 1 created', + (result.data as { id?: string } | null)?.id, + pay.charges[0]?.id, + ); + checkCharges('(b)', pay.chargeCount(), 1, 1); + note( + '(b) the point', + 'the retry recovered the outcome of a request whose response was lost — no query needed', + ); + } + + // ── (c) a transport failure (connection dies before processing) ────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, dropBeforeProcessingOn: [1, 2] }); + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 3, backoff: BACKOFF }, + timeout: { perAttempt: '5s' }, + clock, + }); + + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check('(c) requests that reached the wire', pay.calls.length, 3); + check('(c) DISTINCT keys across 3 attempts', pay.distinctKeys(), 1); + check('(c) the call SUCCEEDED on attempt 3', result.ok, true); + checkCharges('(c)', pay.chargeCount(), 1, 1); + } + + // ── (d) the key is applied ONCE, before the loop — and re-applied per attempt is not needed ── + // A caller-supplied `headers['idempotency-key']` wins (case-insensitively) and is likewise + // reused by every attempt. That is the seam a job runner that owns its own key ids would use. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, failChargeOn: [1, 2] }); + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 2, on: [500], backoff: BACKOFF }, + clock, + }); + + const pending = call({ + body: PAYMENT, + headers: { 'idempotency-key': 'job-7f3a-attempt-owned' }, + }).safe(); + await runOut(clock, 30_000); + await pending; + + checkSeq('(d) keys on the wire', pay.keys(), [ + 'job-7f3a-attempt-owned', + 'job-7f3a-attempt-owned', + ]); + note( + '(d) mechanism', + 'applyIdempotency skips when a header of that name is already present (engine.ts:165-170)', + ); + } + + // ── (e) the boundary of "once per call": PAGINATION mints a key per PAGE ───────────────── + // `buildRequest` is called once per page (engine.ts:936,940), so `applyIdempotency` runs again + // each time. Harmless for a read; a paginated WRITE surface would be dedupe-free after page 1. + // Included here because it is the exact edge of the property (a)-(d) establish. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + let page = 0; + const call = stitch({ + method: 'POST', + url: 'https://api.pay.test/v1/charges/search', + // The fake treats every non-GET as a charge attempt, so these page requests land in + // its ledger; only the KEYS are read here, which is the whole point of the case. + adapter: async (req) => { + page++; + await pay.adapter()(req); + return { + status: 200, + headers: {}, + body: { items: [page], next: page < 3 ? page : null }, + }; + }, + idempotency: true, + retry: { attempts: 2 }, + paginate: { + items: (v: unknown) => (v as { items: unknown[] }).items, + next: (prev: unknown) => { + const p = (prev as { next: number | null }).next; + return p === null ? undefined : { query: { p } }; + }, + }, + clock, + }); + await call({ body: PAYMENT }).safe(); + + check('(e) pages fetched', pay.calls.length, 3); + check('(e) DISTINCT keys across those 3 pages', pay.distinctKeys(), 3); + note( + '(e) reading it', + 'the "once per logical call" guarantee is once per REQUEST BUILD, and pagination builds one per page', + ); + } + + finish( + 'C1', + 'the key is minted ONCE per logical call and every attempt carries it — 3 attempts, 1 distinct key, 1 charge; a response lost after processing was RECOVERED by the retry (stored 200 replayed), and a caller-supplied header wins and is likewise reused. The boundary: pagination builds a request per page, so 3 pages minted 3 distinct keys', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c2-restart-stability.ts b/docs/scenarios/proofs/unconfirmed-write/c2-restart-stability.ts new file mode 100644 index 00000000..ea4995fc --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c2-restart-stability.ts @@ -0,0 +1,228 @@ +// C2 — THE DECIDING CLAIM. Is the DEFAULT idempotency key stable across a PROCESS RESTART? +// +// A key that survives a retry protects against the failure you can see. A key that survives a +// RESTART protects against the one that costs money: a worker dies mid-charge, the queue re-drives +// the job, and the second run is a second charge. Every claim in this file builds the stitch FRESH +// before each run — a new `stitch({...})` from the same declaration and the same input — which is +// what a re-driven job actually does. The vendor persists across; only the client is new. +// +// MEASURED, and the capture's central worry is CONFIRMED: +// (a) The default key is `randomUUID()` (engine.ts:172), evaluated per logical call. Two runs of +// the same declaration with the same input produced TWO DISTINCT keys and TWO CHARGES for one +// intended payment. The configuration is `idempotency: true` + `retry` — the exact shape the +// docs recommend — and it double-charges. +// (b) It is not even stable within ONE process: two calls of the SAME stitch instance also mint +// two keys. So this is not about restarts at all; the key is per-CALL, and a restart is just +// the most likely way two calls happen. +// (c) `keyOf` fixes it: same declaration, same input, fresh process → ONE key, ONE charge. +// (d) THE FOOTGUN. `keyOf` is function sugar and is STRIPPED from the public `__config` +// (stitch.ts:936-940). A stitch rebuilt from `__config` — a JSON round-trip through a +// registry, `serve`, MCP, a config file — keeps `idempotency` and SILENTLY LOSES `keyOf`, +// falling back to the random default. Measured: `__config.idempotency` is `{}`, the rebuilt +// stitch charged twice, and nothing warned. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c2-restart-stability.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { ManualClock } from '../../../../packages/core/src/testing'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; + +/** + * One run of the job, in a fresh "process": build the stitch from the declaration, call it once with + * the payment, return. The vendor (`pay`) is the only thing that persists. + * + * This is the whole experiment. Everything a real queue re-drive does that matters is here: the + * module-level `stitch({...})` is evaluated again, and the input is rebuilt from the same durable + * record. + */ +async function driveJob( + pay: FakePayments, + clock: ManualClock, + extra: Partial, + body: unknown = PAYMENT, +): Promise { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + retry: { attempts: 3, backoff: { curve: 'fixed', base: '2s' } }, + timeout: { perAttempt: '5s' }, + clock, + ...extra, + }); + const pending = call({ body }).safe(); + await runOut(clock, 60_000); + await pending; +} + +async function main(): Promise { + heading( + 'C2 — the same job, re-driven after a crash: one key or two? (two keys = two charges)', + ); + + // ── (a) THE DECIDING MEASUREMENT ───────────────────────────────────────────────────────── + // Run 1 creates the charge and loses the response — the worker dies knowing nothing. The queue + // re-drives the job. Run 2 is a fresh process with the same declaration and the same input. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2, 3] }); + + await driveJob(pay, clock, { idempotency: true }); // the worker that died + await driveJob(pay, clock, { idempotency: true }); // the queue's re-drive + + note( + '(a) keys seen by the vendor', + JSON.stringify(pay.keys().map((k) => k?.slice(0, 8))), + ); + check( + '(a) DISTINCT keys across the two runs', + pay.distinctKeys(), + // 3 attempts in run 1 (all lost) share one key; run 2's 3 attempts share another. + 2, + ); + checkCharges('(a) `idempotency: true`', pay.chargeCount(), 1, 2); + checkSeq( + '(a) the two charges the vendor now holds', + pay.charges.map((c) => `${c.id}:${c.ref}:${String(c.amount)}`), + ['ch_0001:inv-1001:4999', 'ch_0002:inv-1001:4999'], + ); + note( + '(a) the configuration that did this', + "idempotency: true + retry — the shape the idempotency guide's first example uses", + ); + } + + // ── (b) it is per-CALL, not per-process ────────────────────────────────────────────────── + // The same stitch INSTANCE, called twice. No restart involved. Two keys, two charges. The + // restart in (a) was not the cause; it was just the reason two calls happened. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 2 }, + clock, + }); + + await call({ body: PAYMENT }).safe(); + await call({ body: PAYMENT }).safe(); + + check( + '(b) DISTINCT keys from ONE stitch instance called twice', + pay.distinctKeys(), + 2, + ); + checkCharges('(b) same instance, twice', pay.chargeCount(), 1, 2); + note( + '(b) mechanism', + '`applyIdempotency` runs inside `buildRequest` (engine.ts:257), and `buildRequest` runs per call', + ); + } + + // ── (c) `keyOf` is restart-stable ──────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2, 3] }); + + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + + checkSeq( + '(c) DISTINCT keys across the two runs', + [pay.distinctKeys(), ...new Set(pay.keys())], + [1, 'chg-inv-1001'], + ); + checkCharges('(c) `keyOf: refKeyOf`', pay.chargeCount(), 1, 1); + checkSeq( + '(c) replayed? per request — run 1 is requests 1-3, the re-drive is request 4', + pay.replays(), + // Request 1 creates the charge and is lost; 2 and 3 are served from the record and are + // ALSO lost (so run 1 fails outright); the re-drive's first request is served from the + // record and comes back, because only requests 1-3 were configured lost. + [false, true, true, true], + ); + note( + '(c) the re-driven job', + 'was handed the ORIGINAL charge id, not a new charge', + ); + } + + // ── (d) THE FOOTGUN: a config round-trip silently drops `keyOf` ────────────────────────── + // `keyOf` is function sugar. `redactConfig` strips every function-valued field so `__config` + // round-trips as JSON (stitch.ts:936-940). The `idempotency` SLOT survives — as `{}` — so the + // rebuilt stitch still has idempotency ON, with the random default. Everything reads correct. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + const declared = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + retry: { attempts: 2 }, + clock, + }); + const published = JSON.parse( + JSON.stringify( + (declared as unknown as { __config: unknown }).__config, + ), + ) as Partial; + + check( + '(d) `__config.idempotency` after the round-trip', + JSON.stringify(published.idempotency), + '{}', + ); + check( + '(d) does the slot still LOOK configured?', + published.idempotency !== undefined, + true, + ); + + const rebuilt = stitch({ + ...published, + adapter: pay.adapter(), + clock, + }) as Stitch; + await rebuilt({ body: PAYMENT }).safe(); + await rebuilt({ body: PAYMENT }).safe(); + + check( + '(d) DISTINCT keys from the rebuilt stitch, called twice', + pay.distinctKeys(), + 2, + ); + checkCharges('(d) rebuilt from `__config`', pay.chargeCount(), 1, 2); + note( + '(d) what warned', + 'nothing — the construction nudge only fires for a random key with NO `retry`, and `retry` survived the round-trip', + ); + } + + finish( + 'C2', + 'the DEFAULT key is NOT restart-stable and not even call-stable — it is `randomUUID()` per call, so a re-driven job created 2 charges for 1 intended payment under `idempotency: true` + `retry`; `keyOf` fixes it (1 key, 1 charge) but is STRIPPED by a `__config` JSON round-trip, which silently restores the double charge', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c3-derived-key-stability.ts b/docs/scenarios/proofs/unconfirmed-write/c3-derived-key-stability.ts new file mode 100644 index 00000000..16bef15d --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c3-derived-key-stability.ts @@ -0,0 +1,219 @@ +// C3 — is a `keyOf`-derived key restart-stable, and is it stable against BODY RE-SERIALISATION? +// +// C2 established that a derived key survives a restart. This claim asks the harder half: the second +// process rebuilds the body from a database row, or a decimal library, or a struct with an optional +// field — the PARAMETERS are identical and the OBJECT is not. A key derived from that object moves +// with it, and a key that moves is a second charge. +// +// MEASURED, and it splits by what the derivation reads: +// (a) `JSON.stringify(input.body)` — the obvious implementation — is NOT stable. Key ORDER alone +// produced a different key and A SECOND CHARGE. Note what did NOT happen: no 409. The server +// never got to compare parameters, because it never saw the same key twice. A body-derived key +// fails by SILENTLY charging twice, which is worse than the error the capture expected. +// (b) Two re-serialisations that DON'T move the key: `1` vs `1.0` (JS has one number type, so +// `4999.0` and `4999` are the same value), and a present-but-`undefined` optional field +// (`JSON.stringify` drops it). Both are stable. The capture listed number formatting as a +// risk; in JavaScript it is not one. +// (c) `refKeyOf` — the business fact alone — is stable across all three variants: 1 key, 1 charge. +// (d) `canonicalKeyOf` — a sha256 over sorted-key JSON — is likewise stable across all three, and +// still moves when the AMOUNT changes, which is the property you actually want. +// (e) An `input.body` SCHEMA does not save you: `validateInput` (engine.ts:384-409) validates and +// DISCARDS the parsed value, so `keyOf` sees the caller's raw object, not a normalised one. A +// Zod schema with `.default()` / key stripping canonicalises nothing for keying purposes. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c3-derived-key-stability.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { ManualClock } from '../../../../packages/core/src/testing'; +import type { StitchInput } from '../../../../packages/core/src/types'; +import type { Validator } from '../../../../packages/core/src/validator'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { bodyVariants, canonicalKeyOf, naiveKeyOf, refKeyOf } from './keys'; +import type { Payment } from './keys'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; + +/** + * Drive the SAME logical payment once per body variant, each from a freshly constructed stitch (a + * fresh process). The vendor persists, so the charge ledger accumulates across all three. + */ +async function driveVariants( + keyOf: (input: StitchInput) => string, + clock: ManualClock, + pay: FakePayments, +): Promise { + for (const variant of bodyVariants(PAYMENT)) { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf }, + retry: { attempts: 2 }, + clock, + }); + await call({ body: variant.body }).safe(); + } +} + +async function main(): Promise { + heading( + 'C3 — the same payment, three ways of building the body: does the key move? (moving = extra charges)', + ); + + note( + 'the three bodies', + JSON.stringify(bodyVariants(PAYMENT).map((v) => v.label)), + ); + + // ── (a) `JSON.stringify(body)` — the obvious derivation ────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + await driveVariants(naiveKeyOf, clock, pay); + + checkSeq('(a) keys on the wire', pay.keys(), [ + 'chg-{"ref":"inv-1001","amount":4999,"currency":"usd"}', + 'chg-{"currency":"usd","amount":4999,"ref":"inv-1001"}', + 'chg-{"ref":"inv-1001","amount":4999,"currency":"usd"}', + ]); + check('(a) DISTINCT keys for one payment', pay.distinctKeys(), 2); + checkCharges('(a) `JSON.stringify(body)`', pay.chargeCount(), 1, 2); + checkSeq( + '(a) status per request — note there is NO 409', + pay.statuses(), + [200, 200, 200], + ); + note( + '(a) which variant broke it', + 'key re-ordering. `1.0` and a present-but-undefined field did NOT move the key — variant 3 matched variant 1', + ); + note( + '(a) the failure shape', + 'a silent second charge, not the error the capture expected — the server never saw a repeated key', + ); + } + + // ── (b) the two re-serialisations that are harmless in JavaScript ──────────────────────── + // Worth isolating, because the capture named number formatting as a risk and it is not one here: + // JS has a single number type, so `4999.0 === 4999` and both serialise to `4999`. + { + check( + '(b) `4999.0` and `4999` serialise the same', + JSON.stringify({ amount: 4999.0 }), + JSON.stringify({ amount: 4999 }), + ); + check( + '(b) a present-but-undefined optional field is dropped by JSON.stringify', + JSON.stringify({ ref: 'x', description: undefined }), + JSON.stringify({ ref: 'x' }), + ); + note( + '(b) what remains', + 'key ORDER — the one difference `JSON.stringify` preserves and the vendor does not care about', + ); + } + + // ── (c) the business fact alone ───────────────────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + await driveVariants(refKeyOf, clock, pay); + + checkSeq('(c) keys on the wire', pay.keys(), [ + 'chg-inv-1001', + 'chg-inv-1001', + 'chg-inv-1001', + ]); + checkCharges('(c) `refKeyOf`', pay.chargeCount(), 1, 1); + checkSeq('(c) replayed? per request', pay.replays(), [ + false, + true, + true, + ]); + } + + // ── (d) a canonical hash of the whole parameter set ────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + await driveVariants(canonicalKeyOf, clock, pay); + + check('(d) DISTINCT keys across the 3 variants', pay.distinctKeys(), 1); + checkCharges('(d) `canonicalKeyOf`', pay.chargeCount(), 1, 1); + + // …and it still separates two genuinely different payments, which `refKeyOf` would too but + // a constant would not. This is the property that makes a derived key safe rather than just + // stable: it must be unique across distinct writes. + const changed = canonicalKeyOf({ + body: { ...PAYMENT, amount: 5999 }, + }); + check( + '(d) a different AMOUNT produces a different key', + changed !== canonicalKeyOf({ body: PAYMENT }), + true, + ); + } + + // ── (e) an input SCHEMA does not canonicalise the body for `keyOf` ─────────────────────── + // `validateInput` runs before `buildRequest`, so it is tempting to think a schema normalises the + // input the key is derived from. It does not: the validated value is discarded (engine.ts:401). + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + // A validator that returns a CONSTANT canonical value. If `keyOf` saw the validated value, + // every key would be identical — so two distinct keys is a direct measurement that it does + // not. (`Validator.validate` is async, hence the `Promise.resolve`.) + const canonicalising: Validator = { + validate: () => + Promise.resolve({ + ok: true as const, + value: { ref: 'inv-1001', amount: 4999, currency: 'usd' }, + }), + }; + for (const variant of bodyVariants(PAYMENT)) { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + input: { body: canonicalising }, + idempotency: { keyOf: naiveKeyOf }, + retry: { attempts: 2 }, + clock, + }); + await call({ body: variant.body }).safe(); + } + + check( + '(e) DISTINCT keys with a canonicalising body schema in place', + pay.distinctKeys(), + 2, + ); + checkCharges( + '(e) schema + `JSON.stringify(body)`', + pay.chargeCount(), + 1, + 2, + ); + note( + '(e) why', + '`validateInput` checks the value and throws away the parsed result — `keyOf` gets the caller’s raw object', + ); + } + + finish( + 'C3', + 'a derived key is restart-stable but only as stable as what it reads: `JSON.stringify(body)` moved on KEY ORDER alone and produced 2 charges for 1 payment with NO 409, while `refKeyOf` and a canonical sha256 held at 1 key / 1 charge across all three variants; number formatting and undefined fields are harmless in JS, and an input SCHEMA does not canonicalise what `keyOf` sees', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c4-cached-failure.ts b/docs/scenarios/proofs/unconfirmed-write/c4-cached-failure.ts new file mode 100644 index 00000000..203945ba --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c4-cached-failure.ts @@ -0,0 +1,309 @@ +// C4 — the CACHED FAILURE. The vendor stored the outcome of the first request for a key, that +// outcome was a 500, and every later request with that key gets the same 500 forever. Does `retry` +// burn its whole budget against a recording? Can a cached failure be told apart from a fresh one? +// +// MEASURED, and the first half goes the library's way: +// (a) NOT retried by default. `retry.on` defaults to `[429, 502, 503, 504]` (engine.ts:612) and +// 500 is not in it, so `retry: { attempts: 4 }` produced ONE request. The capture guessed the +// default might include 500; it does not. +// (b) But adding 500 to `retry.on` — the natural thing to do, since a 500 usually IS transient — +// burns every attempt against the recording. 4 attempts, 4 requests, 3 of them served from +// the record, all four identical. Nothing changed and nothing could have. +// (c) The replay IS distinguishable on the wire: the vendor sets `Idempotent-Replayed: true`, and +// `hooks.onResponse` sees it. But `StitchError` carries `status`/`attempts`/`body`/`url` and +// NO HEADERS, so by the time the failure reaches the caller the marker is gone. A replay +// marker in the BODY survives; one in a HEADER does not. +// (d) `Surface.interpret` CANNOT veto a status-driven retry. The retry check (engine.ts:743) sits +// ABOVE the terminal verdict (engine.ts:775), so `interpret` is not consulted until the last +// attempt: 4 requests, and only then the message. `retry.on`'s predicate form is no help +// either — it is handed the STATUS and nothing else (measured: `[[500],[500],[500]]`). +// (e) What DOES work, in 3 lines: `hooks.onResponse` runs at engine.ts:705, BEFORE the retry +// check, and it is handed the live `res`. Rewriting `res.status` there when the replay marker +// is present takes the response out of `retry.on`. Measured: 2 requests instead of 4 — the +// floor, since a failure cannot be known to be cached until it has been seen twice. It works +// and it is a hack: the status the caller is then told (409) is the one the hook invented, +// and only `error.body` still says `card_declined`. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c4-cached-failure.ts +import { stitch, verdictOf } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; +const BACKOFF = { curve: 'fixed', base: '2s', max: '10s' } as const; +/** The declined card is stored on request 1; every later request for that key replays it. */ +const DECLINED = { failChargeOn: [1], failStatus: 500 } as const; + +async function main(): Promise { + heading( + 'C4 — the vendor replays a stored 500: does `retry` burn its budget against a recording?', + ); + + const rig = ( + extra: Partial, + ): { + pay: FakePayments; + call: ReturnType; + clock: ReturnType; + } => { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, ...DECLINED }); + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + clock, + ...extra, + }); + return { pay, call, clock }; + }; + + // ── (a) is a 500 retried by default? ──────────────────────────────────────────────────── + { + const { pay, call, clock } = rig({ retry: { attempts: 4 } }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check( + '(a) requests that reached the wire, with `retry: { attempts: 4 }`', + pay.calls.length, + 1, + ); + check('(a) status the caller was given', result.error?.status, 500); + note( + '(a) why', + '`retry.on` defaults to [429,502,503,504] (engine.ts:612); 500 is not in it', + ); + checkCharges( + '(a) the card was DECLINED, so 0 is the correct outcome', + pay.chargeCount(), + 1, + 0, + ); + } + + // ── (b) and if 500 is added to `retry.on`? ────────────────────────────────────────────── + { + const { pay, call, clock } = rig({ + retry: { + attempts: 4, + on: [429, 500, 502, 503, 504], + backoff: BACKOFF, + }, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check('(b) requests that reached the wire', pay.calls.length, 4); + checkSeq( + '(b) status per request', + pay.statuses(), + [500, 500, 500, 500], + ); + checkSeq( + '(b) replayed? per request — 3 of the 4 were a recording', + pay.replays(), + [false, true, true, true], + ); + check( + '(b) DISTINCT keys — every attempt hit the same record', + pay.distinctKeys(), + 1, + ); + check('(b) attempts reported to the caller', result.error?.attempts, 4); + note( + '(b) what the extra 3 attempts changed', + 'nothing — the response was byte-identical each time, by construction', + ); + } + + // ── (c) can the caller SEE that it was a replay? ──────────────────────────────────────── + { + const seenHeaders: (string | undefined)[] = []; + const { call, clock } = rig({ + retry: { attempts: 2, on: [500], backoff: BACKOFF }, + hooks: { + onResponse: ({ res }) => { + seenHeaders.push(res?.headers['idempotent-replayed']); + }, + }, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + checkSeq( + '(c) `Idempotent-Replayed` per response, as hooks.onResponse sees it', + seenHeaders.map((h) => h ?? '(absent)'), + ['(absent)', 'true'], + ); + checkSeq( + '(c) fields on the StitchError the caller got', + Object.keys(result.error ?? {}).sort(), + ['attempts', 'body', 'name', 'status', 'url'], + ); + check( + '(c) can the caller read the replay header off the error?', + 'headers' in (result.error ?? {}), + false, + ); + check( + '(c) the BODY does survive to the caller', + (result.error?.body as { error?: { code?: string } } | undefined) + ?.error?.code, + 'card_declined', + ); + note( + '(c) the rule', + 'a replay marker in a header needs a hook or a custom surface to capture it; one in the body rides `StitchError.body` for free', + ); + } + + // ── (d) can `Surface.interpret` or a `retry.on` predicate stop the burn? ──────────────── + { + const replayAware: Surface = { + id: 'http+replay', + interpret: (res, cfg) => { + if (res.headers['idempotent-replayed'] === 'true') + return { + ok: false, + message: 'cached failure replayed — do not retry', + status: res.status, + }; + return verdictOf(res, cfg) ?? { ok: true, data: res.body }; + }, + }; + const { pay, call, clock } = rig({ + kind: replayAware, + retry: { attempts: 4, on: [500], backoff: BACKOFF }, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check( + '(d) requests, with an `interpret` that rejects every replay', + pay.calls.length, + 4, + ); + check( + '(d) the message DID reach the caller — on the last attempt', + result.error?.message, + 'cached failure replayed — do not retry', + ); + note( + '(d) why it cannot veto', + 'engine.ts:743 (retry on status) runs before engine.ts:775 (the terminal verdict)', + ); + + // …and the predicate form of `retry.on` is handed the status alone. + const predicateArgs: unknown[][] = []; + const { + pay: pay2, + call: call2, + clock: clock2, + } = rig({ + retry: { + attempts: 3, + // Typed as the widest thing `retry.on` accepts, then spread with the extra args + // captured — the point of the measurement is exactly HOW MANY it is handed. + on: ((...args: unknown[]) => { + predicateArgs.push(args); + return true; + }) as (status: number) => boolean, + backoff: BACKOFF, + }, + }); + const pending2 = call2({ body: PAYMENT }).safe(); + await runOut(clock2, 60_000); + await pending2; + + checkSeq( + '(d) arguments `retry.on`’s predicate receives per call', + predicateArgs, + [[500], [500], [500]], + ); + check('(d) requests under that predicate', pay2.calls.length, 3); + note( + '(d) the consequence', + 'no predicate can say “retry a 500 unless it is a replay” — the response is not in scope', + ); + } + + // ── (e) the seam that DOES work: rewrite the status in `hooks.onResponse` ─────────────── + // Three lines. `onResponse` runs at engine.ts:705, before the retry check, and receives the live + // response object — so a status it writes there is the status `retry.on` matches against. + { + const { pay, call, clock } = rig({ + retry: { attempts: 4, on: [500], backoff: BACKOFF }, + hooks: { + onResponse: ({ res }) => { + if (res?.headers['idempotent-replayed'] === 'true') + (res as { status: number }).status = 409; + }, + }, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 60_000); + const result = await pending; + + check( + '(e) requests, with the replay taken out of `retry.on` by the hook (was 4)', + pay.calls.length, + // Attempt 1's 500 is genuinely fresh, so it is retried once — correctly. Attempt 2 is + // the replay, the hook rewrites it, and the loop stops there. The burn is bounded at + // ONE wasted attempt, which is the least any client-side rule could achieve: you cannot + // know a failure is cached until you have seen it twice. + 2, + ); + checkSeq('(e) replayed? per request', pay.replays(), [false, true]); + check( + '(e) status the caller was told', + result.error?.status, + // The invented one. The real outcome was a 500 (card declined); the caller is handed + // the hook's 409, and only `error.body` still says `card_declined`. + 409, + ); + check( + '(e) `error.body` still carries the truth', + (result.error?.body as { error?: { code?: string } } | undefined) + ?.error?.code, + 'card_declined', + ); + note( + '(e) the cost', + 'a hook that rewrites `res.status` is lying to every other reader of the response — the trace, the circuit, and the caller, who is told 409 for a declined card', + ); + checkCharges( + '(e) declined card, 0 is correct', + pay.chargeCount(), + 1, + 0, + ); + } + + finish( + 'C4', + 'a cached 500 is NOT retried by default (1 request under `retry: { attempts: 4 }`) — the capture’s worry does not hold for the default `retry.on`; adding 500 to `retry.on` burns all 4 attempts against the recording, and NOTHING declarative can stop it: `interpret` runs after the retry check (4 requests) and `retry.on`’s predicate sees only the status. The replay marker is visible to `hooks.onResponse` and absent from `StitchError`, which carries no headers', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c5-key-body-mismatch.ts b/docs/scenarios/proofs/unconfirmed-write/c5-key-body-mismatch.ts new file mode 100644 index 00000000..b263375e --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c5-key-body-mismatch.ts @@ -0,0 +1,201 @@ +// C5 — the KEY/BODY MISMATCH: the same idempotency key arrives with different parameters, and the +// vendor answers `409 idempotency_error`. Is it retried (it must not be)? Does the caller get +// something they can act on? +// +// This is the failure `stripe-ruby#431` is about. It is also, measured here, the LEAST dangerous of +// the failures in this scenario — because a 409 means the vendor REFUSED, so nothing was charged +// twice. The dangerous version of a re-serialised body is C3's: a key that moves with the body never +// produces a 409 at all, it produces a second charge. +// +// MEASURED: +// (a) NOT retried. 409 is not in `retry.on`'s default set, so one request, one failure. Right by +// default, and `retry: { attempts: 4 }` changed nothing. +// (b) The caller gets a `StitchError` with `status: 409` and the vendor's error payload on +// `.body` — `idempotency_key_in_use` is legible and actionable. `.message` is the generic +// `HTTP 409`, so the actionable part is `.body`, not the message. +// (c) The ledger says the refusal was real: 1 charge, for the FIRST body. The second body was +// never applied — which is the correct outcome and not an obvious one. +// (d) THE FOOTGUN. `verdict: { accept: [409], flag: 'ok' }` — the pure-config classification a +// reader might reach for — SWALLOWS the error: the call reports `ok: true` and hands the +// caller the `idempotency_error` payload AS ITS DATA. The caller believes a charge for the +// NEW amount succeeded. It did not; the vendor refused, and the charge that exists is for the +// old amount. (`expiring-signatures` C6 measured the same trap on an AWS skew 403; the shape +// generalises to any vendor whose error envelope has no `ok` field.) +// (e) Whether a re-serialised body reaches a 409 at all depends on the VENDOR's comparison. Under +// Stripe's canonical parameter comparison, a re-ordered body with a STABLE key is not a +// mismatch: 1 charge, no 409. Under a byte-comparing vendor it is: measured, a 409 and 1 +// charge. Either way the money is safe; only the error is different. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c5-key-body-mismatch.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { bodyVariants, refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; +/** The same ref — so the same `refKeyOf` key — with an amount that changed between processes. */ +const AMENDED: Payment = { ...PAYMENT, amount: 5999 }; + +async function main(): Promise { + heading( + 'C5 — the same key, a different body: a 409. Retried? Actionable? Or swallowed?', + ); + + // ── (a) + (b) + (c) the default handling ──────────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + const build = (): ReturnType => + stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + retry: { + attempts: 4, + backoff: { curve: 'fixed', base: '2s', max: '10s' }, + }, + clock, + }); + + await build()({ body: PAYMENT }).safe(); // the charge that ran + const pending = build()({ body: AMENDED }).safe(); // the re-drive with an amended amount + await runOut(clock, 60_000); + const result = await pending; + + check( + '(a) requests for the amended body, with `retry: { attempts: 4 }`', + pay.calls.length - 1, + 1, + ); + checkSeq('(a) status per request', pay.statuses(), [200, 409]); + note( + '(a) why', + '`retry.on` defaults to [429,502,503,504] (engine.ts:612); 409 is not in it', + ); + + check('(b) status on the StitchError', result.error?.status, 409); + check( + '(b) message on the StitchError', + result.error?.message, + 'HTTP 409', + ); + check( + '(b) the vendor’s error code, off `error.body`', + (result.error?.body as { error?: { code?: string } } | undefined) + ?.error?.code, + 'idempotency_key_in_use', + ); + + checkCharges('(c) the 409 refused the write', pay.chargeCount(), 1, 1); + check( + '(c) the amount actually charged is the FIRST body’s', + pay.charges[0]?.amount, + 4999, + ); + } + + // ── (d) THE FOOTGUN: `verdict.accept` swallows it ─────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + const build = (): ReturnType => + stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + // The pure-config classification: "a 409 is a normal outcome, tell me about it via + // the body's `ok` flag". The vendor's error envelope has no `ok` field, so the flag + // is ABSENT — which `verdict` treats as "no signal" (surface.ts:180-190) — and + // `accept` alone succeeds on the 409. + verdict: { accept: [409], flag: 'ok' }, + clock, + }); + + await build()({ body: PAYMENT }).safe(); + const result = await build()({ body: AMENDED }).safe(); + + check('(d) result.ok on the 409', result.ok, true); + check( + '(d) what the caller was handed AS DATA', + (result.data as { error?: { type?: string } } | null)?.error?.type, + 'idempotency_error', + ); + check('(d) result.error', result.error, null); + checkCharges( + '(d) what the caller now believes is a 5999 charge', + pay.chargeCount(), + 1, + 1, + ); + check( + '(d) the amount that actually exists', + pay.charges[0]?.amount, + 4999, + ); + note( + '(d) the damage', + 'the caller records a successful 5999 charge; the vendor holds one 4999 charge and refused the amendment', + ); + } + + // ── (e) does a RE-SERIALISED body reach a 409 at all? ─────────────────────────────────── + // Only if the vendor compares bytes. Stripe compares parameters, so re-ordering is invisible to + // it — which is why C3's body-derived key fails by charging twice rather than by erroring. + { + const clock = manualClock(T0); + const canonical = new FakePayments({ clock, compare: 'canonical' }); + const strict = new FakePayments({ clock, compare: 'bytes' }); + const variants = bodyVariants(PAYMENT); + + for (const pay of [canonical, strict]) { + for (const v of variants) { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + clock, + }); + await call({ body: v.body }).safe(); + } + } + + checkSeq( + '(e) canonical vendor — status per request across the 3 body variants', + canonical.statuses(), + [200, 200, 200], + ); + checkCharges('(e) canonical vendor', canonical.chargeCount(), 1, 1); + checkSeq( + '(e) byte-comparing vendor — status per request', + strict.statuses(), + [200, 409, 200], + ); + checkCharges('(e) byte-comparing vendor', strict.chargeCount(), 1, 1); + note( + '(e) reading it', + 'variant 3 re-serialises identically to variant 1, so even the strict vendor replays it — only the re-ordering trips', + ); + } + + finish( + 'C5', + 'a 409 is NOT retried (1 request under `retry: { attempts: 4 }`) and reaches the caller as `status: 409` with `idempotency_key_in_use` on `error.body`, while the ledger shows the refusal was real (1 charge, the FIRST body’s amount) — but `verdict: { accept: [409], flag: "ok" }` SWALLOWS it, reporting `ok: true` and handing the error payload back as data', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c6-ttl-expiry.ts b/docs/scenarios/proofs/unconfirmed-write/c6-ttl-expiry.ts new file mode 100644 index 00000000..23c38f64 --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c6-ttl-expiry.ts @@ -0,0 +1,226 @@ +// C6 — TTL EXPIRY. The vendor prunes an idempotency record after a while (Stripe: ~24 hours) and +// then treats the key as brand new. A retry that arrives after the prune creates a SECOND CHARGE +// with a key that is doing everything right. +// +// This is the one failure a stable key cannot prevent. C2's `keyOf` makes the second run carry the +// SAME key; the TTL makes that irrelevant, because there is nothing left for the key to match. +// +// MEASURED: +// (a) The base case, with a perfect key. Run 1 charges and loses the response; the job is +// re-driven 25 hours later against a 24-hour TTL. Same key, and 2 CHARGES for 1 payment. +// Nothing in the config is wrong. +// (b) 23 hours later — inside the TTL — is 1 charge. The whole difference is the delay, which is +// a property of the queue and not of the client. +// (c) Is there anything client-side that NOTICES? No. The second charge is a clean `200` with a +// new charge id, indistinguishable from the first at every layer the caller can see: same +// status, same body shape, no replay header. Measured against the ledger, which the client +// does not have. +// (d) `timeout.total` cannot express it either — the budget is per CALL, not per key, and the +// second run is a different call with a fresh budget. Measured: a 1-hour total budget on both +// runs still produced 2 charges. +// (e) What a derived key CAN buy: because the key is a pure function of the payment, the second +// charge is DETECTABLE — a `GET /charges?ref=…` finds both and they are attributable to one +// intent. With the random default the two charges carry two unrelated uuids and the query is +// the only link. Measured both ways. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c6-ttl-expiry.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { ManualClock } from '../../../../packages/core/src/testing'; +import type { StitchConfig } from '../../../../packages/core/src/types'; +import { DEFAULT_KEY_TTL_MS, FakePayments } from './fake-payments'; +import type { Charge } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const HOUR = 60 * 60 * 1000; +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; + +/** One run of the job in a fresh process. Identical to C2's, so the only variable here is the delay. */ +async function driveJob( + pay: FakePayments, + clock: ManualClock, + extra: Partial, +): Promise { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + retry: { attempts: 2, backoff: { curve: 'fixed', base: '2s' } }, + timeout: { perAttempt: '5s' }, + clock, + ...extra, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 30_000, 1_000); + await pending; +} + +async function main(): Promise { + heading( + 'C6 — the queue re-drives the job 25 hours later, against a 24-hour key TTL', + ); + + // ── (a) past the TTL: a perfect key, and two charges ──────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2] }); + + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + await runOut(clock, 25 * HOUR, HOUR); // the delayed queue + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + + check('(a) DISTINCT keys — the key did its job', pay.distinctKeys(), 1); + check( + '(a) live idempotency records at the end', + pay.liveRecords(), + // The first record was pruned; the second run stored a fresh one under the same key. + 1, + ); + checkCharges('(a) 25h delay vs a 24h TTL', pay.chargeCount(), 1, 2); + checkSeq( + '(a) hours from t0 at which each charge was created', + pay.charges.map((c) => Math.round((c.createdAt - T0) / HOUR)), + [0, 25], + ); + note( + '(a) the key on both charges', + JSON.stringify([...new Set(pay.charges.map((c) => c.key))]), + ); + } + + // ── (b) the control: 23 hours, inside the TTL ─────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2] }); + + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + await runOut(clock, 23 * HOUR, HOUR); + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + + checkCharges('(b) 23h delay vs a 24h TTL', pay.chargeCount(), 1, 1); + check( + '(b) the re-drive was served from the record', + pay.replays().filter(Boolean).length > 0, + true, + ); + note( + '(b) the entire difference from (a)', + `${String(DEFAULT_KEY_TTL_MS / HOUR)}h of TTL against a delay the client does not control`, + ); + } + + // ── (c) does anything client-side notice? ─────────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2] }); + const seenReplayHeaders: string[] = []; + const hooks: StitchConfig['hooks'] = { + onResponse: ({ res }) => { + seenReplayHeaders.push( + res?.headers['idempotent-replayed'] ?? '(absent)', + ); + }, + }; + + await driveJob(pay, clock, { + idempotency: { keyOf: refKeyOf }, + hooks, + }); + await runOut(clock, 25 * HOUR, HOUR); + await driveJob(pay, clock, { + idempotency: { keyOf: refKeyOf }, + hooks, + }); + + checkSeq( + '(c) `Idempotent-Replayed` on every response the client actually received', + seenReplayHeaders, + // Only one response came back at all: the second run's first request, post-prune. It is + // a fresh 200 and carries no marker, because from the vendor's side it IS fresh. + ['(absent)'], + ); + checkCharges('(c)', pay.chargeCount(), 1, 2); + note( + '(c) the finding', + 'the duplicate arrives as a clean 200 with a new charge id — there is no client-side signal to key on', + ); + } + + // ── (d) `timeout.total` is per CALL and cannot bound this ─────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2] }); + + await driveJob(pay, clock, { + idempotency: { keyOf: refKeyOf }, + timeout: { perAttempt: '5s', total: '1h' }, + }); + await runOut(clock, 25 * HOUR, HOUR); + await driveJob(pay, clock, { + idempotency: { keyOf: refKeyOf }, + timeout: { perAttempt: '5s', total: '1h' }, + }); + + checkCharges('(d) `timeout: { total: "1h" }`', pay.chargeCount(), 1, 2); + note( + '(d) why', + 'the budget is stamped per call (engine.ts `totalBudget`); the re-drive is a new call with a new budget', + ); + } + + // ── (e) a derived key at least makes the second charge DETECTABLE ─────────────────────── + // `GET /charges?ref=…` is the recovery query, and it is only useful because the payment carries + // a business reference the charges can be grouped by — the same fact `refKeyOf` derives from. + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2] }); + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + await runOut(clock, 25 * HOUR, HOUR); + await driveJob(pay, clock, { idempotency: { keyOf: refKeyOf } }); + + const find = stitch({ + method: 'GET', + url: URL_CHARGES, + adapter: pay.adapter(), + pick: 'data', + clock, + }); + const found = (await find({ + query: { ref: PAYMENT.ref }, + })) as Charge[]; + + check('(e) charges the recovery query found', found.length, 2); + checkSeq( + '(e) the key each charge was created under', + found.map((c) => c.key), + ['chg-inv-1001', 'chg-inv-1001'], + ); + note( + '(e) with the RANDOM default instead', + 'the same query still finds 2 charges, but they carry 2 unrelated uuids — the only thing linking them is the ref, which is what `keyOf` was reading anyway', + ); + note( + '(e) the honest summary', + 'a derived key does not prevent the TTL duplicate; it makes reconciling it a lookup rather than an investigation', + ); + } + + finish( + 'C6', + 'a TTL prune turns a correct, stable key into a SECOND CHARGE — 2 charges for 1 payment at a 25h delay against a 24h TTL, 1 charge at 23h — and NOTHING client-side notices: the duplicate is a clean 200 with no replay marker, and `timeout.total` is per call so it cannot bound the gap', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c7-timeout-ambiguity.ts b/docs/scenarios/proofs/unconfirmed-write/c7-timeout-ambiguity.ts new file mode 100644 index 00000000..2d6edb15 --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c7-timeout-ambiguity.ts @@ -0,0 +1,315 @@ +// C7 — THE TIMEOUT ITSELF. Can the caller tell "the request never arrived" from "it was processed +// and the response was lost"? What do `attempts`, the thrown error, `.safe()` and the event stream +// actually carry? +// +// The honest answer is that no client library can distinguish these — the information is not on the +// client. So the real question is a narrower one: does StitchAPI carry ENOUGH for the caller to +// start a recovery, and does it carry it without lying? +// +// MEASURED, and it is worse than "no information": +// (a) The two cases are byte-identical at the caller. A request dropped before processing and a +// request that CREATED A CHARGE and lost its response produce the same `StitchError`: same +// `name`, same `status` (undefined), same `message`, same `attempts`, same `body` +// (undefined). The ledgers differ — 0 charges vs 1 — and the client cannot see the ledger. +// (b) The error CLASS is lost. The engine throws a `TimeoutError` (resilience.ts:17,233), and +// `errEvt` (engine.ts:346-355) reduces it to a message on an event, from which `rebuildError` +// constructs a plain `StitchError`. Measured: `constructor.name === 'StitchError'`, and the +// only thing left of the timeout is the string `timed out after 5000ms`. `TimeoutError` is +// not exported from the package either, so even a string-free check is unavailable. +// (c) `hooks.onError` DOES receive the live error — `constructor.name === 'TimeoutError'` there. +// That hook is the only place the class survives, and it cannot change the outcome. +// (d) A TRANSPORT failure is retried UNCONDITIONALLY: `retry.on` gates statuses only, and the +// throw path (engine.ts:675-703) has no status to match. Measured: `retry: { attempts: 3, +// on: [] }` still made 3 requests. You cannot configure "retry a 503 but not a timeout" — +// the only way to not retry into the unknown is `retry` off entirely. +// (e) The EVENT STREAM does not carry the idempotency key. `start` carries `name`/`method`/`url`/ +// `input` and no headers, so a caller who used the default random key can never learn which +// key their lost request carried — which makes the standard "query by key" recovery +// impossible by construction. `hooks.onRequest` is the only seam that sees it. +// (f) `attempts` is truthful and useful in one specific way: with a stable key, `attempts: N` +// still means AT MOST ONE charge. It bounds the damage; it does not report it. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c7-timeout-ambiguity.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + StitchConfig, + StitchEvent, +} from '../../../../packages/core/src/types'; +import { FakePayments } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const PAYMENT: Payment = { ref: 'inv-1001', amount: 4999, currency: 'usd' }; + +/** + * The caller's whole view of a failed call, reduced to the fields they can branch on. + * + * `status` and `body` are rendered as STRINGS rather than left `undefined`, so the side-by-side + * comparison in (a) is a real one — `JSON.stringify` drops an `undefined` field, and two objects + * that agree only because both dropped it would compare equal for the wrong reason. + */ +interface CallerView { + ok: boolean; + name: string; + ctor: string; + status: string; + attempts: number; + message: string; + body: string; +} + +async function runOnce( + pay: FakePayments, + clock: ReturnType, + extra: Partial = {}, +): Promise { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + timeout: { perAttempt: '5s' }, + clock, + ...extra, + }); + const pending = call({ body: PAYMENT }).safe(); + await runOut(clock, 120_000); + const r = await pending; + return { + ok: r.ok, + name: r.error?.name ?? '(none)', + ctor: r.error?.constructor.name ?? '(none)', + status: String(r.error?.status), + attempts: r.error?.attempts ?? 0, + message: r.error?.message ?? '', + body: String(r.error?.body), + }; +} + +async function main(): Promise { + heading( + 'C7 — "never arrived" vs "processed, response lost": can the caller tell? (the ledgers differ; the errors do not)', + ); + + // ── (a) the two cases, side by side ───────────────────────────────────────────────────── + let neverArrived: CallerView; + let processedThenLost: CallerView; + { + const clockA = manualClock(T0); + const payA = new FakePayments({ + clock: clockA, + dropBeforeProcessingOn: [1], + }); + neverArrived = await runOnce(payA, clockA); + + const clockB = manualClock(T0); + const payB = new FakePayments({ clock: clockB, loseResponseOn: [1] }); + processedThenLost = await runOnce(payB, clockB); + + checkCharges('(a) case 1 — never arrived', payA.chargeCount(), 1, 0); + checkCharges( + '(a) case 2 — PROCESSED, response lost', + payB.chargeCount(), + 1, + 1, + ); + check( + '(a) the two callers’ views, compared field by field', + JSON.stringify(neverArrived) === JSON.stringify(processedThenLost), + true, + ); + note('(a) that identical view', JSON.stringify(processedThenLost)); + note( + '(a) the ledgers behind them', + `${String(payA.chargeCount())} charge vs ${String(payB.chargeCount())} charge — the difference the caller cannot see`, + ); + } + + // ── (b) the error class is flattened away ─────────────────────────────────────────────── + { + check( + '(b) `error.constructor.name` on a timeout', + processedThenLost.ctor, + 'StitchError', + ); + check('(b) `error.name`', processedThenLost.name, 'StitchError'); + check( + '(b) `error.status` — nothing to branch on', + processedThenLost.status, + 'undefined', + ); + check( + '(b) the only thing that identifies a timeout', + processedThenLost.message, + 'timed out after 5000ms', + ); + note( + '(b) and a transport error’s message', + 'is the underlying error’s (`ECONNRESET`), so "is this a timeout?" is a string test either way', + ); + note( + '(b) `TimeoutError`', + 'is declared at resilience.ts:17 and exported from NO public entry point (only `RateLimitError` is, index.ts:86)', + ); + } + + // ── (c) `hooks.onError` is where the class survives ───────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1] }); + const seen: string[] = []; + await runOnce(pay, clock, { + hooks: { + onError: ({ error }) => { + const e = error as Error; + seen.push(`${e.constructor.name}/${e.name}`); + }, + }, + }); + + checkSeq('(c) what hooks.onError was handed', seen, [ + // The constructor is `TimeoutError`; `.name` is `Error` because the class never sets it. + 'TimeoutError/Error', + ]); + note( + '(c) the gap', + 'the class is legible in the hook and gone by the time the caller sees it — and the hook cannot change the outcome', + ); + } + + // ── (d) a transport failure is retried unconditionally ────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2, 3] }); + const view = await runOnce(pay, clock, { + retry: { + attempts: 3, + on: [], + backoff: { curve: 'fixed', base: '2s' }, + }, + }); + + check( + '(d) requests, with `retry: { attempts: 3, on: [] }`', + pay.calls.length, + 3, + ); + check('(d) attempts reported', view.attempts, 3); + note( + '(d) why', + 'the throw path (engine.ts:675-703) retries on `attempt < max` alone — there is no status to match `retry.on` against', + ); + checkCharges( + '(d) and the stable key is what keeps it at one', + pay.chargeCount(), + 1, + 1, + ); + } + + // ── (e) the event stream does not carry the key ───────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1] }); + const events: StitchEvent[] = []; + const sentKeys: string[] = []; + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, // the DEFAULT random key — the case that matters + retry: { attempts: 1 }, // present only to silence the construction nudge + timeout: { perAttempt: '5s' }, + hooks: { + onRequest: ({ req }) => { + sentKeys.push( + req?.headers['Idempotency-Key'] ?? '(absent)', + ); + }, + }, + clock, + }); + const consuming = (async () => { + for await (const ev of call.stream({ body: PAYMENT })) + events.push(ev); + })(); + await runOut(clock, 60_000); + await consuming; + + checkSeq( + '(e) event types on a lost write', + events.map((e) => e.type), + ['start', 'progress', 'error', 'done'], + ); + const start = events[0] as Extract; + checkSeq('(e) fields on the `start` event', Object.keys(start).sort(), [ + 'at', + 'input', + 'method', + 'name', + 'spanId', + 'traceId', + 'type', + 'url', + ]); + check( + '(e) does ANY event carry the idempotency key?', + JSON.stringify(events).includes(sentKeys[0] ?? ''), + false, + ); + check( + '(e) hooks.onRequest saw it', + (sentKeys[0] ?? '').length, + 36, // a uuid + ); + checkCharges('(e)', pay.chargeCount(), 1, 1); + note( + '(e) the consequence', + 'with the default key, the caller cannot name the key their lost request carried — so "ask the vendor about key X" is not available to them', + ); + } + + // ── (f) what `attempts` is actually good for ──────────────────────────────────────────── + { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, loseResponseOn: [1, 2, 3, 4] }); + const view = await runOnce(pay, clock, { + retry: { + attempts: 4, + backoff: { curve: 'fixed', base: '2s' }, + }, + }); + + check('(f) attempts', view.attempts, 4); + check('(f) requests that reached the server', pay.calls.length, 4); + checkCharges( + '(f) 4 attempts, all lost, under one stable key', + pay.chargeCount(), + 1, + 1, + ); + note( + '(f) the reading', + '`attempts: 4` bounds the damage at one charge — it does not tell you whether that charge exists', + ); + } + + finish( + 'C7', + 'the caller CANNOT distinguish the two cases — a dropped request and a charge whose response was lost produced field-for-field identical `StitchError`s (`StitchError` / status undefined / "timed out after 5000ms") over ledgers of 0 and 1 charges; the `TimeoutError` class is flattened away and unexported, a transport failure is retried even with `retry.on: []`, and NO event carries the idempotency key, so the default random key makes a query-by-key recovery impossible', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/c8-assembled.ts b/docs/scenarios/proofs/unconfirmed-write/c8-assembled.ts new file mode 100644 index 00000000..f5fadaaf --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/c8-assembled.ts @@ -0,0 +1,354 @@ +// C8 — assemble the safest answer available and run it against every workload in this scenario, +// reporting CHARGES CREATED against CHARGES INTENDED. Then run the same workloads with the config +// the docs' first example uses, as the control. +// +// The safe answer is three declarations and one function: +// +// 1. `idempotency: { keyOf: refKeyOf }` — a key derived from the business fact, so it survives a +// restart (C2) and does not move when the body is rebuilt (C3). +// 2. `retry` with the DEFAULT `retry.on` — 500 stays out of it, so a cached failure is not +// retried into the ground (C4 (a)). A transport failure is still retried unconditionally, and +// the stable key is what makes that safe (C7 (d)). +// 3. `settleCharge` — the user code, and it is QUERY-FIRST, not query-on-failure. The first draft +// of this file queried only after an ambiguous outcome and still double-charged on W6: a +// recovery that runs after the write cannot un-write it. Asking before writing is the only +// ordering that prevents the TTL duplicate. +// +// MEASURED, over six workloads and six intended payments: +// • assembled: 5 charges — every workload settled correctly, the sixth payment being a card the +// vendor declined. +// • control (`idempotency: true`, no recovery): 8 charges, with TWO duplicates (the restart, C2; +// the TTL prune, C6) and one payment that went through only because the key changed. +// +// The two results that surprised this file are both in the write-up below: W5, where the stable key +// makes a recorded failure STICKY and the random key quietly re-tries it into a success; and the +// concurrency race, where query-first is safe only because the key is stable. +// +// pnpm exec tsx docs/scenarios/proofs/unconfirmed-write/c8-assembled.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { ManualClock } from '../../../../packages/core/src/testing'; +import { FakePayments } from './fake-payments'; +import type { Charge, FakePaymentsOptions } from './fake-payments'; +import { + check, + checkCharges, + checkSeq, + finish, + heading, + note, +} from './harness'; +import { refKeyOf } from './keys'; +import type { Payment } from './keys'; +import { runOut } from './virtual-time'; + +const URL_CHARGES = 'https://api.pay.test/v1/charges'; +const T0 = Date.UTC(2026, 7, 5, 12, 0, 0); +const HOUR = 60 * 60 * 1000; + +// ── THE USER CODE UNDER TEST ──────────────────────────────────────────────────────────────── +// Everything between the two markers is what a caller writes on top of the library. +// +// >>> BEGIN USER CODE +type Settlement = + | { state: 'settled'; charge: Charge } + | { state: 'refused'; reason: unknown } + | { state: 'unknown' }; + +const charge = (pay: FakePayments, clock: ManualClock) => + stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: { keyOf: refKeyOf }, + retry: { attempts: 3, backoff: { curve: 'fixed', base: '2s' } }, + timeout: { perAttempt: '5s' }, + clock, + }); + +const findByRef = (pay: FakePayments, clock: ManualClock) => + stitch({ + method: 'GET', + url: URL_CHARGES, + pick: 'data', + adapter: pay.adapter(), + clock, + }); + +async function settleCharge( + pay: FakePayments, + clock: ManualClock, + payment: Payment, +): Promise { + const look = async (): Promise => + ( + (await findByRef( + pay, + clock, + )({ query: { ref: payment.ref } })) as Charge[] + )[0]; + const already = await look(); // ask BEFORE writing — the TTL duplicate is not undoable + if (already) return { state: 'settled', charge: already }; + const r = await charge(pay, clock)({ body: payment }).safe(); + if (r.ok) return { state: 'settled', charge: r.data as Charge }; + // A status means the vendor answered: the outcome is known, even when it is a refusal. + if (r.error.status !== undefined) + return { state: 'refused', reason: r.error.body }; + const landed = await look(); // no status = transport-level = genuinely unknown (C7) + return landed ? { state: 'settled', charge: landed } : { state: 'unknown' }; +} +// <<< END USER CODE + +/** The control: `idempotency: true` + `retry`, awaited, no recovery. The docs' first example. */ +async function chargeNaively( + pay: FakePayments, + clock: ManualClock, + payment: Payment, +): Promise { + const call = stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: true, + retry: { attempts: 3, backoff: { curve: 'fixed', base: '2s' } }, + timeout: { perAttempt: '5s' }, + clock, + }); + await call({ body: payment }).safe(); +} + +type Driver = ( + pay: FakePayments, + clock: ManualClock, + payment: Payment, +) => Promise; + +/** + * The six workloads, each one intended payment, each one a case an earlier claim measured. Running + * them against a single driver is what turns eight separate findings into one number. + */ +const WORKLOADS: { + label: string; + ref: string; + opts: Omit; + /** How many times the job runs — 2 models a crash and a queue re-drive. */ + runs: number; + /** Virtual ms between runs. Past the 24h TTL is the C6 case. */ + gapMs?: number; +}[] = [ + { label: 'W1 clean success', ref: 'inv-1', opts: {}, runs: 1 }, + { + label: 'W2 transport blip, retried in-call', + ref: 'inv-2', + opts: { dropBeforeProcessingOn: [1] }, + runs: 1, + }, + { + label: 'W3 processed then response lost, retried in-call', + ref: 'inv-3', + opts: { loseResponseOn: [1] }, + runs: 1, + }, + { + label: 'W4 every attempt lost, job re-driven after a crash', + ref: 'inv-4', + opts: { loseResponseOn: [1, 2, 3] }, + runs: 2, + }, + { + label: 'W5 card declined, the failure is recorded', + ref: 'inv-5', + opts: { failChargeOn: [1] }, + runs: 2, + }, + { + label: 'W6 re-driven 25h later, past the key TTL', + ref: 'inv-6', + opts: { loseResponseOn: [1, 2, 3] }, + runs: 2, + gapMs: 25 * HOUR, + }, +]; + +async function runWorkloads(drive: Driver): Promise<{ + created: number; + intended: number; + perWorkload: string[]; + writes: number; +}> { + let created = 0; + let intended = 0; + let writes = 0; + const perWorkload: string[] = []; + for (const w of WORKLOADS) { + const clock = manualClock(T0); + const pay = new FakePayments({ clock, ...w.opts }); + const payment: Payment = { ref: w.ref, amount: 4999, currency: 'usd' }; + intended++; + for (let run = 0; run < w.runs; run++) { + if (run > 0 && w.gapMs) await runOut(clock, w.gapMs, HOUR); + const pending = drive(pay, clock, payment); + await runOut(clock, 120_000); + await pending; + } + created += pay.chargeCount(); + writes += pay.calls.filter((c) => c.method === 'POST').length; + perWorkload.push(`${w.label}: ${String(pay.chargeCount())}`); + } + return { created, intended, perWorkload, writes }; +} + +async function main(): Promise { + heading( + 'C8 — six workloads, six intended payments: how many charges does the vendor end up holding?', + ); + + // ── the control ───────────────────────────────────────────────────────────────────────── + { + const { created, intended, perWorkload, writes } = await runWorkloads( + (pay, clock, payment) => chargeNaively(pay, clock, payment), + ); + checkSeq('control — charges per workload', perWorkload, [ + 'W1 clean success: 1', + 'W2 transport blip, retried in-call: 1', + 'W3 processed then response lost, retried in-call: 1', + // The re-drive mints a new key and charges again — C2. + 'W4 every attempt lost, job re-driven after a crash: 2', + // The re-drive's NEW key misses the recorded failure entirely and is processed fresh, + // so the declined payment quietly goes through on the second try. See the note below. + 'W5 card declined, the failure is recorded: 1', + // The prune makes the second run fresh whatever the key — C6. + 'W6 re-driven 25h later, past the key TTL: 2', + ]); + checkCharges( + 'control — `idempotency: true`, no recovery', + created, + intended, + 8, + ); + note('control — write requests issued', writes); + note( + 'control — W5 is the subtle one', + 'a random key never reaches the recorded failure, so the "cached 500" the vendor stored protects nobody — the second run just charged', + ); + } + + // ── the assembled answer ──────────────────────────────────────────────────────────────── + { + const { created, intended, perWorkload, writes } = await runWorkloads( + (pay, clock, payment) => settleCharge(pay, clock, payment), + ); + checkSeq('assembled — charges per workload', perWorkload, [ + 'W1 clean success: 1', + 'W2 transport blip, retried in-call: 1', + 'W3 processed then response lost, retried in-call: 1', + 'W4 every attempt lost, job re-driven after a crash: 1', + // The stable key DOES reach the recorded failure, so the decline stands. Correct, and + // it is the mirror image of the control's W5 — see the note. + 'W5 card declined, the failure is recorded: 0', + 'W6 re-driven 25h later, past the key TTL: 1', + ]); + checkCharges( + 'assembled — derived key + default `retry.on` + query-first', + created, + intended, + 5, + ); + note('assembled — write requests issued', writes); + note( + 'assembled — reading it', + 'five charges for six payments; the missing one is W5, whose card the vendor declined. Every workload settled correctly', + ); + note( + 'assembled — W5’s cost', + 'a stable key makes a RECORDED failure sticky for the whole TTL: a 500 that was transient is replayed as a decision. That is the price of the property that fixes W4', + ); + } + + // ── why query-FIRST, and why it is only safe with a stable key ────────────────────────── + // The capture calls out the race: between the query and the write, the original can land. It is + // real, and the derived key is what covers it — measured both ways, running the two jobs + // CONCURRENTLY so both queries return empty before either write goes out. + { + const race = async ( + keyed: boolean, + ): Promise<{ charges: number; keys: number }> => { + const clock = manualClock(T0); + const pay = new FakePayments({ clock }); + const payment: Payment = { + ref: 'inv-race', + amount: 4999, + currency: 'usd', + }; + const build = (): ReturnType => + stitch({ + method: 'POST', + url: URL_CHARGES, + adapter: pay.adapter(), + idempotency: keyed ? { keyOf: refKeyOf } : true, + retry: { attempts: 2 }, + clock, + }); + const find = findByRef(pay, clock); + const worker = async (): Promise => { + const found = (await find({ + query: { ref: payment.ref }, + })) as Charge[]; + if (found.length > 0) return; + await build()({ body: payment }).safe(); + }; + const both = Promise.all([worker(), worker()]); + await runOut(clock, 60_000); + await both; + return { charges: pay.chargeCount(), keys: pay.distinctKeys() }; + }; + + const keyed = await race(true); + const unkeyed = await race(false); + checkCharges( + '(race) two concurrent runs, DERIVED key', + keyed.charges, + 1, + 1, + ); + check('(race) distinct keys, derived', keyed.keys, 1); + checkCharges( + '(race) two concurrent runs, DEFAULT key', + unkeyed.charges, + 1, + 2, + ); + check('(race) distinct keys, default', unkeyed.keys, 2); + note( + '(race) the finding', + 'query-first does not close the race — the KEY does. Query-first handles the TTL prune; the key handles concurrency. Both are needed and neither substitutes', + ); + } + + // ── the seam and the line count ───────────────────────────────────────────────────────── + { + note( + 'seam', + '`idempotency.keyOf` for the key; `.safe()` + a second stitch for the recovery. A hook cannot do the recovery — a hook cannot change the outcome', + ); + note( + 'user code', + 'the `settleCharge` block between the BEGIN/END markers — 12 statements, of which 4 are the recovery and 2 the query-first guard', + ); + note( + 'what needed NO code at all', + 'per-attempt key reuse (C1), the retry that recovers a lost response (C1 (b)), not retrying a cached 500 (C4 (a)), not retrying a 409 (C5 (a))', + ); + note( + 'what the recovery costs', + 'one extra GET per payment, and it only works because the vendor stores a business ref the client can search on — an idempotency key alone is not queryable at most vendors', + ); + } + + finish( + 'C8', + 'the assembled answer settled all six workloads correctly — 5 charges for 6 intended payments, the sixth a declined card — where the control (`idempotency: true`, no recovery) produced 8 charges with two duplicates; the derived key fixes the restart and the concurrency race in configuration alone, and only a QUERY-FIRST recovery prevents the TTL duplicate, because a recovery that runs after the write cannot un-write it', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unconfirmed-write/fake-payments.ts b/docs/scenarios/proofs/unconfirmed-write/fake-payments.ts new file mode 100644 index 00000000..79d57077 --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/fake-payments.ts @@ -0,0 +1,402 @@ +// The Stripe-ish payment server this scenario measures against: an `Adapter` that implements real +// idempotency-key semantics, and — crucially — a LEDGER OF CHARGES that is the ground truth every +// claim asserts on. +// +// The whole scenario is one question: at the end of a workload, how many charges does the vendor +// hold, and how many did the caller mean to create? Everything else (how many requests reached the +// wire, what the caller was told, how many keys were minted) only matters as an explanation of that +// number. So `charges` is a plain array, `chargeCount()` reads it, and no claim is allowed to +// conclude anything from the client's own view of events. +// +// The five behaviours modelled, each because a real vendor has it and each because it breaks +// something a client would otherwise get away with: +// +// 1. FIRST-WRITE-WINS REPLAY. The status AND body of the first request for a key are stored and +// replayed on reuse — INCLUDING A STORED FAILURE. Stripe replays the recorded outcome +// "regardless of whether it succeeds or fails", so a retry after a cached 500 gets that same +// 500 forever. A replay carries `Idempotent-Replayed: true`, the way Stripe's does. +// 2. PARAMETER COMPARISON. The same key with different parameters is a `409 idempotency_error`. +// The comparison is on the CANONICAL parameters by default (sorted keys — what Stripe actually +// compares), with a `compare: 'bytes'` mode for the vendors that diff the raw serialisation. +// The difference is load-bearing: under `canonical`, a re-serialised body is harmless to the +// SERVER and still fatal to a body-derived CLIENT key. +// 3. A TTL. Records are pruned `keyTtlMs` after they are stored, and a pruned key is a fresh key — +// it creates a SECOND CHARGE. Stripe prunes after ~24 hours; a job queue does not care. +// 4. LOST RESPONSES. A request can be PROCESSED — charge created, record stored — and then never +// answered. This is the case the whole scenario exists for, and it is the only one where the +// client's view and the ledger genuinely disagree. +// 5. NO KEY, NO DEDUPE. A request without an `Idempotency-Key` always creates a new charge. That +// is what makes a lost key measurable rather than invisible. +// +// The recovery endpoint (`GET /charges?ref=…`) is modelled too, because "query, then decide" is the +// standard fallback and a claim that recommends it has to run it. +import type { + Adapter, + AdapterRequest, + AdapterResponse, + Clock, +} from '../../../../packages/core/src/types'; + +/** Stripe prunes an idempotency record after roughly this long. The default here, for the same reason. */ +export const DEFAULT_KEY_TTL_MS = 24 * 60 * 60 * 1000; + +/** A charge that actually exists at the vendor. The ledger of these IS the ground truth. */ +export interface Charge { + id: string; + /** The caller's business reference (invoice / order id) — what a recovery query searches on. */ + ref: string; + amount: number; + currency: string; + /** The idempotency key the creating request carried, or `undefined` when it carried none. */ + key: string | undefined; + createdAt: number; +} + +/** One stored idempotency record: the outcome of the FIRST request for a key. */ +interface Record_ { + key: string; + status: number; + body: unknown; + /** The parameters the first request carried, rendered for comparison. */ + fingerprint: string; + storedAt: number; +} + +/** One request as the server saw it. The per-request spine several claims assert on. */ +export interface PayCall { + /** 1-based arrival order across the whole server lifetime — a restart does NOT reset it. */ + n: number; + /** + * 1-based order among CHARGE ATTEMPTS only (`POST`s), which is what the failure knobs count. + * `undefined` on a recovery `GET`. Keeping this separate from `n` is what lets a workload be + * described once and run against drivers that issue different numbers of recovery queries. + */ + writeN: number | undefined; + method: string; + path: string; + /** The `Idempotency-Key` header as it arrived, or `undefined` when absent. */ + key: string | undefined; + status: number; + /** True when this response came from a stored record rather than fresh processing. */ + replayed: boolean; + /** True when this request created a row in the charge ledger. */ + createdCharge: boolean; + /** True when the server processed the request and then never answered (the response was lost). */ + lost: boolean; + at: number; +} + +export interface FakePaymentsOptions { + /** The clock `createdAt`/`storedAt` and TTL pruning read. Inject the stitch's clock. */ + clock: Clock; + /** How long a stored record survives. Default {@link DEFAULT_KEY_TTL_MS}. */ + keyTtlMs?: number; + /** + * How the server decides two requests carry "the same parameters". + * - `'canonical'` (default) — sorted-key JSON, so key ORDER is not a difference. This is what + * Stripe does, and it is the setting under which a re-serialised body is the CLIENT's problem. + * - `'bytes'` — the raw serialisation, so any re-ordering is a mismatch. The stricter vendors. + */ + compare?: 'canonical' | 'bytes'; + // The three failure knobs below count CHARGE ATTEMPTS (`POST`s), not HTTP requests — see + // `PayCall.writeN`. A recovery `GET` therefore does not shift them, so the same workload + // description drives a client that queries and one that does not. (It cost this file a wrong + // result once: with the knobs on the global ordinal, adding a query-first `GET` moved "the first + // charge attempt fails" onto the query and silently let the declined card through.) + /** + * 1-based CHARGE-ATTEMPT ordinals that are fully PROCESSED (charge created, record stored) and + * then never answered. THE case: the money moved and the caller will never hear about it. + */ + loseResponseOn?: readonly number[]; + /** + * 1-based CHARGE-ATTEMPT ordinals that never reach processing at all — the connection dies on + * the way out. Indistinguishable from `loseResponseOn` at the client, and the ledger proves it. + */ + dropBeforeProcessingOn?: readonly number[]; + /** + * 1-based CHARGE-ATTEMPT ordinals whose charge fails. The failure is stored and replayed like + * any other outcome — that is the cached-failure case. No charge row is created. + */ + failChargeOn?: readonly number[]; + /** Status for a failed charge. Default 500 (Stripe's own "we recorded a failure" case). */ + failStatus?: number; +} + +/** Sorted-key JSON — the canonical rendering of a parameter set (recursively, arrays kept in order). */ +export function canonicalJson(value: unknown): string { + if (value === null || typeof value !== 'object') + return JSON.stringify(value) ?? 'null'; + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`; +} + +/** The vendor's payment API, as a plain `Adapter`. `charges` is the ground truth; `calls` is the spine. */ +export class FakePayments { + /** THE GROUND TRUTH. Every claim's verdict is a statement about the length of this array. */ + readonly charges: Charge[] = []; + readonly calls: PayCall[] = []; + private readonly records = new Map(); + private readonly clock: Clock; + private readonly keyTtlMs: number; + private readonly compare: 'canonical' | 'bytes'; + private readonly loseResponseOn: Set; + private readonly dropBeforeProcessingOn: Set; + private readonly failChargeOn: Set; + private readonly failStatus: number; + private nextCharge = 1; + /** Charge-attempt counter — what the failure knobs index on (see `PayCall.writeN`). */ + private writes = 0; + + constructor(opts: FakePaymentsOptions) { + this.clock = opts.clock; + this.keyTtlMs = opts.keyTtlMs ?? DEFAULT_KEY_TTL_MS; + this.compare = opts.compare ?? 'canonical'; + this.loseResponseOn = new Set(opts.loseResponseOn ?? []); + this.dropBeforeProcessingOn = new Set( + opts.dropBeforeProcessingOn ?? [], + ); + this.failChargeOn = new Set(opts.failChargeOn ?? []); + this.failStatus = opts.failStatus ?? 500; + } + + /** THE measurement. How many charges the vendor holds, whatever the client believes. */ + chargeCount(): number { + return this.charges.length; + } + + /** Every idempotency key that reached the server, in arrival order. Duplicates are the point. */ + keys(): (string | undefined)[] { + return this.calls.map((c) => c.key); + } + + /** + * How many DISTINCT keys were seen — 1 across a whole workload is what restart-safety looks + * like. Requests that carried NO key are excluded, so a recovery `GET` mixed into the ledger + * does not read as an extra key; use {@link FakePayments.keys} to see absences. + */ + distinctKeys(): number { + return new Set( + this.calls.filter((c) => c.key !== undefined).map((c) => c.key), + ).size; + } + + /** Status per request, in arrival order. */ + statuses(): number[] { + return this.calls.map((c) => c.status); + } + + /** Which requests were answered from a stored record rather than processed. */ + replays(): boolean[] { + return this.calls.map((c) => c.replayed); + } + + /** How many stored records are live right now (after pruning). Used by the TTL claim. */ + liveRecords(): number { + this.prune(); + return this.records.size; + } + + /** Drop every record past its TTL. Runs on every request, the way a real store's expiry would. */ + private prune(): void { + const now = this.clock.now(); + for (const [k, r] of this.records) + if (now - r.storedAt >= this.keyTtlMs) this.records.delete(k); + } + + private fingerprint(body: unknown): string { + return this.compare === 'canonical' + ? canonicalJson(body) + : JSON.stringify(body); + } + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const url = new URL(req.url); + if (req.method === 'GET') return this.handleQuery(url); + return this.handleCharge(req, url); + }; + } + + /** `GET /charges?ref=…` — the recovery query. Authoritative: it reads the same ledger. */ + private handleQuery(url: URL): AdapterResponse { + const ref = url.searchParams.get('ref'); + const found = this.charges.filter((c) => c.ref === ref); + this.calls.push({ + n: this.calls.length + 1, + method: 'GET', + path: url.pathname, + key: undefined, + writeN: undefined, + status: 200, + replayed: false, + createdCharge: false, + lost: false, + at: this.clock.now(), + }); + return { status: 200, headers: {}, body: { data: found } }; + } + + private async handleCharge( + req: AdapterRequest, + url: URL, + ): Promise { + this.prune(); + const n = this.calls.length + 1; + const writeN = ++this.writes; + const at = this.clock.now(); + // Header lookup is case-insensitive: the engine writes `Idempotency-Key`, a caller may pass + // `idempotency-key` on the call's `headers`, and a real server would not care. + const key = Object.entries(req.headers).find( + ([h]) => h.toLowerCase() === 'idempotency-key', + )?.[1]; + // Losing the response is a property of the NETWORK, not of what the server did — so it is + // applied at every exit below, including a replay. (A replay whose response is lost is how + // a retry loop burns its whole budget against a charge that already exists.) + const lost = this.loseResponseOn.has(writeN); + const answer = async ( + status: number, + headers: Record, + body: unknown, + replayed: boolean, + createdCharge: boolean, + ): Promise => { + this.calls.push({ + n, + writeN, + method: req.method, + path: url.pathname, + key, + at, + status, + replayed, + createdCharge, + lost, + }); + if (!lost) return { status, headers, body }; + await this.clock.sleep(Number.MAX_SAFE_INTEGER); + throw new Error('unreachable: the response never arrives'); + }; + + // (i) the connection dies before the server does anything. No charge, no record — and the + // client cannot tell this apart from (iv) below, which is the entire scenario. + if (this.dropBeforeProcessingOn.has(writeN)) { + this.calls.push({ + n, + writeN, + method: req.method, + path: url.pathname, + key, + at, + status: 0, + replayed: false, + createdCharge: false, + lost: true, + }); + await this.clock.sleep(Number.MAX_SAFE_INTEGER); + throw new Error('unreachable: the connection never answers'); + } + + const params = req.body; + const fingerprint = this.fingerprint(params); + + // (ii) a live record for this key: replay it, or reject a parameter mismatch. + if (key !== undefined) { + const existing = this.records.get(key); + if (existing) { + if (existing.fingerprint !== fingerprint) { + // Stripe's `idempotency_error`. Note what does NOT happen: no charge is created + // and the stored record is untouched. The FIRST body is still the one that ran. + return answer( + 409, + {}, + { + error: { + type: 'idempotency_error', + code: 'idempotency_key_in_use', + message: + 'Keys for idempotent requests can only be used with the same parameters they were first used with.', + }, + }, + false, + false, + ); + } + return answer( + existing.status, + // The marker Stripe sets on a replay. Whether a client can ACT on it is C4. + { 'idempotent-replayed': 'true' }, + existing.body, + true, + false, + ); + } + } + + // (iii) fresh processing. A failure is stored exactly like a success — that is the whole + // point of a cached failure — but creates no charge. + const failing = this.failChargeOn.has(writeN); + const p = (params ?? {}) as { + ref?: string; + amount?: number; + currency?: string; + }; + let status: number; + let body: unknown; + let createdCharge = false; + if (failing) { + status = this.failStatus; + body = { + error: { + type: 'card_error', + code: 'card_declined', + message: 'Your card was declined.', + }, + }; + } else { + const charge: Charge = { + id: `ch_${String(this.nextCharge++).padStart(4, '0')}`, + ref: p.ref ?? '(none)', + amount: p.amount ?? 0, + currency: p.currency ?? 'usd', + key, + createdAt: at, + }; + this.charges.push(charge); + createdCharge = true; + status = 200; + body = charge; + } + if (key !== undefined) + this.records.set(key, { + key, + status, + body, + fingerprint, + storedAt: at, + }); + + // (iv) THE CASE. Everything above already happened — the money moved, the record is stored + // — and `answer` may now swallow the response. The client will time out and know + // nothing, and `charges` will say otherwise. + return answer(status, {}, body, false, createdCharge); + } +} + +/** + * Run one call and reduce it to a short outcome token — `'ok'`, or `''` / `''` for + * a failure. `PromiseLike`, not `Promise`: a stitch call returns a lazy `StitchResult` thenable. + */ +export async function outcomeOf( + call: () => PromiseLike, +): Promise { + try { + await call(); + return 'ok'; + } catch (e) { + const err = e as Error & { status?: number }; + return err.status === undefined ? err.message : String(err.status); + } +} diff --git a/docs/scenarios/proofs/unconfirmed-write/harness.ts b/docs/scenarios/proofs/unconfirmed-write/harness.ts new file mode 100644 index 00000000..eff75ff1 --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/harness.ts @@ -0,0 +1,117 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is A COUNT OF CHARGES. Every other number here — how many requests +// reached the wire, how many distinct idempotency keys they carried, what the caller was told — is +// only interesting because it explains that count. So `checkCharges` is the assertion that matters +// and it prints BOTH sides of the comparison, always: how many charges the vendor ended up holding +// against how many the caller meant to create. `created 2, intended 1` is the finding, and it has to +// be readable out of context. +// +// Everything else follows `expiring-signatures/harness.ts`: `check` for an exact value, `checkSeq` +// for a measured sequence, `note` for a reported-but-not-asserted number, `finish` for the verdict. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-attempt key spine + * (`["chg-inv-1001","chg-inv-1001"]`) and the per-request status spine (`[500,500,500]`) ARE the + * evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** + * THE assertion of this scenario: how many charges the vendor actually holds, against how many the + * caller intended to create. + * + * It takes both numbers because the pair is the finding — a bare "2" means nothing, and + * `created 2, intended 1 ← DOUBLE CHARGE` is the whole result of C2. `expected` is what the claim + * predicts the library will do, which is NOT always `intended`: several claims here PASS by + * measuring a double charge that the configuration made inevitable. + */ +export function checkCharges( + label: string, + created: number, + intended: number, + expected: number, +): void { + checks++; + const ok = created === expected; + if (!ok) failures++; + const verdict = + created > intended + ? ` <- ${String(created - intended)} MORE THAN INTENDED` + : created < intended + ? ` <- ${String(intended - created)} FEWER THAN INTENDED` + : ' <- matches intent'; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: CHARGES created ${String(created)}, intended ${String(intended)}${verdict}${ + ok ? '' : ` (expected created ${String(expected)})` + }`, + ); +} + +/** Assert a measured number is at most `bound`. */ +export function checkAtMost( + label: string, + actual: number, + bound: number, +): void { + checks++; + const ok = actual <= bound; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? ` (<= ${String(bound)})` : ` (expected <= ${String(bound)})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C2'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Half the claims here PASS by measuring the library doing the right thing and half by measuring it + * doing something that costs money, so the verdict statement always carries the direction. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/unconfirmed-write/keys.ts b/docs/scenarios/proofs/unconfirmed-write/keys.ts new file mode 100644 index 00000000..43dde6dd --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/keys.ts @@ -0,0 +1,75 @@ +// The key strategies under test, and the jobs that drive them. +// +// `idempotency.keyOf` is `(input: StitchInput) => string` — SYNCHRONOUS, so `crypto.subtle` (async) +// is not available inside it. Node's `createHash` is, and so is any pure function of the input; that +// constraint is worth stating because "hash the body" is the obvious implementation and half the +// obvious implementations of it are async. +// +// Three strategies, and the difference between them is the difference between one charge and two: +// +// • `naiveKeyOf` — `JSON.stringify(input.body)`. Restart-stable, and NOT stable against +// re-serialisation: `{a,b}` and `{b,a}` are the same parameters and different +// keys. The failure it produces is a SECOND CHARGE, not a 409 — the server never +// gets to compare parameters, because it never sees the same key twice. +// • `refKeyOf` — derived from the business fact alone (the invoice ref). Restart-stable and +// immune to how the body was built, because it does not read the body's shape. +// • `canonicalKeyOf` — a sha256 over the sorted-key rendering of the whole parameter set. Stable +// against ordering and absent-vs-undefined, and still sensitive to a genuinely +// different amount, which is what you want. +import type { StitchInput } from '../../../../packages/core/src/types'; +import { canonicalJson } from './fake-payments'; + +import { createHash } from 'node:crypto'; + +/** The payment a caller intends to make. One of these is one intended charge, forever. */ +export interface Payment { + ref: string; + amount: number; + currency: string; +} + +/** `JSON.stringify` of the body, verbatim. The obvious implementation, and the unstable one. */ +export const naiveKeyOf = (input: StitchInput): string => + `chg-${JSON.stringify(input.body)}`; + +/** The business fact and nothing else. Immune to how the body was built. */ +export const refKeyOf = (input: StitchInput): string => + `chg-${String((input.body as { ref?: string } | undefined)?.ref)}`; + +/** sha256 over the CANONICAL parameter set — order-independent, value-sensitive. */ +export const canonicalKeyOf = (input: StitchInput): string => + `chg-${createHash('sha256').update(canonicalJson(input.body)).digest('hex').slice(0, 16)}`; + +/** + * The same logical payment, expressed three ways a real client would produce it between one process + * and the next. Nothing here changes the PARAMETERS — the amount, currency and ref are identical in + * all three — only how the object was assembled. + * + * Why these three: a body rebuilt from a database row comes out in column order, not literal order; + * a number that round-tripped through a JSON column or a decimal library comes back `1.0`; an + * optional field that was absent in one process is explicitly `undefined` in another. + */ +export function bodyVariants(p: Payment): { + label: string; + body: Record; +}[] { + return [ + { + label: 'as written', + body: { ref: p.ref, amount: p.amount, currency: p.currency }, + }, + { + label: 'keys re-ordered', + body: { currency: p.currency, amount: p.amount, ref: p.ref }, + }, + { + label: 'optional field present-but-undefined, amount as 1.0', + body: { + ref: p.ref, + amount: Number(p.amount.toFixed(1)), + currency: p.currency, + description: undefined, + }, + }, + ]; +} diff --git a/docs/scenarios/proofs/unconfirmed-write/virtual-time.ts b/docs/scenarios/proofs/unconfirmed-write/virtual-time.ts new file mode 100644 index 00000000..3ef022cf --- /dev/null +++ b/docs/scenarios/proofs/unconfirmed-write/virtual-time.ts @@ -0,0 +1,53 @@ +// Driving a `manualClock` when the code under test does a little real async work. +// +// `manualClock.advance(ms)` fires every timer due before the target and drains the MICROTASK queue +// between fires, which is enough for code whose only asynchrony is the clock. Everything in this +// directory is that code — the fake payment server is pure in-memory logic with no crypto and no +// I/O, so `advance` alone is faithful here in a way it was not for `expiring-signatures` (whose +// SigV4 signing settled on the macrotask queue several turns deep). +// +// `runOut` exists anyway, for one reason: several claims here run a call to completion while the +// engine is sleeping on a RETRY BACKOFF, and the sleep is armed only after the failing attempt +// settles. Advancing in slices, with a macrotask turn between them, means a backoff armed during +// slice N is fired by slice N+1 rather than being missed — so a claim can say "advance past +// everything" instead of hand-computing the schedule. +import type { ManualClock } from '../../../../packages/core/src/testing'; + +// Yield one full turn of the event loop. `setImmediate` fires in the check phase and costs +// microseconds; `setTimeout(…, 0)` is clamped to ~1ms by Node and would make a deep drain slow. +const macrotask: () => Promise = + typeof setImmediate === 'function' + ? () => + new Promise((resolve) => { + setImmediate(resolve); + }) + : () => + new Promise((resolve) => { + setTimeout(resolve, 0); + }); + +/** Yield `turns` times, so anything sitting on the macrotask queue settles. */ +export async function drain(turns = 5): Promise { + for (let i = 0; i < turns; i++) await macrotask(); +} + +/** + * Advance `clock` by `totalMs` in `stepMs` slices, draining real macrotasks before the first slice + * and after every one. + * + * `stepMs` only has to be smaller than the smallest interval being measured; it does not have to + * divide anything evenly. + */ +export async function runOut( + clock: ManualClock, + totalMs: number, + stepMs = 1_000, +): Promise { + await drain(); + for (let left = totalMs; left > 0;) { + const slice = Math.min(stepMs, left); + await clock.advance(slice); + await drain(); + left -= slice; + } +} diff --git a/docs/scenarios/proofs/unstable-pagination/README.md b/docs/scenarios/proofs/unstable-pagination/README.md new file mode 100644 index 00000000..4eac6f77 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/README.md @@ -0,0 +1,221 @@ +# Proofs — the page that moved while you were reading it + +Runnable evidence for the claims in [`../../unstable-pagination.md`](../../unstable-pagination.md). + +**The scenario's answer is a list of row ids, and the first two of them are the wrong way round.** +The capture says an insert before your cursor loses a row and a delete before your cursor repeats +one. Measured against a fake server that knows the ground truth: an insert **duplicates** `r04`, a +delete **skips** `r05`. That is not a quibble — the two failures have completely different +detectability. A duplicate is visible in the data you were handed. A skip is not, and the delete +that caused it takes exactly one row off the declared `total` at the same instant, so +`collected.length === total === 9` with a row missing and every reconciliation check reading clean. + +Every script is standalone and offline. The numbers are **ids**, not durations: each claim prints +what the server served page by page, what the caller ended up with, and the `skipped` / `duplicated` +sets computed against the server's own record of which rows existed at the start and at the end. +Time is not load-bearing in this scenario (pagination is sequential and nothing sleeps), so only C8 +injects a `manualClock()` — where it measures per-page retry across a `500`. + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c1-insert-before-cursor.ts + +# all of them +for f in docs/scenarios/proofs/unstable-pagination/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/unstable-pagination/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ----------------------------- | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `c1-insert-before-cursor.ts` | does an insert behind the cursor LOSE a row? | **No — it DUPLICATES one.** skipped `[]`, duplicated `["r04"]`, run ok, 0 findings | +| `c2-delete-before-cursor.ts` | does a delete behind the cursor REPEAT a row? | **No — it SKIPS one.** skipped `["r05"]`, and `length === total === 9`, so nothing can see it | +| `c3-tie-order.ts` | ties, non-unique sort key, ZERO writes | **skipped `["r05"]` + duplicated `["r03"]` on a frozen collection**, and they cancel in `total` | +| `c4-keyset.ts` | is keyset expressible through `next`? is it correct? | **Yes, in 4 lines — clean on every workload.** And a non-total server ORDER BY breaks it anyway | +| `c5-detection-seams.ts` | can a duplicate be seen or removed inside the library? | **5 seams; the 2 obvious ones truncate the run.** A deduping `items` lost 6 rows, successfully | +| `c6-total-reconciliation.ts` | is the last page's `total` reachable? does it detect? | **Reachable from 3 places. Fires 0/4 raw, and never on C2** — the delete moves the total too | +| `c7-drift-meets-the-edges.ts` | drift vs the zero-item break, the wrap, and the page cap | **Drift makes an empty page mid-run → skipped `["r10","r11","r12"]`, ok.** And `pages: 50` | +| `c8-assembled.ts` | the best available answer, priced | **0 false negatives / 3 false alarms over 8 workloads — and 84 lines vs 74 hand-rolled** | + +## Files + +- `fake-collection.ts` — the live table. One collection served three ways so the only variable + between claims is the pagination contract: `?offset=&limit=`, `?after_ts=&after_id=&limit=` + (keyset), and both answering `{ rows, total, limit, offset? }`. Writes land BETWEEN page fetches + via `afterRequest(n, mutation)` — request-indexed, not timed, so every run is deterministic. The + `ties` variant sorts by `created_at` alone and rotates each tie group one position per query, + which is a legal answer to an `ORDER BY` that is not a total order; `brokenSeek` extends that to + the seek endpoint. `audit(collected)` is the ground truth: **STABLE** = ids present at the start + AND at the end, and a correct paginator returns every stable id exactly once. Rows created or + destroyed mid-run are **transient** and never counted as damage. +- `offset-loop.ts` — the offset/limit `paginate` block every damage claim runs. `items` pulls the + rows, `next` advances `offset` by `limit` and stops when it passes the declared `total`. There is + nothing wrong with it, which is the point. +- `keyset-loop.ts` — the seek block. Four lines between the ``/`` markers. +- `sync-collection.ts` — the assembled answer C8 runs: keyset where available, offset with the + damage detected where not, rows and verdict returned as ONE value. The counted region is what + C8(d) measures. +- `hand-rolled.ts` — the same feature set with no library at all, against the same fake server. The + baseline C8 prices against. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C1 and C2 refute the capture, in opposite directions from each other and both away from the + written text.** Offset counts from the start of the result set, so an insert BEHIND the cursor + pushes rows to higher indices and the fixed next offset lands on a row already read + (`r01,r02,r03,r04 | r04,r05,r06,r07` — duplicated `["r04"]`); a delete behind the cursor pulls + rows to lower indices and the row that was about to be page 2's first slides into territory the + client already passed (`r01,r02,r03,r04 | r06,r07,r08,r09` — skipped `["r05"]`). Two inserts + duplicate two rows; two deletes skip two. An insert or delete AHEAD of the cursor does nothing. + **An offset insert can never cause a skip.** +- **The delete case is the one that matters, and it is undetectable.** Measured: 9 rows collected, + 9 distinct, `total` 9 — because the delete removed one row from `total` at the same instant it + removed one from the result. `length === total`, no duplicate to find, `error: null`, + `findings: []`, `done.ok: true`. The only per-page signal the library emits is a running item + count (`page 1 (+4, total 4)`, engine.ts:976-982), which reads identically on a clean run. +- **C3 is the case people don't believe, and it defeats both cheap detections at once.** Ten rows, + zero writes, nothing created or destroyed, a non-unique `created_at` whose ties come back rotated: + skipped `["r05"]`, duplicated `["r03"]`. They cancel, so `length === total === 10`. Only comparing + the DEDUPED count (9) against `total` (10) fires. A tie group crossing two page boundaries loses + `["r05","r10"]` and repeats `["r03","r04"]` and still balances at 16 === 16. The same rows under a + total order are clean — **the fix is the server's `ORDER BY`, not anything the client sends.** +- **C4 confirms the capture: keyset is the thing the library does well.** `next` receives the + previous page's RAW body (engine.ts:985), so a composite `(created_at, id)` cursor is four lines, + and it measured skipped `[]` / duplicated `[]` on every workload that broke offset — C1's insert, + C2's delete, C3's ties. The loop terminates on the zero-item break at engine.ts:984, which is the + one thing that break is right for. +- **And keyset has a caveat the capture does not raise.** The same four lines against a vendor that + ACCEPTS `(after_ts, after_id)` and still orders by `created_at` alone lost `["r03"]` on one + collection and duplicated `["r13"]` on another, with no writes in either. A composite cursor only + works if the server's sort is the composite key; the client half is four lines and it is not the + half that decides. +- **C5: five seams reach the duplicate, and the two a caller reaches for first are booby-trapped.** + `items` and `transform` both run per page and both carry closure state, so both dedupe — until a + page is ENTIRELY duplicates. Four rows inserted behind the cursor made page 2 a verbatim repeat of + page 1; the deduper returned zero items, `paginated` broke at engine.ts:984 **before** calling + `next`, and the run ended `ok` with 4 rows, skipping `["r05".."r10"]` against a declared total of 14. **Without the dedupe the same run returns all ten rows.** The fix for duplicates is a + mechanism for losing rows. +- **`output` is the safe seam, because it runs after the loop.** `validateOutput` executes once over + the aggregated array (engine.ts:993) and its return value REPLACES the result (engine.ts:1005), so + a hand-rolled `Validator` is a genuine post-processing hook: deduping returned 10 rows with the + page count unchanged, and rejecting produced a failed call. Nothing it does can shorten the run. +- **A rejecting `output` loses the detail on `.safe()` and keeps it on `.report()`.** The issue text + (`pagination drift: duplicate ids r04`) does not reach `error.message`, which is the generic + `contract violation (drift)`; `.report().findings` carries it, and `.report().raw` still holds the + 11 aggregated rows that `.safe()` discards. +- **`drift()` fires on the right event and describes the wrong thing.** Wrapping the deduping + validator produced **four** findings — `warn|coerced|[].id`, `[].created_at`, `[].name` and + `info|undeclared|[]` — because removing one element RE-INDEXES the array and the positional diff + reads every later row as a changed field. Nothing in the vocabulary says "duplicate". +- **A custom `Surface.interpret` is the only seam that stops at the drifted page.** It runs per page + inside the attempt loop (engine.ts:775) and a `{ ok: false, message }` on a 200 is returned rather + than thrown (engine.ts:824-831), so `paginated` turns it into an error + `done(false)` + (engine.ts:960-964) with the message intact: `pagination drift: page repeated r04`, after 2 pages. + The cost is total — a failed paginated run emits **no `result` event**, so every row already + collected is discarded. +- **`hooks.onResponse` is the most complete view in the library and it is read-only.** It fired on + all three pages with the raw body, and the result was unchanged by it. +- **C6 corrects scenario 3's "`next` never sees the terminal page" into something more useful.** + That is true of the ZERO-ITEM break; it is false of a loop that ends because `next` returned + `undefined`, which is how `offset < total` naturally terminates — `next` observed `[10,10,10]` + over 3 pages, final total included. Switch to the `rows.length < limit` spelling over a collection + that is an exact multiple of `limit` and a fourth, EMPTY page is fetched: `next` saw `[12,12,12]` + and the terminal page's `total` of 10 was unreachable from it. `transform` and `hooks.onResponse` + saw `[12,12,12,10]` in both cases. **Which of the two spellings you wrote decides whether the + final total exists.** +- **Reconciling against `total` is reachable, actionable, and does not detect the skip.** Capturing + the total in `transform` and deciding in `output` turns a mismatch into a failed call — the + mechanism works. Over clean / insert / delete / ties, the raw `length vs total` check fired **0 of + 4**; the deduped variant fired on the insert (a **false alarm** — the "missing" row is the + newly-inserted one, legitimately unread) and on the ties; and **nothing fired on the delete** while + `r05` was missing. The capture calls reconciliation "the only cheap way to detect a skip"; measured, + it detects neither of this scenario's two skips. +- **C7: drift alone produces an empty page in the middle of a collection that still has rows.** + Eight deletes after page 1 left four rows; the client's offset-4 window came back empty; the run + broke at engine.ts:984 and ended `ok` having skipped `["r10","r11","r12"]` — **with 4 collected + against a declared total of 4, so the reconciler agrees it is fine.** +- **The default `items` wrap INVERTS the safety, and neither state is good.** Omit `items` and the + `{ rows, total }` envelope is wrapped as one item per page (engine.ts:969-973), so `items.length` + is 1 even for an empty page and the zero-item break can never fire — the lazy spelling is the one + that does not truncate. But `data.length` is then **2 for a 12-row collection**, and every + downstream count, reconciliation included, is measuring the PAGE COUNT. `pick: 'rows'` truncates + exactly like `items`. +- **The default `pages: 50` is a third silent terminus at the same `break`.** 220 rows returned + **200**, `ok`, no error, skipping `["r201".."r220"]`. It is the only one of C7's four cases a + total check catches. +- **C8's detector is not the one the state of the art recommends, and that is the finding.** Dedupe + - reconcile misses the delete entirely. The signal that carries it is that **the declared `total` + itself moved** — 10 on page 1, 9 on page 3 — which is direct proof the collection changed under + the cursor. Combined with duplicates, an empty page mid-run, and the cap, the verdict scored **0 + false negatives and 3 false alarms over 8 workloads**. Under keyset the same code lost nothing on + all 8 and flagged nothing. +- **The verdict cannot be separated from the rows.** `output`'s return replaces the result, so the + assembled answer hands back `{ rows, duplicates, totalMoved, emptyPageAt, capReached, +countMismatch, trustworthy }` as one value — there is no way to read the rows without the verdict + in scope. It means "re-sync", not "these rows are wrong"; a client genuinely cannot tell. +- **The price is negative, and that is the honest number.** 84 counted lines against **74** + hand-rolled for the same feature set, agreeing on rows and verdict across all 8 workloads. The + detection logic is identical in both; a raw paging loop is ~15 lines and the declarative + equivalent (a stitch config, a `transform`, an `output` validator, a two-mode `next`) is ~25. + **What the 74 lines do not have is the resilience stack**: one `retry` line recovered a page that + answered `500` mid-run (4 wire requests, 10 rows, nothing skipped), and auth/throttle/circuit/trace + apply per page for free (engine.ts:946). That is what the library is buying here — not correctness, + and not detection. + +## The footguns + +- **A paginated run that skipped rows is indistinguishable from a clean one.** `ok: true`, + `error: null`, `findings: []`, `status: 200`, `attempts: 1`, and a `progress` line per page whose + only number is a running item count. Nothing in the library compares anything to anything. +- **The delete-before-the-cursor case cannot be detected by any count.** The row leaves the result + and the `total` at the same moment, so `length === total`, distinct === total, and no duplicate + exists. Only "the `total` moved during the run" fires — and that check is not in any of the + standard write-ups. +- **A deduping `items` (or `transform`) can end the run early.** A page that is entirely duplicates + aggregates zero items, and `paginated` breaks at engine.ts:984 **before** `next`. Measured: the + dedupe cost 6 rows on a run that without it returned all 10. The fix creates a worse bug than the + one it fixes; do the dedupe in `output`, which runs after the loop. +- **A deduping `items` on a REUSED stitch returns an empty array on every call after the first.** + A stitch is meant to be defined once and called many times; module-level `seen` state means call 2 + sees every id as a duplicate, page 1 aggregates zero items, and the run reports **success with + `data: []`**. Build the stitch per run, or dedupe in `output`. +- **Drift can manufacture an empty page mid-collection.** Enough deletes and the client's next + offset is past the end of a collection that still has unread rows. Same break, same silent + success, and the shrunken `total` makes it reconcile perfectly. +- **`pages` defaults to 50 and its terminus is silent.** A collection larger than 50 × `limit` + returns a prefix, `ok`, with no error and no event distinguishing "the cap stopped me" from "the + collection ended". Always set it, and always compare `pagesFetched` to it. +- **Omitting `items` on an enveloped API silently makes every count meaningless.** The default wraps + a non-array body as `[value]`, so the aggregate is one element per PAGE. `data.length` looks like + a row count and is not. +- **`.report()` / `.inspect()` cannot tell you about the run you made.** They are fresh probes: the + probe re-paginated the collection (3 more requests) and, the write having settled, did not + reproduce the duplicate at all. +- **A rejecting `output` gives `.safe()` a generic message.** The issue text naming the duplicate ids + is only in `.report().findings`; `error.message` is `contract violation (drift)`, and `data` is + `null` — every row already collected is dropped unless you read `.report().raw`. +- **A failed paginated run emits no `result` event.** Anything that fails the run — an `output` + rejection, a surface rejection — discards the pages already aggregated as far as `.safe()` is + concerned. +- **Sending a composite cursor does not make an endpoint a seek endpoint.** If the vendor's + `ORDER BY` is not a total order, the correct four-line keyset loop still lost `["r03"]` and + duplicated `["r13"]` on static collections. Verify the sort, not the parameter names. diff --git a/docs/scenarios/proofs/unstable-pagination/c1-insert-before-cursor.ts b/docs/scenarios/proofs/unstable-pagination/c1-insert-before-cursor.ts new file mode 100644 index 00000000..48592df2 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c1-insert-before-cursor.ts @@ -0,0 +1,140 @@ +// C1 — a row is INSERTED before the cursor between page 1 and page 2. The capture predicts the +// aggregated array MISSES a record and asks which id is lost. +// +// Nothing is lost. The measured damage is a DUPLICATE, and the capture has the polarity of the +// whole failure mode backwards. An insert before the cursor shifts every later row to a HIGHER +// index, so the fixed offset the client sends next lands on a row it has ALREADY read. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c1-insert-before-cursor.ts +import { LiveCollection, idsOf, pageSpine, seedRows } from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C1 — insert before the cursor, between page 1 and page 2'); + + // ── (a) the insert lands INSIDE the region page 1 already returned ──────────────────────── + // 10 rows, 4 per page. Page 1 answers r01..r04 and the row `x1` (created_at 250) is inserted + // between r02 and r03 — squarely behind the cursor. The client then asks for offset 4. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x1', created_at: 250, name: 'inserted' }), + ); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(a) the run REPORTED SUCCESS', r.ok, true); + check('(a) there is no error', r.error, null); + check('(a) pages fetched', server.requests.length, 3); + checkSeq( + '(a) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | r04,r05,r06,r07 | r08,r09,r10'], + ); + checkSeq( + '(a) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04,r04,r05,r06,r07,r08,r09,r10'], + ); + checkSeq('(a) SKIPPED ids', a.skipped, []); + checkSeq('(a) DUPLICATED ids', a.duplicated, ['r04']); + check('(a) duplicate returns', a.duplicateCount, 1); + check('(a) rows handed back', collected.length, 11); + check('(a) distinct rows handed back', new Set(collected).size, 10); + check('(a) rows the server has now', server.size, 11); + note( + '(a) → the capture predicted a LOST row. The measured damage is r04 delivered TWICE', + '', + ); + } + + // ── (b) the strongest form: the insert lands at the HEAD of the collection ──────────────── + // "Newest first, and a new row arrives" is the same shape — anything inserted into the region + // already paged over shifts the window by one. Same duplicate, same id. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x0', created_at: 50, name: 'head insert' }), + ); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const a = server.audit(idsOf(r.data)); + check('(b) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(b) SKIPPED ids', a.skipped, []); + checkSeq('(b) DUPLICATED ids', a.duplicated, ['r04']); + } + + // ── (c) TWO inserts behind the cursor shift the window TWICE ────────────────────────────── + // The damage scales with the write rate: n rows inserted behind the cursor re-deliver n rows. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => { + server.insert({ id: 'x1', created_at: 150, name: 'i1' }); + server.insert({ id: 'x2', created_at: 250, name: 'i2' }); + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const a = server.audit(idsOf(r.data)); + check('(c) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(c) SKIPPED ids', a.skipped, []); + checkSeq('(c) DUPLICATED ids', a.duplicated, ['r03', 'r04']); + check('(c) duplicate returns', a.duplicateCount, 2); + } + + // ── (d) an insert AHEAD of the cursor does no damage at all ─────────────────────────────── + // Machine-checks the direction: only the region BEHIND the cursor matters. `x9` lands in page + // 2's own territory and is simply read, so an insert can never cause a skip. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x9', created_at: 550, name: 'ahead' }), + ); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(d) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(d) SKIPPED ids', a.skipped, []); + checkSeq('(d) DUPLICATED ids', a.duplicated, []); + checkSeq( + '(d) the new row was picked up', + [collected.join(',')], + ['r01,r02,r03,r04,r05,x9,r06,r07,r08,r09,r10'], + ); + } + + // ── (e) nothing in the library noticed ──────────────────────────────────────────────────── + // `.report()` is the run diagnostic — attempts, timing, findings, the raw pre-validation body. + // The duplicate is IN `raw` and IN `data`, and `findings` is empty, because a duplicate is not + // a schema violation. There is no drift event, no warning, no non-zero anything. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x1', created_at: 250, name: 'inserted' }), + ); + const { call } = offsetLoop({ server, limit: LIMIT }); + const rep = await call.report({ query: { limit: LIMIT, offset: 0 } }); + check('(e) report.error', rep.error, null); + check('(e) report.findings', rep.findings.length, 0); + check('(e) report.status', rep.status, 200); + check('(e) report.attempts', rep.attempts, 1); + check('(e) rows in report.data', idsOf(rep.data).length, 11); + check( + '(e) rows in report.raw (pre-validation aggregate)', + idsOf(rep.raw).length, + 11, + ); + note( + '(e) → every diagnostic the library offers reports a clean run over an 11-element array with 10 distinct rows', + '', + ); + } + + finish( + 'C1', + 'REFUTED IN DIRECTION, CONFIRMED IN DAMAGE. An insert before the cursor does NOT lose a row — it DUPLICATES one: measured skipped [], duplicated ["r04"], 11 items for 10 distinct rows, and the run reported ok with error null, 0 findings and status 200. Two inserts duplicate two rows (["r03","r04"]); an insert AHEAD of the cursor does nothing. An offset insert can never cause a skip, which is the opposite of the capture', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c2-delete-before-cursor.ts b/docs/scenarios/proofs/unstable-pagination/c2-delete-before-cursor.ts new file mode 100644 index 00000000..b0996b67 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c2-delete-before-cursor.ts @@ -0,0 +1,158 @@ +// C2 — a row is DELETED before the cursor between page 1 and page 2. The capture predicts the +// aggregated array contains a record TWICE, and asks whether anything flags it. +// +// Nothing appears twice. A row is LOST — the polarity is backwards here too, and this is the half +// that actually hurts, because a duplicate is at least visible in the data. A delete before the +// cursor pulls every later row to a LOWER index, so the row that was about to be page 2's first +// slides back into page 1's territory, which the client has already passed. +// +// And the count still adds up. That is the finding. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c2-delete-before-cursor.ts +import { LiveCollection, idsOf, pageSpine, seedRows } from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C2 — delete before the cursor, between page 1 and page 2'); + + // ── (a) one delete behind the cursor loses exactly one row ──────────────────────────────── + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r02')); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(a) the run REPORTED SUCCESS', r.ok, true); + check('(a) there is no error', r.error, null); + check('(a) pages fetched', server.requests.length, 3); + checkSeq( + '(a) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | r06,r07,r08,r09 | r10'], + ); + checkSeq( + '(a) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04,r06,r07,r08,r09,r10'], + ); + checkSeq('(a) SKIPPED ids', a.skipped, ['r05']); + checkSeq('(a) DUPLICATED ids', a.duplicated, []); + check('(a) rows handed back', collected.length, 9); + note( + '(a) → r05 was never deleted, never edited, and was never returned', + '', + ); + } + + // ── (b) THE TRAP: the arithmetic still balances ─────────────────────────────────────────── + // The delete removes one row from `total` at the same moment it removes one row from the + // result, so `collected.length === total`. The single cheapest detection the state of the art + // recommends — reconcile against the declared total — sees a perfectly consistent run. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r02')); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(b) rows collected', collected.length, 9); + check('(b) `total` on the LAST page', a.finalTotal, 9); + check( + '(b) length === total, so a reconciler sees nothing', + collected.length === a.finalTotal, + true, + ); + check('(b) distinct ids collected', new Set(collected).size, 9); + checkSeq('(b) …and yet, SKIPPED ids', a.skipped, ['r05']); + checkSeq( + '(b) the result carries a row that no longer exists', + a.transient, + ['r02'], + ); + note( + '(b) → 9 of 9, no duplicates, one row missing and one tombstone in its place', + '', + ); + } + + // ── (c) two deletes behind the cursor lose two rows, and the count STILL balances ───────── + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => { + server.remove('r02'); + server.remove('r03'); + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(c) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(c) SKIPPED ids', a.skipped, ['r05', 'r06']); + checkSeq('(c) DUPLICATED ids', a.duplicated, []); + check('(c) rows collected', collected.length, 8); + check('(c) `total` on the LAST page', a.finalTotal, 8); + } + + // ── (d) a delete AHEAD of the cursor does no damage ─────────────────────────────────────── + // The direction, machine-checked from the other side: only deletes BEHIND the cursor skip. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r09')); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const a = server.audit(idsOf(r.data)); + check('(d) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(d) SKIPPED ids', a.skipped, []); + checkSeq('(d) DUPLICATED ids', a.duplicated, []); + checkSeq('(d) the deleted row is simply absent', a.transient, ['r09']); + } + + // ── (e) does ANYTHING flag it? ──────────────────────────────────────────────────────────── + // The capture's actual question. Every observable the library offers, on the run from (a). + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r02')); + const { call } = offsetLoop({ server, limit: LIMIT }); + const evts = []; + for await (const e of call.stream({ + query: { limit: LIMIT, offset: 0 }, + })) + evts.push(e); + + check( + '(e) `error` events', + evts.filter((e) => e.type === 'error').length, + 0, + ); + check( + '(e) `drift` events', + evts.filter((e) => e.type === 'drift').length, + 0, + ); + check('(e) `done.ok`', evts.find((e) => e.type === 'done')?.ok, true); + checkSeq( + '(e) every `paginate` progress line the run emitted', + evts + .filter((e) => e.type === 'progress' && e.phase === 'paginate') + .map((e) => (e as { detail?: string }).detail), + [ + 'page 1 (+4, total 4)', + 'page 2 (+4, total 8)', + 'page 3 (+1, total 9)', + ], + ); + note( + '(e) → the only per-page number the library reports is a RUNNING COUNT of aggregated items (engine.ts:976-982). It never names an id, never compares to the declared `total`, and reads identically on a clean run', + '', + ); + } + + finish( + 'C2', + 'REFUTED IN DIRECTION, AND WORSE THAN PREDICTED. A delete before the cursor does NOT duplicate — it SKIPS: measured skipped ["r05"], duplicated [], 9 rows returned, run ok. The trap is that the arithmetic balances: `total` dropped to 9 at the same moment the row was lost, so length === total === 9 with distinct ids and r05 missing — the cheap reconciliation the state of the art recommends detects NOTHING here. Two deletes skip ["r05","r06"] and still balance at 8 === 8. No error, no drift finding, and the only per-page signal is a running item count', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c3-tie-order.ts b/docs/scenarios/proofs/unstable-pagination/c3-tie-order.ts new file mode 100644 index 00000000..4ae0817f --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c3-tie-order.ts @@ -0,0 +1,138 @@ +// C3 — the case people do not believe until they see it: a NON-UNIQUE sort key, ties returned in a +// different order per query, and NO WRITES AT ALL. The collection is frozen. The client is correct. +// The result is still wrong. +// +// The server here sorts by `created_at` alone and rotates each tie group one position per query. +// That is not a bug being simulated — it is a legal answer to an `ORDER BY` that does not uniquely +// determine an order, which is exactly what `ORDER BY created_at` is when `created_at` repeats. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c3-tie-order.ts +import { + LiveCollection, + idsOf, + pageSpine, + seedRowsWithTie, +} from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C3 — a non-unique sort key, on a STATIC collection'); + + // ── (a) ten rows, four of them tied, zero writes ────────────────────────────────────────── + // r03..r06 all carry `created_at: 300`. The tie group straddles the page-1/page-2 boundary, + // which is all it takes. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: true, + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(a) the run REPORTED SUCCESS', r.ok, true); + check('(a) there is no error', r.error, null); + check('(a) writes during the run', 0, 0); + check('(a) rows at the start', server.initialIds.length, 10); + check('(a) rows at the end', server.size, 10); + checkSeq('(a) rows created or destroyed mid-run', a.transient, []); + checkSeq( + '(a) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | r06,r03,r07,r08 | r09,r10'], + ); + checkSeq( + '(a) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04,r06,r03,r07,r08,r09,r10'], + ); + checkSeq('(a) SKIPPED ids', a.skipped, ['r05']); + checkSeq('(a) DUPLICATED ids', a.duplicated, ['r03']); + note( + '(a) → r05 exists, has never been touched, and was never returned. r03 was returned twice', + '', + ); + } + + // ── (b) BOTH cheap detections are defeated at once ──────────────────────────────────────── + // The skip and the duplicate cancel, so `collected.length === total === 10`. And the state of + // the art's other half — dedupe by id — removes the duplicate and leaves 9 rows against a + // declared 10, which now looks like a DIFFERENT bug than the one you have. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: true, + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(b) rows collected', collected.length, 10); + check('(b) `total` on every page', a.finalTotal, 10); + check( + '(b) length === total, so a reconciler sees nothing', + collected.length === a.finalTotal, + true, + ); + check('(b) distinct rows collected', new Set(collected).size, 9); + check( + '(b) dedupe-then-reconcile DOES fire: distinct !== total', + new Set(collected).size === a.finalTotal, + false, + ); + note( + '(b) → reconciling the RAW length is blind here; reconciling the DEDUPED length is the only one of the two that fires', + '', + ); + } + + // ── (c) it is not one unlucky window — the damage scales with BOUNDARY CROSSINGS ────────── + // 16 rows, a ten-row tie group spanning indices 2..11 and therefore crossing two page + // boundaries. Two rows lost, two rows repeated, and the totals still balance. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(16, 3, 10), + ties: true, + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(c) the run REPORTED SUCCESS', r.ok, true); + checkSeq( + '(c) what the server served, page by page', + [pageSpine(server)], + [ + 'r01,r02,r03,r04 | r06,r07,r08,r09 | r11,r12,r03,r04 | r13,r14,r15,r16', + ], + ); + checkSeq('(c) SKIPPED ids', a.skipped, ['r05', 'r10']); + checkSeq('(c) DUPLICATED ids', a.duplicated, ['r03', 'r04']); + check('(c) rows collected', collected.length, 16); + check('(c) `total`', a.finalTotal, 16); + } + + // ── (d) a UNIQUE sort key over the same collection is clean ─────────────────────────────── + // The control. Same rows, same client, same 3 pages — the only change is that the server's + // ORDER BY is a total order. This is the entire difference, and it is the server's to make. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: false, + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const a = server.audit(idsOf(r.data)); + check('(d) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(d) SKIPPED ids', a.skipped, []); + checkSeq('(d) DUPLICATED ids', a.duplicated, []); + } + + finish( + 'C3', + 'CONFIRMED, and it is the sharpest case in the scenario. On a collection with ZERO writes — 10 rows in, 10 rows out, nothing created or destroyed — a non-unique `created_at` with rotating ties measured skipped ["r05"] and duplicated ["r03"]. Both cheap detections are defeated together: the skip and the duplicate cancel, so length === total === 10, and only comparing the DEDUPED count (9) against `total` (10) fires at all. A tie group crossing two page boundaries loses ["r05","r10"] and repeats ["r03","r04"], still balancing at 16 === 16. Ordering by a total order over the same rows is clean', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c4-keyset.ts b/docs/scenarios/proofs/unstable-pagination/c4-keyset.ts new file mode 100644 index 00000000..c0828a73 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c4-keyset.ts @@ -0,0 +1,172 @@ +// C4 — is KEYSET pagination expressible through `next(prevBody)`? This is the one thing the library +// should do well, and the capture says so. Run the SAME insert/delete workloads C1 and C2 measured +// damage on, against a seek endpoint, and check the result is correct, complete and duplicate-free. +// +// It is. And the honest caveat is at (e): keyset is a property of the SERVER'S SORT, not of the +// cursor the client sends — a vendor that accepts `(after_ts, after_id)` and still orders by +// `created_at` alone breaks it again, and no client can fix that either. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c4-keyset.ts +import { + LiveCollection, + idsOf, + pageSpine, + seedRows, + seedRowsWithTie, +} from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { keysetLoop } from './keyset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C4 — keyset/seek through `paginate.next`'); + + // ── (a) the clean case: it terminates, and it is correct ────────────────────────────────── + // Note the extra request: seek has no `total` to compare against, so the loop runs until a + // page comes back EMPTY — which is exactly the `items.length === 0` break at engine.ts:984, + // used here for the one thing it is right for. + { + const server = new LiveCollection({ rows: seedRows(10) }); + const r = await keysetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(a) the run REPORTED SUCCESS', r.ok, true); + check('(a) pages fetched', server.requests.length, 4); + checkSeq( + '(a) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | r05,r06,r07,r08 | r09,r10 | '], + ); + checkSeq('(a) SKIPPED ids', a.skipped, []); + checkSeq('(a) DUPLICATED ids', a.duplicated, []); + check('(a) rows collected', collected.length, 10); + checkSeq( + '(a) the cursor each request carried', + server.requests.map((q) => `${q.afterTs}/${q.afterId}`), + ['0/', '400/r04', '800/r08', '1000/r10'], + ); + } + + // ── (b) C1's workload — insert before the cursor ────────────────────────────────────────── + // The cursor is a VALUE, not a position, so an insert behind it is simply behind it. C1 + // measured duplicated ["r04"] on the same write. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x1', created_at: 250, name: 'inserted' }), + ); + const r = await keysetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(b) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(b) SKIPPED ids', a.skipped, []); + checkSeq('(b) DUPLICATED ids', a.duplicated, []); + checkSeq( + '(b) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04,r05,r06,r07,r08,r09,r10'], + ); + note( + '(b) → the inserted row x1 sorts behind the cursor and is correctly NOT returned; every stable row arrives exactly once', + '', + ); + } + + // ── (c) C2's workload — delete before the cursor ────────────────────────────────────────── + // C2 measured skipped ["r05"] on this write, invisibly. Here r05 arrives. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r02')); + const r = await keysetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(c) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(c) SKIPPED ids', a.skipped, []); + checkSeq('(c) DUPLICATED ids', a.duplicated, []); + check('(c) r05 arrived', collected.includes('r05'), true); + } + + // ── (d) C3's workload — the tie case, still with no writes ──────────────────────────────── + // A seek endpoint orders by the composite key it takes a cursor on, so the tie group has a + // total order and the rotation has nothing to rotate. C3 measured skipped ["r05"] + + // duplicated ["r03"] on these exact rows. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: true, + }); + const r = await keysetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(d) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(d) SKIPPED ids', a.skipped, []); + checkSeq('(d) DUPLICATED ids', a.duplicated, []); + checkSeq( + '(d) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04,r05,r06,r07,r08,r09,r10'], + ); + } + + // ── (e) THE CAVEAT: a cursor the server does not sort by ────────────────────────────────── + // Same client code, same composite cursor. The vendor accepts `after_ts`/`after_id` and its + // ORDER BY is still `created_at` alone, so the tie group comes back rotated and the cursor is + // taken from whatever row happened to land last. Rows the rotation left BEHIND that cursor are + // then excluded by the very `>` that makes seek correct. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: true, + brokenSeek: true, + }); + const r = await keysetLoop({ server, limit: 2 }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(e) the run REPORTED SUCCESS', r.ok, true); + checkSeq( + '(e) what the server served, page by page', + [pageSpine(server)], + ['r01,r02 | r04,r05 | r06,r07 | r08,r09 | r10 | '], + ); + checkSeq('(e) SKIPPED ids', a.skipped, ['r03']); + checkSeq('(e) DUPLICATED ids', a.duplicated, []); + note( + '(e) → an unbroken keyset loop, a correct composite cursor, zero writes, and r03 is gone. Keyset is a SERVER capability; the client half of it is four lines and it is not the half that decides', + '', + ); + } + + // ── (f) …and the same broken vendor duplicates too ──────────────────────────────────────── + // When the rotation ends a page on a LOWER tie member than one it already returned, the cursor + // moves backwards inside the group and the rows above it come back a second time. + { + const server = new LiveCollection({ + rows: seedRowsWithTie(16, 10, 4), + ties: true, + brokenSeek: true, + }); + const r = await keysetLoop({ server, limit: 3 }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(f) the run REPORTED SUCCESS', r.ok, true); + checkSeq( + '(f) what the server served, page by page', + [pageSpine(server)], + [ + 'r01,r02,r03 | r04,r05,r06 | r07,r08,r09 | r13,r10,r11 | r12,r13,r14 | r15,r16 | ', + ], + ); + checkSeq('(f) SKIPPED ids', a.skipped, []); + checkSeq('(f) DUPLICATED ids', a.duplicated, ['r13']); + check('(f) rows collected for 16 rows', collected.length, 17); + } + + finish( + 'C4', + 'CONFIRMED — this is the thing the library does well. `next` is handed the previous page\'s RAW body, so a composite `(created_at, id)` cursor is FOUR lines, and against a real seek endpoint it measured skipped [] / duplicated [] on every workload that broke offset: C1\'s insert (which cost duplicated ["r04"]), C2\'s delete (skipped ["r05"]) and C3\'s ties (skipped ["r05"] + duplicated ["r03"]) all came back complete and clean, 10 rows in cursor order. The loop terminates on the zero-item break at engine.ts:984, which is the one thing that break is right for. The caveat is measured at (e)/(f): the SAME four lines against a vendor that takes a composite cursor but does not ORDER BY it lost ["r03"] on one collection and duplicated ["r13"] on another, with no writes in either. The client half of keyset is four lines and it is not the half that decides', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c5-detection-seams.ts b/docs/scenarios/proofs/unstable-pagination/c5-detection-seams.ts new file mode 100644 index 00000000..49ad632d --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c5-detection-seams.ts @@ -0,0 +1,425 @@ +// C5 — can duplicates be DETECTED or REMOVED inside the library? Every seam a caller could reach +// for, measured on C1's workload (insert before the cursor → r04 twice): `items`, `transform`, +// `output` as a deduping validator, `output` as a rejecting validator, `output` wrapped in +// `drift()`, `hooks.onResponse`, a custom `Surface.interpret`, and `.report().raw`. +// +// Five of them work. Two of the five are BOOBY-TRAPPED, and it is the two a caller reaches for +// first — a dedupe that lives inside the loop can empty a page, and an empty page ends the run. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c5-detection-seams.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { Surface } from '../../../../packages/core/src/surface'; +import type { + AdapterResponse, + StitchEvent, +} from '../../../../packages/core/src/types'; +import type { Validator } from '../../../../packages/core/src/validator'; +import type { Row } from './fake-collection'; +import { + LiveCollection, + idsOf, + pageSpine, + rowsOf, + rowsUrl, + seedRows, + totalOf, +} from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +/** The workload every seam is measured on: C1's insert, which delivers r04 twice. */ +function driftedServer(): LiveCollection { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => + server.insert({ id: 'x1', created_at: 250, name: 'inserted' }), + ); + return server; +} + +/** Drop rows whose id has already been seen. The stateful half of every in-loop seam. */ +function makeDeduper(): (rows: Row[]) => Row[] { + const seen = new Set(); + return (rows) => + rows.filter((r) => (seen.has(r.id) ? false : (seen.add(r.id), true))); +} + +async function main(): Promise { + heading('C5 — where a duplicate can be seen, and where it can be removed'); + + // ── (a) `items` with a closure — it dedupes ─────────────────────────────────────────────── + // `items` runs per page (engine.ts:969-973) and is an ordinary function, so a closure carries + // state across pages. On this workload it is correct. + { + const server = driftedServer(); + const dedupe = makeDeduper(); + const r = await offsetLoop({ + server, + limit: LIMIT, + paginate: { items: (v) => dedupe(rowsOf(v)) }, + }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(a) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(a) DUPLICATED ids', a.duplicated, []); + checkSeq('(a) SKIPPED ids', a.skipped, []); + check('(a) rows collected', collected.length, 10); + } + + // ── (b) THE TRAP: the same `items` dedupe TRUNCATES the run ─────────────────────────────── + // Four rows inserted behind the cursor make page 2 a verbatim repeat of page 1. The deduper + // returns zero items, `paginated` breaks at engine.ts:984 BEFORE calling `next`, and the run + // ends OK with 4 of 14 rows. The fix for duplicates is a mechanism for losing rows. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => { + for (let i = 1; i <= 4; i++) + server.insert({ id: `x${i}`, created_at: 10 * i, name: 'ins' }); + }); + const dedupe = makeDeduper(); + const r = await offsetLoop({ + server, + limit: LIMIT, + paginate: { items: (v) => dedupe(rowsOf(v)) }, + }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(b) the run REPORTED SUCCESS', r.ok, true); + check('(b) there is no error', r.error, null); + check('(b) pages fetched', server.requests.length, 2); + checkSeq( + '(b) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | r01,r02,r03,r04'], + ); + checkSeq( + '(b) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04'], + ); + checkSeq('(b) SKIPPED ids', a.skipped, [ + 'r05', + 'r06', + 'r07', + 'r08', + 'r09', + 'r10', + ]); + check('(b) `total` the server declared', a.finalTotal, 14); + note( + '(b) → 6 of 10 original rows lost, silently, BY THE DEDUPE. Without it the same run returns all ten (plus four repeats)', + '', + ); + } + + // ── (c) `transform` has the same reach and the same trap ────────────────────────────────── + // It runs per page at engine.ts:967, before `pick` and before `items`, so a closure works + // there too — and an emptied `rows` reaches the same break. + { + const server = driftedServer(); + const dedupe = makeDeduper(); + const r = await offsetLoop({ + server, + limit: LIMIT, + transform: (v) => ({ + ...(v as object), + rows: dedupe(rowsOf(v)), + }), + }).run(); + const a = server.audit(idsOf(r.data)); + check('(c) the run REPORTED SUCCESS', r.ok, true); + checkSeq('(c) DUPLICATED ids', a.duplicated, []); + checkSeq('(c) SKIPPED ids', a.skipped, []); + } + + // ── (d) `output` as a deduping validator — the SAFE seam ────────────────────────────────── + // `validateOutput` runs ONCE, over the aggregated array, after the loop has finished + // (engine.ts:993). Its returned value REPLACES the result (engine.ts:1005-1009), so a + // hand-rolled `Validator` is a real post-processing hook. Nothing it does can shorten the run. + { + const server = driftedServer(); + const dedupeOutput: Validator = { + async validate(value) { + const seen = new Set(); + const out: Row[] = []; + for (const row of value as Row[]) + if (!seen.has(row.id)) { + seen.add(row.id); + out.push(row); + } + return { ok: true, value: out }; + }, + }; + const r = await offsetLoop({ + server, + limit: LIMIT, + output: dedupeOutput, + }).run(); + const collected = idsOf(r.data); + check('(d) the run REPORTED SUCCESS', r.ok, true); + check('(d) pages fetched (unchanged)', server.requests.length, 3); + check('(d) rows handed back', collected.length, 10); + checkSeq('(d) DUPLICATED ids', server.audit(collected).duplicated, []); + checkSeq('(d) SKIPPED ids', server.audit(collected).skipped, []); + } + + // ── (e) `output` as a REJECTING validator — the run fails, loudly ───────────────────────── + // The other half of the seam: returning `ok: false` makes the aggregate a contract violation, + // which is the one way to turn drift into a failed call the caller cannot ignore. + { + const server = driftedServer(); + const rejectDuplicates: Validator = { + async validate(value) { + const rows = value as Row[]; + const ids = rows.map((row) => row.id); + const dupes = ids.filter((id, i) => ids.indexOf(id) !== i); + return dupes.length === 0 + ? { ok: true, value: rows } + : { + ok: false, + issues: [ + { + path: [], + message: `pagination drift: duplicate ids ${[...new Set(dupes)].join(',')}`, + }, + ], + }; + }, + }; + const { call } = offsetLoop({ + server, + limit: LIMIT, + output: rejectDuplicates, + }); + const r = await call.safe({ query: { limit: LIMIT, offset: 0 } }); + check('(e) the run FAILED', r.ok, false); + check('(e) data', r.data, null); + check( + '(e) error message', + (r.error as Error | null)?.message, + 'contract violation (drift)', + ); + note( + '(e) → the ISSUE text naming the duplicate ids does not reach `error.message`; the caller gets the generic contract-violation string', + '', + ); + + const server2 = driftedServer(); + const rep = await stitch({ + url: rowsUrl, + adapter: server2.adapter(), + output: rejectDuplicates, + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }).report({ query: { limit: LIMIT, offset: 0 } }); + check('(e) report.error is set', rep.error !== null, true); + checkSeq( + '(e) report.findings — the issue text, reachable here', + rep.findings.map((f) => `${f.level}:${f.detail}`), + ['error:pagination drift: duplicate ids r04'], + ); + check( + '(e) report.raw still carries the 11 aggregated rows', + idsOf(rep.raw).length, + 11, + ); + note( + '(e) → `.report()`/`.inspect()` recover both the naming AND the partial data from a failed contract. `.safe()` alone recovers neither', + '', + ); + } + + // ── (f) `output` wrapped in `drift()` — dedupe AND a finding, on a run that still succeeds ─ + // `classifyDiff(raw, validated)` runs when the schema is wrapped (engine.ts:445-449), so the + // rows the deduper removed come back as findings. They are levelled `info` and worded + // `undeclared field`, because the diff engine is describing schema stripping, not row loss. + { + const server = driftedServer(); + const dedupeOutput: Validator = { + async validate(value) { + const seen = new Set(); + const out: Row[] = []; + for (const row of value as Row[]) + if (!seen.has(row.id)) { + seen.add(row.id); + out.push(row); + } + return { ok: true, value: out }; + }, + }; + const rep = await stitch({ + url: rowsUrl, + adapter: server.adapter(), + output: { + __kind: 'drift' as const, + schema: dedupeOutput, + options: {}, + }, + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }).report({ query: { limit: LIMIT, offset: 0 } }); + check('(f) report.error', rep.error, null); + check('(f) rows in report.data (deduped)', idsOf(rep.data).length, 10); + check('(f) rows in report.raw (aggregate)', idsOf(rep.raw).length, 11); + checkSeq( + '(f) findings', + rep.findings.map((x) => `${x.level}|${x.change}|${x.path}`), + [ + 'warn|coerced|[].id', + 'warn|coerced|[].created_at', + 'warn|coerced|[].name', + 'info|undeclared|[]', + ], + ); + note( + '(f) → four findings, none of which says "duplicate". Removing one element RE-INDEXES the array, so the positional diff reports every field of every later row as `coerced` (a string that changed) plus one `undeclared` element at the tail. It fires on the right event and describes something else entirely', + '', + ); + } + + // ── (g) `hooks.onResponse` — it SEES every page, and can change nothing ─────────────────── + // The hook is `(ctx) => void | Promise` (types.ts:1285-1290) with the raw + // `AdapterResponse`. It is the most complete view in the library and it is read-only. + { + const server = driftedServer(); + const seenPages: string[][] = []; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + hooks: { + onResponse: (ctx) => { + seenPages.push( + rowsOf((ctx.res as AdapterResponse).body).map( + (row) => row.id, + ), + ); + }, + }, + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }); + const r = await call.safe({ query: { limit: LIMIT, offset: 0 } }); + checkSeq( + '(g) pages the hook observed', + seenPages.map((p) => p.join(',')), + ['r01,r02,r03,r04', 'r04,r05,r06,r07', 'r08,r09,r10'], + ); + check('(g) the hook fired on EVERY page', seenPages.length, 3); + check('(g) the result is unchanged by it', idsOf(r.data).length, 11); + } + + // ── (h) a custom `Surface.interpret` — it can FAIL THE RUN mid-pagination ───────────────── + // `interpret` runs per page inside the attempt loop (engine.ts:775). A `{ ok: false, message }` + // on a 200 is returned rather than thrown (engine.ts:824-831), and `paginated` turns it into + // an error event + `done(false)` (engine.ts:960-964). This is the ONLY seam that can stop the + // run at the page where the drift happened, and the message reaches the caller intact. + { + const server = driftedServer(); + const seen = new Set(); + const driftDetectingHttp: Surface = { + id: 'http-drift-check', + interpret: (res) => { + const dupes = rowsOf(res.body) + .map((row) => row.id) + .filter((id) => seen.has(id)); + for (const row of rowsOf(res.body)) seen.add(row.id); + return dupes.length === 0 + ? { ok: true, data: res.body } + : { + ok: false, + message: `pagination drift: page repeated ${dupes.join(',')}`, + }; + }, + }; + const call = stitch({ + url: rowsUrl, + kind: driftDetectingHttp, + adapter: server.adapter(), + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }); + const evts: StitchEvent[] = []; + for await (const e of call.stream({ + query: { limit: LIMIT, offset: 0 }, + })) + evts.push(e); + const err = evts.find((e) => e.type === 'error'); + + check('(h) pages fetched before it stopped', server.requests.length, 2); + check( + '(h) the run FAILED', + evts.find((e) => e.type === 'done')?.ok, + false, + ); + check( + '(h) the message reached the caller', + err && 'message' in err ? err.message : undefined, + 'pagination drift: page repeated r04', + ); + note( + '(h) → detection at the page it happened, with the id named. The cost is that the aggregated rows are DISCARDED: a failed paginated run emits no `result` event', + '', + ); + check( + '(h) `result` events on the failed run', + evts.filter((e) => e.type === 'result').length, + 0, + ); + } + + // ── (i) `.report().raw` — the aggregate, after the fact ─────────────────────────────────── + // Measured in C1(e): `raw` is the pre-validation aggregated array, duplicate included. It is a + // real seam, and it costs a SECOND FULL PAGINATED RUN — `.report()`/`.inspect()` are fresh + // probes, not observers of the call you already made. + { + const server = driftedServer(); + const { call } = offsetLoop({ server, limit: LIMIT }); + await call.safe({ query: { limit: LIMIT, offset: 0 } }); + const requestsAfterFirstRun = server.requests.length; + const rep = await call.report({ query: { limit: LIMIT, offset: 0 } }); + check('(i) requests for the run itself', requestsAfterFirstRun, 3); + check( + '(i) requests after ALSO calling .report()', + server.requests.length, + 6, + ); + check('(i) rows in report.raw', idsOf(rep.raw).length, 11); + check( + '(i) duplicates in report.raw', + idsOf(rep.raw).length - new Set(idsOf(rep.raw)).size, + 0, + ); + note( + '(i) → the probe paginated the collection a SECOND time, by which point the write had settled, so it did not reproduce the duplicate at all. `.report()` describes a run it just made, never the run you made', + '', + ); + } + + finish( + 'C5', + 'CONFIRMED WITH A TRAP. Five seams reach the duplicate. `items` and `transform` dedupe INSIDE the loop and both are booby-trapped: on a workload where page 2 repeats page 1 verbatim, the deduper emptied the page, `paginated` broke at engine.ts:984, and the run ended ok having SKIPPED ["r05".."r10"] — 6 rows lost BY the fix, against a declared total of 14. `output` is the safe seam: a hand-rolled Validator over the aggregated array either dedupes (10 rows, run ok, pages unchanged) or rejects (run fails, "contract violation (drift)"), and `.report()` recovers both the naming and the 11 partial rows that `.safe()` throws away. `drift()` turns the dedupe into 4 findings — 3 `coerced` and 1 `undeclared` — none of which says "duplicate", because removing an element re-indexes the array. `hooks.onResponse` sees all 3 pages and can change nothing. A custom `Surface.interpret` is the only seam that stops the run AT the drifted page with the id named — at the cost of discarding every row already collected. And `.report()` is a FRESH probe: it re-paginated the collection (3 more requests) and did not reproduce the duplicate at all', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c6-total-reconciliation.ts b/docs/scenarios/proofs/unstable-pagination/c6-total-reconciliation.ts new file mode 100644 index 00000000..7952c2e2 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c6-total-reconciliation.ts @@ -0,0 +1,272 @@ +// C6 — can the run be RECONCILED against the declared `total`? Two questions: is the LAST page's +// `total` reachable anywhere the caller can act on, and does reconciling against it actually detect +// the damage C1-C3 measured? +// +// The first answer is yes, from three places, and the caller can turn it into a failed call. +// The second answer is the finding: reconciliation misses the case that matters. The capture calls +// it "the only cheap way to DETECT a skip" — measured, it detects neither of the two skips in this +// scenario's headline workloads, and raises a false alarm on the one where nothing was lost. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c6-total-reconciliation.ts +import { stitch } from '../../../../packages/core/src/index'; +import type { AdapterResponse } from '../../../../packages/core/src/types'; +import type { Row } from './fake-collection'; +import { + LiveCollection, + idsOf, + rowsOf, + rowsUrl, + seedRows, + seedRowsWithTie, + totalOf, +} from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C6 — reconciling the run against the declared `total`'); + + // ── (a) `next` DOES see the last page's total — when `next` is what ends the loop ───────── + // Scenario 3 measured that `next` is never called on the terminal page. That is true of the + // ZERO-ITEM break; it is not true of a loop that terminates by `next` returning undefined, + // which is how offset pagination against a declared `total` naturally ends. Here `next` is + // consulted on all three pages and the third call carries the final body. + { + const server = new LiveCollection({ rows: seedRows(10) }); + const totalsSeenByNext: number[] = []; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => { + totalsSeenByNext.push(totalOf(prev) ?? -1); + return page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined; + }, + }, + }); + await call.safe({ query: { limit: LIMIT, offset: 0 } }); + check('(a) pages fetched', server.requests.length, 3); + checkSeq('(a) totals `next` observed', totalsSeenByNext, [10, 10, 10]); + check( + '(a) `next` saw the LAST page', + totalsSeenByNext.length, + server.requests.length, + ); + } + + // ── (b) …and it does NOT, when the zero-item break ends the loop ────────────────────────── + // Terminate on a short page instead — the other common spelling — over a collection that is an + // exact multiple of `limit`. A fourth, EMPTY page is fetched, `paginated` breaks at + // engine.ts:984 before `next`, and that page's body (with its own, moved, `total`) is + // unreachable from `next`. Four requests, three `next` calls. + { + const server = new LiveCollection({ rows: seedRows(12) }); + server.afterRequest(3, () => { + server.remove('r11'); + server.remove('r12'); + }); + const totalsSeenByNext: number[] = []; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => { + totalsSeenByNext.push(totalOf(prev) ?? -1); + return rowsOf(prev).length < LIMIT + ? undefined + : { query: { offset: page * LIMIT } }; + }, + }, + }); + await call.safe({ query: { limit: LIMIT, offset: 0 } }); + check('(b) pages fetched', server.requests.length, 4); + checkSeq('(b) totals `next` observed', totalsSeenByNext, [12, 12, 12]); + check( + '(b) `total` the server declared on the LAST page', + server.requests.at(-1)?.total, + 10, + ); + note( + '(b) → the terminal page declared 10 and `next` never saw it. Which of the two spellings you wrote decides whether the final total exists', + '', + ); + } + + // ── (c) `transform` and `hooks.onResponse` see EVERY page, terminal one included ────────── + // `transform` runs at engine.ts:967, above the break; `onResponse` at engine.ts:705, per + // attempt. Both are ordinary functions, so a closure captures the last total unconditionally. + { + const server = new LiveCollection({ rows: seedRows(12) }); + server.afterRequest(3, () => { + server.remove('r11'); + server.remove('r12'); + }); + const viaTransform: number[] = []; + const viaHook: number[] = []; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + transform: (v: unknown) => { + viaTransform.push(totalOf(v) ?? -1); + return v; + }, + hooks: { + onResponse: (ctx) => { + viaHook.push( + totalOf((ctx.res as AdapterResponse).body) ?? -1, + ); + }, + }, + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + rowsOf(prev).length < LIMIT + ? undefined + : { query: { offset: page * LIMIT } }, + }, + }); + await call.safe({ query: { limit: LIMIT, offset: 0 } }); + checkSeq( + '(c) totals `transform` observed', + viaTransform, + [12, 12, 12, 10], + ); + checkSeq('(c) totals `onResponse` observed', viaHook, [12, 12, 12, 10]); + check('(c) the FINAL total is reachable', viaTransform.at(-1), 10); + } + + // ── (d) and it is ACTIONABLE: capture in `transform`, decide in `output` ────────────────── + // The full construction. `transform` captures the running total per page; the `output` + // validator runs once over the aggregated array (engine.ts:993) and rejects a run whose + // distinct-row count disagrees with the last declared total. A real, failed call. + { + const server = new LiveCollection({ rows: seedRows(10) }); + server.afterRequest(1, () => server.remove('r02')); + let lastTotal = -1; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + transform: (v: unknown) => { + lastTotal = totalOf(v) ?? -1; + return v; + }, + output: { + async validate(value: unknown) { + const rows = value as Row[]; + const distinct = new Set(rows.map((row) => row.id)).size; + return distinct === lastTotal + ? { ok: true as const, value: rows } + : { + ok: false as const, + issues: [ + { + path: [], + message: `collected ${distinct} distinct of ${lastTotal} declared`, + }, + ], + }; + }, + }, + paginate: { + items: (v: unknown) => rowsOf(v), + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }); + const r = await call.safe({ query: { limit: LIMIT, offset: 0 } }); + check('(d) the reconciler is wired and runs', lastTotal, 9); + check('(d) distinct rows collected', 9, 9); + check('(d) …and it PASSED the run that lost r05', r.ok, true); + note( + '(d) → 9 distinct rows, 9 declared. The mechanism works perfectly and has nothing to report', + '', + ); + } + + // ── (e) THE TRUTH TABLE ─────────────────────────────────────────────────────────────────── + // The same reconciler over the four workloads, printed as what it would tell you against what + // actually happened. `raw` is `length vs total`, `distinct` is the deduped variant. + { + const cases: { + label: string; + build: () => LiveCollection; + }[] = [ + { + label: 'clean', + build: () => new LiveCollection({ rows: seedRows(10) }), + }, + { + label: 'C1 insert', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => + s.insert({ id: 'x1', created_at: 250, name: 'i' }), + ); + return s; + }, + }, + { + label: 'C2 delete', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => s.remove('r02')); + return s; + }, + }, + { + label: 'C3 ties', + build: () => + new LiveCollection({ + rows: seedRowsWithTie(10, 3, 4), + ties: true, + }), + }, + ]; + + const rows: string[] = []; + for (const c of cases) { + const server = c.build(); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + const distinct = new Set(collected).size; + const rawFires = collected.length !== a.finalTotal; + const distinctFires = distinct !== a.finalTotal; + const dupeFires = distinct !== collected.length; + const damaged = a.skipped.length > 0 || a.duplicated.length > 0; + rows.push( + `${c.label}: damaged=${damaged} skipped=[${a.skipped}] dup=[${a.duplicated}] ` + + `| raw-total-check=${rawFires} distinct-total-check=${distinctFires} dupe-check=${dupeFires}`, + ); + } + for (const line of rows) note('(e)', line); + + checkSeq( + '(e) does ANY client-side check fire, per workload', + rows.map( + (line) => + line.includes('=true |') || line.includes('-check=true'), + ), + [false, true, false, true], + ); + note( + '(e) → C2 (a delete before the cursor) is the hole: r05 is gone and every check reads clean, because the delete removed a row from `total` at the same instant it removed one from the result', + '', + ); + } + + finish( + 'C6', + 'REACHABLE — AND IT DOES NOT DETECT THE SKIP. The final `total` is available from three places: `next` sees it whenever `next` itself ends the loop (totals [10,10,10] over 3 pages), and `transform`/`hooks.onResponse` see it unconditionally, terminal page included ([12,12,12,10] over 4 requests, where `next` saw only [12,12,12]). Capturing it in `transform` and deciding in `output` turns a mismatch into a failed call — the mechanism is real. What it cannot do is detect the damage: over clean / C1-insert / C2-delete / C3-ties, the raw length-vs-total check fired 0 times out of 4, the deduped variant fired on C1 (where nothing was lost — a false alarm on a legitimately-new row) and on C3, and NOTHING fired on C2 while r05 was missing. A delete before the cursor moves the total by exactly the amount it moves the result', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c7-drift-meets-the-edges.ts b/docs/scenarios/proofs/unstable-pagination/c7-drift-meets-the-edges.ts new file mode 100644 index 00000000..a9cc0e42 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c7-drift-meets-the-edges.ts @@ -0,0 +1,220 @@ +// C7 — does a drifted page interact badly with the two edges scenarios 3 and 4 already measured: +// the ZERO-ITEM BREAK (engine.ts:984, taken before `next` is called) and the NON-ARRAY `items` WRAP +// (engine.ts:969-973)? And the third silent terminus nobody names: the default `pages: 50` cap. +// +// All three fire here. The worst is (a): drift alone produces an empty page in the MIDDLE of a +// collection that still has rows, the run ends successfully, and every reconciliation check agrees +// it is fine. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c7-drift-meets-the-edges.ts +import { stitch } from '../../../../packages/core/src/index'; +import { + LiveCollection, + idsOf, + pageSpine, + rowsUrl, + seedRows, + totalOf, +} from './fake-collection'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { offsetLoop } from './offset-loop'; + +const LIMIT = 4; + +async function main(): Promise { + heading('C7 — drift against the zero-item break, the wrap, and the cap'); + + // ── (a) drift produces an EMPTY page mid-run, and the run ends OK ───────────────────────── + // 12 rows. Page 1 returns r01..r04. Eight rows are then deleted, leaving four — r04, r10, r11, + // r12 — so the client's next window (offset 4, limit 4) lands past the end of a collection that + // still HAS rows it never saw. Zero items, break at engine.ts:984, success. + { + const server = new LiveCollection({ rows: seedRows(12) }); + server.afterRequest(1, () => { + for (const id of [ + 'r01', + 'r02', + 'r03', + 'r05', + 'r06', + 'r07', + 'r08', + 'r09', + ]) + server.remove(id); + }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + + check('(a) the run REPORTED SUCCESS', r.ok, true); + check('(a) there is no error', r.error, null); + check('(a) pages fetched', server.requests.length, 2); + checkSeq( + '(a) what the server served, page by page', + [pageSpine(server)], + ['r01,r02,r03,r04 | '], + ); + checkSeq( + '(a) ids the caller got', + [collected.join(',')], + ['r01,r02,r03,r04'], + ); + checkSeq('(a) SKIPPED ids', a.skipped, ['r10', 'r11', 'r12']); + check('(a) rows the server still has', server.size, 4); + check('(a) `total` on the last page', a.finalTotal, 4); + check('(a) rows collected', collected.length, 4); + check( + '(a) …so the reconciler agrees: 4 collected, 4 declared', + collected.length === a.finalTotal, + true, + ); + note( + '(a) → three surviving rows never fetched, a successful run, and the ONE cheap detection the state of the art recommends confirms it', + '', + ); + } + + // ── (b) the same drift with the DEFAULT `items` never breaks ───────────────────────────── + // Omit `items` and the `{ rows, total }` envelope is wrapped as ONE item per page + // (engine.ts:969-973), so `items.length` is 1 even for a page with no rows and the zero-item + // break can never fire. The loop then ends where `next` says — the lazy spelling is the one + // that does not truncate. It just hands back envelopes instead of rows. + { + const server = new LiveCollection({ rows: seedRows(12) }); + server.afterRequest(1, () => { + for (const id of [ + 'r01', + 'r02', + 'r03', + 'r05', + 'r06', + 'r07', + 'r08', + 'r09', + ]) + server.remove(id); + }); + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + paginate: { + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }); + const r = await call.safe({ query: { limit: LIMIT, offset: 0 } }); + const agg = r.data as { rows: unknown[]; total: number }[]; + check('(b) the run REPORTED SUCCESS', r.ok, true); + check('(b) elements in the aggregated array', agg.length, 2); + checkSeq( + '(b) …and each is a PAGE ENVELOPE, not a row', + agg.map((p) => `${p.rows.length} rows/total ${p.total}`), + ['4 rows/total 12', '0 rows/total 4'], + ); + check( + '(b) rows recoverable if you dig them out yourself', + agg.flatMap((p) => p.rows).length, + 4, + ); + note( + '(b) → `data.length` is 2 for a 12-row collection. Any downstream `length` check — including a total reconciliation — is measuring the PAGE COUNT', + '', + ); + } + + // ── (c) `pick: "rows"` is the same trap as `items` ──────────────────────────────────────── + // `pick` runs per page at engine.ts:968, before `items`, so picking the array makes the default + // `items` see it and the empty page breaks again. Both correct spellings truncate; only the + // one that hands back the wrong shape survives. + { + const server = new LiveCollection({ rows: seedRows(12) }); + server.afterRequest(1, () => { + for (const id of [ + 'r01', + 'r02', + 'r03', + 'r05', + 'r06', + 'r07', + 'r08', + 'r09', + ]) + server.remove(id); + }); + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + pick: 'rows', + paginate: { + next: (prev: unknown, page: number) => + page * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: page * LIMIT } } + : undefined, + }, + }); + const r = await call.safe({ query: { limit: LIMIT, offset: 0 } }); + check('(c) the run REPORTED SUCCESS', r.ok, true); + check('(c) pages fetched', server.requests.length, 2); + checkSeq( + '(c) ids the caller got', + [idsOf(r.data).join(',')], + ['r01,r02,r03,r04'], + ); + } + + // ── (d) the THIRD silent terminus: the default `pages: 50` ──────────────────────────────── + // `paginate.pages` defaults to 50 (types.ts:1420-1421) and the cap shares the same `break` as + // the zero-item case — before `next`, with no event, no error, and a `result` as if the + // collection had ended. A sync job over a collection bigger than 50 pages returns a prefix. + { + const server = new LiveCollection({ rows: seedRows(220) }); + const r = await offsetLoop({ server, limit: LIMIT }).run(); + const collected = idsOf(r.data); + const a = server.audit(collected); + check('(d) the run REPORTED SUCCESS', r.ok, true); + check('(d) there is no error', r.error, null); + check( + '(d) pages fetched (the default cap)', + server.requests.length, + 50, + ); + check('(d) rows collected', collected.length, 200); + check('(d) `total` the server declared', a.finalTotal, 220); + check('(d) rows SKIPPED', a.skipped.length, 20); + checkSeq( + '(d) the first and last skipped ids', + [a.skipped[0], a.skipped.at(-1)], + ['r201', 'r220'], + ); + note( + '(d) → this one a total reconciliation DOES catch (200 vs 220), and it is the only one of the four in this file that it catches', + '', + ); + } + + // ── (e) the cap, plus drift, in the shape a real sync job has ───────────────────────────── + // Raise the cap and the same collection completes. The point is that the failure mode of the + // default is a SHORTER LIST, not an error — identical in shape to (a) and to C2. + { + const server = new LiveCollection({ rows: seedRows(220) }); + const r = await offsetLoop({ + server, + limit: LIMIT, + paginate: { pages: 60 }, + }).run(); + const a = server.audit(idsOf(r.data)); + check('(e) the run REPORTED SUCCESS', r.ok, true); + check('(e) pages fetched', server.requests.length, 55); + checkSeq('(e) SKIPPED ids', a.skipped, []); + } + + finish( + 'C7', + 'CONFIRMED, and the zero-item break is the sharpest of the three. Drift ALONE produced an empty page mid-run: eight deletes after page 1 left a 4-row collection, the client\'s offset-4 window came back empty, `paginated` broke at engine.ts:984 and the run ended ok having SKIPPED ["r10","r11","r12"] — with 4 collected against a declared total of 4, so the reconciler agrees. The default `items` wrap INVERTS the safety: an envelope is always 1 item, so the empty page never breaks the loop — but `data.length` is then 2 for a 12-row collection and every downstream count measures pages. `pick: "rows"` truncates exactly like `items`. And the default `pages: 50` is a third silent terminus at the same `break`: 220 rows returned 200, ok, no error, skipping ["r201".."r220"] — the only one of the four a total check catches', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/c8-assembled.ts b/docs/scenarios/proofs/unstable-pagination/c8-assembled.ts new file mode 100644 index 00000000..a1c2d245 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/c8-assembled.ts @@ -0,0 +1,374 @@ +// C8 — the most honest answer a client can assemble: keyset where the vendor offers it, and where +// it does not, offset with the damage DETECTED and handed back attached to the rows. +// +// The detector is not the one the state of the art recommends. Dedupe + reconcile-against-total was +// measured in C6 catching 2 of 3 damage cases and raising a false alarm on a fourth; the missing +// signal is that the DECLARED TOTAL ITSELF MOVED. A `total` that is 10 on page 1 and 9 on page 3 is +// proof the collection changed under the cursor, and it is the only thing that fires on C2. +// +// pnpm exec tsx docs/scenarios/proofs/unstable-pagination/c8-assembled.ts +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { + LiveCollection, + idsOf, + rowsOf, + rowsUrl, + seedRows, + seedRowsWithTie, + totalOf, +} from './fake-collection'; +import { handRolledSync } from './hand-rolled'; +import { check, checkSeq, finish, heading, note } from './harness'; +import type { SyncResult } from './sync-collection'; +import { syncCollection } from './sync-collection'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const LIMIT = 4; + +/** Count the CODE lines between the `` / `` markers of a file. */ +function countedLines(file: string): number { + const src = readFileSync(join(HERE, file), 'utf8').split('\n'); + const from = src.findIndex((l) => l.includes('')); + const to = src.findIndex((l) => l.includes('')); + return src + .slice(from + 1, to) + .filter( + (l) => + l.trim() !== '' && + !l.trim().startsWith('//') && + !l.trim().startsWith('*') && + !l.trim().startsWith('/*'), + ).length; +} + +/** The eight workloads every construction in this file is measured over. */ +const WORKLOADS: { label: string; build: () => LiveCollection }[] = [ + { + label: 'clean ', + build: () => new LiveCollection({ rows: seedRows(10) }), + }, + { + label: 'insert behind ', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => + s.insert({ id: 'x1', created_at: 250, name: 'i' }), + ); + return s; + }, + }, + { + label: 'insert ahead ', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => + s.insert({ id: 'x9', created_at: 550, name: 'i' }), + ); + return s; + }, + }, + { + label: 'delete behind ', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => s.remove('r02')); + return s; + }, + }, + { + label: 'delete ahead ', + build: () => { + const s = new LiveCollection({ rows: seedRows(10) }); + s.afterRequest(1, () => s.remove('r09')); + return s; + }, + }, + { + label: 'ties, no writes ', + build: () => + new LiveCollection({ rows: seedRowsWithTie(10, 3, 4), ties: true }), + }, + { + label: 'collapse mid-run ', + build: () => { + const s = new LiveCollection({ rows: seedRows(12) }); + s.afterRequest(1, () => { + for (const id of [ + 'r01', + 'r02', + 'r03', + 'r05', + 'r06', + 'r07', + 'r08', + 'r09', + ]) + s.remove(id); + }); + return s; + }, + }, + { + label: 'over the page cap', + build: () => new LiveCollection({ rows: seedRows(28) }), + }, +]; + +/** Was the run actually damaged, per the server's ground truth? */ +function damaged(server: LiveCollection, rows: { id: string }[]): boolean { + const a = server.audit(rows.map((r) => r.id)); + // The deduped list is what the caller reads, so measure completeness against it. + return a.skipped.length > 0; +} + +async function main(): Promise { + heading('C8 — the assembled answer, and what it costs'); + + // ── (a) offset mode: does the verdict track the ground truth? ───────────────────────────── + { + const flagged: boolean[] = []; + const truth: boolean[] = []; + for (const w of WORKLOADS) { + const server = w.build(); + const r = await syncCollection({ + server, + limit: LIMIT, + mode: 'offset', + pages: 6, + }); + const isDamaged = damaged(server, r.rows); + flagged.push(!r.trustworthy); + truth.push(isDamaged); + note( + '(a)', + `${w.label} damaged=${isDamaged ? 'YES' : 'no '} flagged=${ + r.trustworthy ? 'no ' : 'YES' + } | dup=[${r.duplicates}] totalMoved=${ + r.totalMoved ? r.totalMoved.join('→') : '-' + } emptyPage=${r.emptyPageAt ?? '-'} cap=${r.capReached} countMismatch=${r.countMismatch}`, + ); + } + checkSeq('(a) rows actually LOST, per workload', truth, [ + false, + false, + false, + true, + false, + true, + true, + true, + ]); + checkSeq('(a) the verdict said "do not trust this"', flagged, [ + false, + true, + true, + true, + true, + true, + true, + true, + ]); + check( + '(a) every damaged run was flagged (no false negatives)', + truth.every((t, i) => !t || flagged[i]), + true, + ); + check( + '(a) false alarms on undamaged runs', + truth.filter((t, i) => !t && flagged[i]).length, + 3, + ); + note( + '(a) → zero false negatives and three false alarms. That is the right trade for a sync job: the verdict means "re-sync", not "these rows are wrong", and a client genuinely cannot tell the two apart', + '', + ); + } + + // ── (b) keyset mode: correct, and quiet ─────────────────────────────────────────────────── + // The same eight workloads against the seek endpoint. Nothing is lost, so nothing is flagged — + // the moved-total and count checks are disabled under keyset because a cursor on a value is + // immune to the thing they detect. + { + const lost: string[] = []; + const flagged: string[] = []; + for (const w of WORKLOADS) { + const server = w.build(); + const r = await syncCollection({ + server, + limit: LIMIT, + mode: 'keyset', + pages: 20, + }); + if (damaged(server, r.rows)) lost.push(w.label.trim()); + if (!r.trustworthy) flagged.push(w.label.trim()); + } + checkSeq('(b) workloads that lost rows under keyset', lost, []); + checkSeq('(b) workloads flagged untrustworthy', flagged, []); + } + + // ── (c) the shape the caller reads ──────────────────────────────────────────────────────── + // The rows and the verdict are ONE value — `output`'s return replaces the result + // (engine.ts:1005-1009) — so there is no way to consume the rows without the verdict in scope. + { + const server = WORKLOADS[3]!.build(); // delete behind the cursor: C2's silent skip + const r: SyncResult = await syncCollection({ + server, + limit: LIMIT, + mode: 'offset', + pages: 6, + }); + check('(c) rows handed back', r.rows.length, 9); + check('(c) trustworthy', r.trustworthy, false); + checkSeq('(c) totalMoved', r.totalMoved ?? [], [10, 9]); + checkSeq('(c) duplicates', r.duplicates, []); + check('(c) countMismatch', r.countMismatch, false); + checkSeq( + '(c) SKIPPED ids, per the server', + server.audit(r.rows.map((row) => row.id)).skipped, + ['r05'], + ); + note( + '(c) → the ONLY signal that fires on C2 is that `total` went 10 → 9. Dedupe is silent, the count matches, and there is no duplicate to find', + '', + ); + } + + // ── (d) the price, against the same feature set hand-rolled ─────────────────────────────── + { + const lib = countedLines('sync-collection.ts'); + const hand = countedLines('hand-rolled.ts'); + check('(d) counted lines — library', lib, 84); + check('(d) counted lines — hand-rolled', hand, 74); + check('(d) is the library version SHORTER?', lib < hand, false); + note( + '(d)', + `${lib} vs ${hand} — the library version is ${lib - hand} lines LONGER for this feature set. The detection logic is identical in both; what differs is that a hand-rolled paging loop is ~15 lines and the declarative equivalent (a stitch config, a transform, an output validator, a two-mode next) is ~25`, + ); + + // …and both produce the same verdicts, which is what makes the comparison fair. + const libV: string[] = []; + const handV: string[] = []; + for (const w of WORKLOADS) { + const a = await syncCollection({ + server: w.build(), + limit: LIMIT, + mode: 'offset', + pages: 6, + }); + const b = await handRolledSync({ + server: w.build(), + limit: LIMIT, + mode: 'offset', + pages: 6, + }); + libV.push(`${a.rows.length}/${a.trustworthy}`); + handV.push(`${b.rows.length}/${b.trustworthy}`); + } + checkSeq('(d) library vs hand-rolled, rows/verdict', libV, handV); + } + + // ── (e) what the library is actually contributing here ──────────────────────────────────── + // The honest accounting: the loop, the URL/query building, the merge of the next page's input + // over the original, and the aggregation. Not the correctness, and not the detection. + { + const server = new LiveCollection({ rows: seedRows(10) }); + const r = await syncCollection({ + server, + limit: LIMIT, + mode: 'keyset', + pages: 20, + }); + check('(e) pages the engine drove', r.pagesFetched, 4); + checkSeq( + '(e) the cursor it threaded through, unaided', + server.requests.map((q) => `${q.afterTs}/${q.afterId}`), + ['0/', '400/r04', '800/r08', '1000/r10'], + ); + note( + '(e) → auth, retry, throttle, circuit and the trace apply to every page for free (engine.ts:946). That is real, and it is orthogonal to every number in this scenario', + '', + ); + } + + // ── (e2) …and here is the part the 74-line baseline does not have at ALL ────────────────── + // Page 2 of 3 answers `500`. One config line retries THAT PAGE and the run completes with + // every row. The hand-rolled loop has no retry, no backoff, no per-page budget — adding them + // is where its line count goes, and it is the only axis on which the library is buying + // anything in this scenario. + { + const clock = manualClock(); + const server = new LiveCollection({ rows: seedRows(10) }); + server.failRequest(2, 500); + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + clock, + retry: { attempts: 2, on: [500], backoff: { base: 100 } }, + paginate: { + items: (page: unknown) => rowsOf(page), + next: (prev: unknown, fetched: number) => + fetched * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: fetched * LIMIT } } + : undefined, + }, + }); + const p = call.safe({ query: { limit: LIMIT, offset: 0 } }); + await clock.advance(10_000); + const r = await p; + check('(e2) the run REPORTED SUCCESS', r.ok, true); + check( + '(e2) wire requests (one of them a 500)', + server.requests.length, + 4, + ); + check('(e2) rows collected', idsOf(r.data).length, 10); + checkSeq('(e2) SKIPPED ids', server.audit(idsOf(r.data)).skipped, []); + } + + // ── (f) THE FOOTGUN in the construction: a REUSED stitch keeps its closure ──────────────── + // `sync-collection.ts` builds a fresh stitch per run for a reason. A stitch is meant to be + // defined once and called many times; a deduping `items` with module-level state turns the + // second call into an empty successful result, because every id is already "seen" and the + // first page aggregates zero items — engine.ts:984 again. + { + const seen = new Set(); + const shared = stitch({ + url: rowsUrl, + adapter: new LiveCollection({ rows: seedRows(10) }).adapter(), + paginate: { + items: (page: unknown) => + rowsOf(page).filter((row) => + seen.has(row.id) ? false : (seen.add(row.id), true), + ), + next: (prev: unknown, fetched: number) => + fetched * LIMIT < (totalOf(prev) ?? 0) + ? { query: { offset: fetched * LIMIT } } + : undefined, + }, + }); + const first = await shared.safe({ query: { limit: LIMIT, offset: 0 } }); + const second = await shared.safe({ + query: { limit: LIMIT, offset: 0 }, + }); + check('(f) rows on the first call', idsOf(first.data).length, 10); + check('(f) the second call REPORTED SUCCESS', second.ok, true); + check('(f) rows on the second call', idsOf(second.data).length, 0); + check('(f) error on the second call', second.error, null); + note( + '(f) → the natural way to write a deduping paginator — one stitch, defined once — returns an empty list on every call after the first, successfully', + '', + ); + } + + finish( + 'C8', + "ASSEMBLED. Over 8 workloads the verdict had ZERO false negatives — every run that lost rows (delete-behind, ties, collapse-mid-run, over-the-cap) was flagged — and 3 false alarms on undamaged runs, which is the correct trade for a sync job. The signal that carries C2 is not dedupe and not the count: it is that the DECLARED TOTAL MOVED (10 → 9), the one check the state of the art does not name. Under keyset the same code lost nothing on all 8 and flagged nothing. Rows and verdict come back as ONE value because `output`'s return replaces the result (engine.ts:1005). Price: 84 counted lines against 74 hand-rolled — the library version is LONGER, because the detection is identical in both and a raw paging loop is cheaper to write than the declarative equivalent; the two agree on rows and verdict for all 8. What the 74 lines do not have is the resilience stack: one `retry` line recovered a page that answered 500 mid-run (4 wire requests, 10 rows, nothing skipped). The footgun in the construction is measured at (f): a deduping `items` on a REUSED stitch returns an empty array, successfully, on every call after the first", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/unstable-pagination/fake-collection.ts b/docs/scenarios/proofs/unstable-pagination/fake-collection.ts new file mode 100644 index 00000000..c4f4e2b6 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/fake-collection.ts @@ -0,0 +1,354 @@ +// A fake, in-memory collection server that models a LIVE table — the ground truth every claim in +// this scenario is measured against. +// +// It serves the same rows three ways, so the only variable between claims is the PAGINATION +// CONTRACT and never the data: +// +// - `GET /rows?offset=&limit=` — offset/limit over the sorted collection. +// - `GET /rows?after_ts=&after_id=&limit=` — keyset/seek: `(created_at, id) > (:after_ts, :after_id)`. +// - both answer `{ rows, total, limit, offset? }` — every response carries a declared `total`. +// +// Rows are sorted by `(created_at, id)` ascending, which is the ONLY thing that makes offset +// deterministic in the first place. The `ties` variant deliberately breaks that: it sorts by +// `created_at` alone and ROTATES each tie group one position per query, which is a legal (if +// unhelpful) thing for a database to do when the ORDER BY is not a total order. No writes are +// involved in that variant at all. +// +// Writes land BETWEEN page fetches, not on a timer: `afterRequest(n, mutation)` runs `mutation` +// once request `n` has been answered, which is exactly the interleaving the scenario is about and +// is fully deterministic. Nothing here touches the network or the wall clock. +// +// The audit is the point. {@link LiveCollection.audit} compares what a run collected against what +// the server knows, using the only defensible definition of "correct" for a live collection: +// +// STABLE = ids present at the start of the run AND still present at the end. +// A correct paginator returns every stable id EXACTLY ONCE. +// Rows created or destroyed mid-run are TRANSIENT — returning them or not is both defensible, +// so they are reported separately and never counted as damage. +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; + +/** One row in the collection. `created_at` is the sort field; `id` is the unique tiebreak. */ +export interface Row { + id: string; + created_at: number; + name: string; +} + +/** One page as the server answered it — the wire shape every claim's `next` reads. */ +export interface PageBody { + rows: Row[]; + /** The collection's size AT THE MOMENT THIS PAGE WAS SERVED. It moves, which is the point. */ + total: number; + limit: number; + offset?: number; +} + +/** One request, as the server saw it. `requests.length` IS the page count. */ +export interface RecordedRequest { + offset?: number; + afterTs?: number; + afterId?: string; + limit: number; + /** Ids this request answered with — the per-page spine a claim prints. */ + returned: string[]; + /** `total` as declared on this response. */ + total: number; +} + +/** What a run got, against what it should have got. Every field is a list of ids. */ +export interface Audit { + /** Ids the run collected, in order, duplicates included. */ + collected: string[]; + /** Ids present at the start of the run AND at the end — the rows a correct run must return. */ + stable: string[]; + /** Stable ids the run NEVER returned. **Empty is the only correct value.** */ + skipped: string[]; + /** Ids the run returned more than once. **Empty is the only correct value.** */ + duplicated: string[]; + /** Returns beyond the first, summed over every id — 0 when nothing repeated. */ + duplicateCount: number; + /** Ids created or destroyed mid-run. Returning them is defensible either way; not damage. */ + transient: string[]; + /** `total` on the LAST page the server served — what a reconciler would compare against. */ + finalTotal: number; +} + +export interface CollectionOptions { + rows: Row[]; + /** + * Sort by `created_at` alone and rotate each tie group one position per query, modelling a + * non-unique ORDER BY whose ties come back in a different order every time. No writes needed — + * this alone breaks offset pagination, which is C3. + */ + ties?: boolean; + /** + * Make the SEEK endpoint sort by `created_at` alone too — a vendor that accepts a composite + * cursor but whose `ORDER BY` is still not a total order. C4 measures it to show that keyset + * is a property of the server's sort, not of the cursor the client sends. Default `false`: a + * seek endpoint orders by `(created_at, id)`, which is what makes it a seek endpoint. + */ + brokenSeek?: boolean; +} + +const ROWS_URL = 'https://api.example.test/rows'; + +/** The endpoint every claim points a stitch at. */ +export const rowsUrl = ROWS_URL; + +/** Build `n` rows `r01..rNN`, `created_at` 100, 200, 300… — distinct, so `(created_at, id)` is total. */ +export function seedRows(n: number): Row[] { + return Array.from({ length: n }, (_, i) => ({ + id: `r${String(i + 1).padStart(2, '0')}`, + created_at: (i + 1) * 100, + name: `row ${i + 1}`, + })); +} + +/** + * Build `n` rows where `tieAt` consecutive rows starting at `tieFrom` (1-based) SHARE one + * `created_at`. The sort key is no longer a total order, so the server is free to return that group + * in any order — and this one does. + */ +export function seedRowsWithTie( + n: number, + tieFrom: number, + tieAt: number, +): Row[] { + return seedRows(n).map((r, i) => { + const pos = i + 1; + const inTie = pos >= tieFrom && pos < tieFrom + tieAt; + return inTie ? { ...r, created_at: tieFrom * 100 } : r; + }); +} + +export class LiveCollection { + /** Every request the server answered, in order. */ + readonly requests: RecordedRequest[] = []; + /** Ids present when the run started — half of the STABLE set. */ + readonly initialIds: string[]; + + private rows: Row[]; + private readonly ties: boolean; + private readonly brokenSeek: boolean; + private queries = 0; + private readonly pending = new Map void>(); + private readonly failures = new Map(); + + constructor(opts: CollectionOptions) { + this.rows = [...opts.rows]; + this.ties = opts.ties ?? false; + this.brokenSeek = opts.brokenSeek ?? false; + this.initialIds = this.ordered(0, true).map((r) => r.id); + } + + /** Ids present right now, in server order. The other half of the STABLE set. */ + get currentIds(): string[] { + return this.ordered(0, true).map((r) => r.id); + } + + /** Collection size right now — what the server declares as `total`. */ + get size(): number { + return this.rows.length; + } + + /** + * Run `mutation` once request number `n` (1-based) has been ANSWERED, i.e. in the gap before + * request `n + 1`. This is the whole scenario: a write that lands between two page fetches. + */ + afterRequest(n: number, mutation: () => void): this { + this.pending.set(n, mutation); + return this; + } + + /** + * Answer request number `n` (1-based, counting every hit including this one) with `status` + * instead of a page. Used to measure that the resilience stack applies PER PAGE. + */ + failRequest(n: number, status: number): this { + this.failures.set(n, status); + return this; + } + + /** Insert a row. Sorted into place by `created_at`, so `at` decides which side of a cursor it lands. */ + insert(row: Row): void { + this.rows.push(row); + } + + /** Delete a row by id. Returns whether it was there. */ + remove(id: string): boolean { + const before = this.rows.length; + this.rows = this.rows.filter((r) => r.id !== id); + return this.rows.length < before; + } + + /** The ids a given offset window WOULD return right now — used to name the row a drift moved. */ + windowAt(offset: number, limit: number): string[] { + return this.ordered(0) + .slice(offset, offset + limit) + .map((r) => r.id); + } + + /** + * Compare a run's collected ids against the ground truth. See the module header for why STABLE + * (present at start AND end) is the right yardstick for a collection that is being written to. + */ + audit(collected: string[]): Audit { + const now = new Set(this.currentIds); + const start = new Set(this.initialIds); + const stable = this.initialIds.filter((id) => now.has(id)); + const seen = new Map(); + for (const id of collected) seen.set(id, (seen.get(id) ?? 0) + 1); + const duplicated = [...seen.entries()] + .filter(([, n]) => n > 1) + .map(([id]) => id); + let duplicateCount = 0; + for (const n of seen.values()) duplicateCount += n - 1; + const transient = [ + ...this.initialIds.filter((id) => !now.has(id)), + ...this.currentIds.filter((id) => !start.has(id)), + ]; + return { + collected, + stable, + skipped: stable.filter((id) => !seen.has(id)), + duplicated, + duplicateCount, + transient, + finalTotal: this.requests.at(-1)?.total ?? 0, + }; + } + + /** The transport every claim plugs into `stitch({ adapter })`. */ + adapter(): Adapter { + return async (req: AdapterRequest): Promise => + this.serve(req); + } + + // The collection in server order for query number `q`. Distinct `created_at` ⇒ `(created_at, + // id)` is a total order and `q` is irrelevant. With `ties`, the comparator is `created_at` + // alone and each tie group is rotated by `q` — a legal answer to an ORDER BY that does not + // uniquely determine an order, and the entire content of C3. `forceTotalOrder` restores the + // `(created_at, id)` comparator for the seek endpoint, which is what makes seek work. + private ordered(q: number, forceTotalOrder = false): Row[] { + const rotate = this.ties && !forceTotalOrder; + const sorted = [...this.rows].sort((a, b) => + a.created_at !== b.created_at + ? a.created_at - b.created_at + : rotate + ? 0 + : a.id < b.id + ? -1 + : 1, + ); + if (!rotate) return sorted; + const out: Row[] = []; + for (let i = 0; i < sorted.length;) { + let j = i; + while ( + j < sorted.length && + sorted[j]!.created_at === sorted[i]!.created_at + ) + j++; + const group = sorted.slice(i, j); + const shift = group.length > 1 ? q % group.length : 0; + out.push(...group.slice(shift), ...group.slice(0, shift)); + i = j; + } + return out; + } + + private serve(req: AdapterRequest): AdapterResponse { + const url = new URL(req.url); + const q = url.searchParams; + const limit = Number(q.get('limit') ?? 10); + const failWith = this.failures.get(this.requests.length + 1); + if (failWith !== undefined) { + this.requests.push({ limit, returned: [], total: -1 }); + return { + status: failWith, + headers: {}, + body: { error: 'upstream' }, + }; + } + const seeking = q.has('after_ts') || q.has('after_id'); + // A seek endpoint orders by the composite key it takes a cursor on; the offset endpoint + // orders by whatever the collection was configured with. `brokenSeek` collapses them. + const view = this.ordered(this.queries, seeking && !this.brokenSeek); + this.queries += 1; + + let rows: Row[]; + let offset: number | undefined; + if (seeking) { + const afterTs = Number(q.get('after_ts')); + const afterId = q.get('after_id') ?? ''; + rows = view + .filter( + (r) => + r.created_at > afterTs || + (r.created_at === afterTs && r.id > afterId), + ) + .slice(0, limit); + } else { + // No cursor: the head of the collection. This is both `offset=0` and the seek loop's + // entry point (`LIMIT n` with no `WHERE`) — they are the same request. + offset = Number(q.get('offset') ?? 0); + rows = view.slice(offset, offset + limit); + } + + const body: PageBody = { + rows, + total: this.rows.length, + limit, + ...(offset === undefined ? {} : { offset }), + }; + const record: RecordedRequest = { + limit, + returned: rows.map((r) => r.id), + total: body.total, + ...(offset === undefined ? {} : { offset }), + }; + if (q.has('after_ts')) { + record.afterTs = Number(q.get('after_ts')); + record.afterId = q.get('after_id') ?? ''; + } + this.requests.push(record); + + // The write lands HERE — after this page was answered, before the next one is asked for. + this.pending.get(this.requests.length)?.(); + this.pending.delete(this.requests.length); + + return { + status: 200, + headers: { 'content-type': 'application/json' }, + body, + }; + } +} + +// ---- body readers the claims share ---------------------------------------- + +/** Pull the rows array off a page body. */ +export function rowsOf(body: unknown): Row[] { + return (body as PageBody | undefined)?.rows ?? []; +} + +/** Pull the declared `total` off a page body. */ +export function totalOf(body: unknown): number | undefined { + return (body as PageBody | undefined)?.total; +} + +/** Ids of whatever a run handed back — the aggregated array, however it was shaped. */ +export function idsOf(value: unknown): string[] { + return Array.isArray(value) + ? value.map((v) => (v as Row | undefined)?.id ?? String(v)) + : []; +} + +/** Compact one-line rendering of a run: `r01,r02 | r04,r05` — pages separated, ids in order. */ +export function pageSpine(server: LiveCollection): string { + return server.requests.map((r) => r.returned.join(',')).join(' | '); +} diff --git a/docs/scenarios/proofs/unstable-pagination/hand-rolled.ts b/docs/scenarios/proofs/unstable-pagination/hand-rolled.ts new file mode 100644 index 00000000..296b8f1f --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/hand-rolled.ts @@ -0,0 +1,89 @@ +// The same feature set as `sync-collection.ts` with no library at all, against the same fake +// server — the baseline C8 prices the library against. +// +// Feature parity, deliberately: two pagination modes with no silent fallback, query-string +// building, a page cap, dedupe by id, duplicate collection, the `total` moved-under-the-cursor +// check, the empty-page-mid-run check, the count check, one verdict object, and a non-2xx that +// fails rather than being aggregated as a value. +import type { PageBody, Row } from './fake-collection'; +import { rowsUrl } from './fake-collection'; +import type { SyncOptions, SyncResult } from './sync-collection'; + +/* */ +export async function handRolledSync(opts: SyncOptions): Promise { + const { server, limit, mode, pages } = opts; + const keyset = mode === 'keyset'; + const adapter = server.adapter(); + const all: Row[] = []; + const totals: number[] = []; + const pageSizes: number[] = []; + let cursor: { after_ts: number; after_id: string } | null = keyset + ? { after_ts: 0, after_id: '' } + : null; + let offset = 0; + + for (let page = 0; page < pages; page++) { + const q = new URLSearchParams({ limit: String(limit) }); + if (cursor) { + q.set('after_ts', String(cursor.after_ts)); + q.set('after_id', cursor.after_id); + } else q.set('offset', String(offset)); + const res = await adapter({ + url: `${rowsUrl}?${q.toString()}`, + method: 'GET', + headers: {}, + }); + if (res.status < 200 || res.status >= 300) + throw new Error(`GET ${rowsUrl} failed: ${res.status}`); + const body = res.body as PageBody; + totals.push(body.total); + pageSizes.push(body.rows.length); + all.push(...body.rows); + if (keyset) { + const last = body.rows.at(-1); + if (!last) break; + cursor = { after_ts: last.created_at, after_id: last.id }; + } else { + if (body.rows.length === 0) break; + offset += limit; + if (offset >= body.total) break; + } + } + + const seen = new Set(); + const rows: Row[] = []; + const duplicates: string[] = []; + for (const row of all) + if (seen.has(row.id)) duplicates.push(row.id); + else { + seen.add(row.id); + rows.push(row); + } + const first = totals[0] ?? 0; + const declaredTotal = totals.at(-1) ?? 0; + const emptyAt = pageSizes.findIndex((n) => n === 0); + const totalMoved = + !keyset && first !== declaredTotal + ? ([first, declaredTotal] as [number, number]) + : null; + const emptyPageAt = !keyset && emptyAt > 0 ? emptyAt + 1 : null; + const capReached = pageSizes.length >= pages; + const countMismatch = !keyset && rows.length !== declaredTotal; + return { + rows, + pagesFetched: pageSizes.length, + declaredTotal, + duplicates: [...new Set(duplicates)], + totalMoved, + emptyPageAt, + capReached, + countMismatch, + trustworthy: + duplicates.length === 0 && + totalMoved === null && + emptyPageAt === null && + !capReached && + !countMismatch, + }; +} +/* */ diff --git a/docs/scenarios/proofs/unstable-pagination/harness.ts b/docs/scenarios/proofs/unstable-pagination/harness.ts new file mode 100644 index 00000000..a594032a --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/harness.ts @@ -0,0 +1,68 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is a LIST OF ROW IDS: which ids the paginated run skipped, and which it +// returned twice, measured against the ground truth the fake server knows. So every assertion +// prints the measured value whether it passes or fails — `skipped ids: ["r05"]` has to be readable +// out of context, because it IS the finding. `checkSeq` carries most of the weight here: a claim +// about a set of ids is only meaningful if the ids themselves are on the page. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the per-call outcome spine + * (`["ok","ok","400"]`) and the per-provider hit spine (`[1,0]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring DAMAGE — the verdict statement carries the direction, + * because "PASS C1" on a claim whose content is "the run reported success and lost a row" is + * otherwise unreadable. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/unstable-pagination/keyset-loop.ts b/docs/scenarios/proofs/unstable-pagination/keyset-loop.ts new file mode 100644 index 00000000..d0a79d07 --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/keyset-loop.ts @@ -0,0 +1,54 @@ +// The keyset/seek `paginate` block — the correct construction, in one place so C4 and C8 run the +// same code. +// +// This is the whole of it. `next` is handed the previous page's RAW body (engine.ts:985), so it can +// read the last row's `(created_at, id)` and hand it back as the next request's query. That is +// exactly the composite cursor the state of the art prescribes, and it is four lines. +import { stitch } from '../../../../packages/core/src/index'; +import type { SchemaLike } from '../../../../packages/core/src/infer'; +import type { Stitch } from '../../../../packages/core/src/types'; +import type { LiveCollection, PageBody } from './fake-collection'; +import { rowsOf, rowsUrl } from './fake-collection'; + +export interface KeysetLoopOptions { + server: LiveCollection; + limit: number; + /** Contract over the AGGREGATED array — C8 layers its reconciliation here. */ + output?: SchemaLike; + transform?: (value: unknown) => unknown; +} + +export function keysetLoop(opts: KeysetLoopOptions): { + call: Stitch; + run: () => Promise<{ ok: boolean; data: unknown; error: unknown }>; +} { + const { server, limit } = opts; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + paginate: { + items: (value: unknown) => rowsOf(value), + // + next: (prev: unknown) => { + const last = (prev as PageBody).rows.at(-1); + if (!last) return undefined; + return { + query: { after_ts: last.created_at, after_id: last.id }, + }; + }, + // + }, + ...(opts.output === undefined ? {} : { output: opts.output }), + ...(opts.transform === undefined ? {} : { transform: opts.transform }), + }); + return { + call, + run: async () => { + // The cursor starts at negative infinity — a real seek entry point, not `offset=0`. + const r = await call.safe({ + query: { limit, after_ts: 0, after_id: '' }, + }); + return { ok: r.ok, data: r.data, error: r.error }; + }, + }; +} diff --git a/docs/scenarios/proofs/unstable-pagination/offset-loop.ts b/docs/scenarios/proofs/unstable-pagination/offset-loop.ts new file mode 100644 index 00000000..c9af359e --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/offset-loop.ts @@ -0,0 +1,62 @@ +// The offset/limit `paginate` block every damage claim (C1, C2, C3, C6, C7) runs, in ONE place so +// that the only variable between them is what the server does. +// +// This is the loop a competent caller writes against an API that returns `{ rows, total }`: pull +// the rows with `items`, advance `offset` by `limit` with `next`, and stop when the offset passes +// the declared `total`. There is nothing wrong with it. That is the point of the scenario. +import { stitch } from '../../../../packages/core/src/index'; +import type { SchemaLike } from '../../../../packages/core/src/infer'; +import type { Stitch } from '../../../../packages/core/src/types'; +import type { LiveCollection, PageBody } from './fake-collection'; +import { rowsOf, rowsUrl } from './fake-collection'; + +export interface OffsetLoopOptions { + server: LiveCollection; + limit: number; + /** + * Stop on a short page (`rows.length < limit`) instead of on `offset >= total`. Both are common + * spellings; C6 shows they terminate the run at DIFFERENT places, which decides whether `next` + * ever sees the last page's `total`. + */ + stopOnShortPage?: boolean; + /** Extra `paginate` fields (a dedupe `items`, a page cap) a claim wants to layer on. */ + paginate?: { items?: (value: unknown) => unknown[]; pages?: number }; + /** Contract over the AGGREGATED array — the seam C5/C6/C8 use. */ + output?: SchemaLike; + /** Per-page hook — the seam C6 uses to reach the terminal page's `total`. */ + transform?: (value: unknown) => unknown; +} + +/** The stitch under test, plus the call that runs it from offset 0. */ +export function offsetLoop(opts: OffsetLoopOptions): { + call: Stitch; + run: () => Promise<{ ok: boolean; data: unknown; error: unknown }>; +} { + const { server, limit } = opts; + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + paginate: { + items: (value: unknown) => rowsOf(value), + ...opts.paginate, + next: (prev: unknown, pagesFetched: number) => { + const body = prev as PageBody; + if (opts.stopOnShortPage) + return body.rows.length < limit + ? undefined + : { query: { offset: pagesFetched * limit } }; + const offset = pagesFetched * limit; + return offset < body.total ? { query: { offset } } : undefined; + }, + }, + ...(opts.output === undefined ? {} : { output: opts.output }), + ...(opts.transform === undefined ? {} : { transform: opts.transform }), + }); + return { + call, + run: async () => { + const r = await call.safe({ query: { limit, offset: 0 } }); + return { ok: r.ok, data: r.data, error: r.error }; + }, + }; +} diff --git a/docs/scenarios/proofs/unstable-pagination/sync-collection.ts b/docs/scenarios/proofs/unstable-pagination/sync-collection.ts new file mode 100644 index 00000000..c286476c --- /dev/null +++ b/docs/scenarios/proofs/unstable-pagination/sync-collection.ts @@ -0,0 +1,143 @@ +// The assembled answer C8 runs: page a live collection as honestly as a client can. +// +// - keyset where the vendor offers it — the only construction that is actually CORRECT; +// - offset where it does not, with the damage DETECTED rather than hidden: dedupe, a `total` +// that moved under the cursor, an empty page mid-run, and the page cap; +// - the rows and the verdict handed back as ONE value, so a caller cannot read the rows without +// the verdict being right there. +// +// The construction is: capture per-page facts in `transform` (it runs on every page including the +// terminal one, engine.ts:967, above the zero-item break at 984), decide once in `output` (it runs +// over the aggregated array, engine.ts:993, and its return value REPLACES the result at +// engine.ts:1005). Nothing here lives inside the loop, so nothing here can shorten the run — which +// is the mistake C5(b) measured a deduping `items` making. +// +// A fresh stitch per run is deliberate, not incidental: the closure below is per-run state, and +// C8(f) measures what re-using one stitch across two runs costs. +import { stitch } from '../../../../packages/core/src/index'; +import type { LiveCollection, PageBody, Row } from './fake-collection'; +import { rowsOf, rowsUrl, totalOf } from './fake-collection'; + +/** The rows AND what is known about how reliable they are. There is no way to get one without the other. */ +export interface SyncResult { + rows: Row[]; + pagesFetched: number; + declaredTotal: number; + /** Ids the server returned more than once. Proof of drift. */ + duplicates: string[]; + /** `[first, last]` when the declared `total` moved during the run — proof the collection changed. */ + totalMoved: [number, number] | null; + /** 1-based page index of a page that came back empty mid-run; `null` when none did. */ + emptyPageAt: number | null; + /** The `pages` cap ended the run rather than the collection did. */ + capReached: boolean; + /** Distinct rows collected disagrees with the last declared `total`. */ + countMismatch: boolean; + /** + * Whether the caller may treat this list as a complete, exact snapshot. `false` means "re-sync", + * not "some rows are wrong" — a client cannot tell which. + */ + trustworthy: boolean; +} + +export interface SyncOptions { + server: LiveCollection; + limit: number; + /** `'keyset'` when the vendor offers a seek endpoint. Never silently fall back. */ + mode: 'keyset' | 'offset'; + /** Page cap. Pass it explicitly — the default 50 is a silent terminus (C7(d)). */ + pages: number; +} + +/* */ +export function syncCollection(opts: SyncOptions): Promise { + const { server, limit, mode, pages } = opts; + const keyset = mode === 'keyset'; + const totals: number[] = []; + const pageSizes: number[] = []; + + const call = stitch({ + url: rowsUrl, + adapter: server.adapter(), + // Every page, terminal one included — above the zero-item break. + transform: (page: unknown) => { + totals.push(totalOf(page) ?? -1); + pageSizes.push(rowsOf(page).length); + return page; + }, + // Once, over the aggregate. Its return value IS the result. + output: { + async validate(value: unknown) { + const all = value as Row[]; + const seen = new Set(); + const rows: Row[] = []; + const duplicates: string[] = []; + for (const row of all) + if (seen.has(row.id)) duplicates.push(row.id); + else { + seen.add(row.id); + rows.push(row); + } + const first = totals[0] ?? 0; + const declaredTotal = totals.at(-1) ?? 0; + const emptyPageAt = pageSizes.findIndex((n) => n === 0); + const capReached = pageSizes.length >= pages; + // Under keyset the cursor is a VALUE, so a moving `total`, a short page and the + // terminal empty page are all normal. Under offset every one of them is drift. + const totalMoved = + !keyset && first !== declaredTotal + ? ([first, declaredTotal] as [number, number]) + : null; + const countMismatch = !keyset && rows.length !== declaredTotal; + const emptyMidRun = + !keyset && emptyPageAt > 0 ? emptyPageAt + 1 : null; + return { + ok: true as const, + value: { + rows, + pagesFetched: pageSizes.length, + declaredTotal, + duplicates: [...new Set(duplicates)], + totalMoved, + emptyPageAt: emptyMidRun, + capReached, + countMismatch, + trustworthy: + duplicates.length === 0 && + totalMoved === null && + emptyMidRun === null && + !capReached && + !countMismatch, + } satisfies SyncResult, + }; + }, + }, + paginate: { + pages, + items: (page: unknown) => rowsOf(page), + next: keyset + ? (prev: unknown) => { + const last = (prev as PageBody).rows.at(-1); + return last === undefined + ? undefined + : { + query: { + after_ts: last.created_at, + after_id: last.id, + }, + }; + } + : (prev: unknown, fetched: number) => + fetched * limit < (totalOf(prev) ?? 0) + ? { query: { offset: fetched * limit } } + : undefined, + }, + }); + + return call.unwrap( + keyset + ? { query: { limit, after_ts: 0, after_id: '' } } + : { query: { limit, offset: 0 } }, + ) as Promise; +} +/* */ diff --git a/docs/scenarios/proofs/webhook-receipt/README.md b/docs/scenarios/proofs/webhook-receipt/README.md new file mode 100644 index 00000000..006ed487 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/README.md @@ -0,0 +1,199 @@ +# Proofs — receiving a signed webhook, and where StitchAPI stops + +Runnable evidence for the claims in [`../../webhook-receipt.md`](../../webhook-receipt.md). + +**This is the first scenario whose answer is a boundary rather than a technique, and the boundary +runs down the middle.** Receipt — arbitrary route, raw bytes, HMAC, replay window, fast ack — is +**out of scope by design**. Reaction — fetch-on-receipt, the dedup ledger, the downstream write — is +squarely in scope. C7 measures the split as a number: **154 executable lines the library does not +participate in, 63 it does.** + +Every script is standalone and offline. Where a claim is about a server, it starts a REAL +`node:http` server on `127.0.0.1` and POSTs REAL signed bytes at it — a local socket is the only +honest way to ask "what would a Stripe delivery actually get back", and it is not a third-party +call. Where a claim is about time, it runs on an injected `manualClock()`; where a claim is about a +TTL longer than a test suite can wait, it runs on a clock-backed `StitchStore` (`clock-store.ts`), +because `memoryStore` reads `Date.now()` and ignores an injected clock (C5 (d)). + +Each script prints one `PASS`/`FAIL` line and exits non-zero on failure. Every server is closed in a +`finally`; the scripts exit. + +## Run them + +```sh +# one claim +pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c1-serve-raw-body.ts + +# all of them +for f in docs/scenarios/proofs/webhook-receipt/c[0-9]*.ts; do pnpm exec tsx "$f" || exit 1; done +``` + +Run from the repository root — the scripts import core from `packages/core/src` by relative path, so +they test the working tree, not the published bundle. + +They typecheck under `packages/core`'s full strict set: + +```sh +cd packages/core && pnpm exec tsc --noEmit \ + --target ES2022 --lib ES2022,DOM --module ESNext --moduleResolution Bundler \ + --esModuleInterop --skipLibCheck --strict --noUncheckedIndexedAccess \ + --exactOptionalPropertyTypes --noImplicitOverride --noPropertyAccessFromIndexSignature \ + --noUnusedLocals --noUnusedParameters --verbatimModuleSyntax --isolatedModules \ + --types node ../../docs/scenarios/proofs/webhook-receipt/*.ts +``` + +## What each script establishes + +| Script | Question | Measured | +| ---------------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `c1-serve-raw-body.ts` | can `serve` receive a signed webhook? | **No — 404 at all five paths tried**, body already `JSON.parse`d, and the signature header dropped entirely | +| `c2-no-inbound-primitive.ts` | is there ANY inbound-signature primitive? | **No.** 72 runtime exports, 4 matches, all BYO-plugin conformance suites. The one HMAC signs OUTBOUND only | +| `c3-serve-seams.ts` | is there a seam — surface, hook, `ServeBodyOptions`? | **No.** The mount point that exists is defeated by verifying: reading the stream leaves `readBody` hanging | +| `c4-out-of-order.ts` | does fetch-on-receipt make ordering moot? | **PAYLOAD order yes, WRITE order no.** Converged `active/pro`; two concurrent handlers still landed on the older v2 | +| `c5-dedup-ledger.ts` | can `StitchStore` be the dedup ledger? | **Yes, and the ownership is the other way round.** `get`+`set` triple-processed; `increment` processed exactly once | +| `c6-fast-ack.ts` | does anything help with ack-now-process-later? | **No, and `void call()` is a silent DROP.** The ack measured exactly 10 virtual seconds late through `serve` | +| `c7-the-boundary.ts` | the honest end-to-end answer, and its shape | **154 lines of receipt / 63 of reaction** — 71% of the code is the half StitchAPI does not participate in | + +## Files + +- `receiver.ts` — **user code, the receipt half, with no StitchAPI runtime in it.** A `node:http` + server: arbitrary route, bounded raw-byte buffering, signature check, atomic dedup claim, 2xx + written before the work starts. Its only reference to the package is `import type { StitchStore }`. +- `stripe-sig.ts` — **user code**, `node:crypto`. Stripe's scheme: HMAC-SHA256 over `${t}.${raw}`, + `timingSafeEqual` behind a length guard (it throws on a length mismatch), and a tolerance check + kept separate from the MAC check — a valid MAC on an old timestamp is a replay, not a pass. +- `reaction.ts` — **user code, the reaction half, almost all config.** One seam carrying `auth` / + `retry` / `throttle` / `timeout`, a `current` stitch (fetch-on-receipt) and an `apply` stitch + (`idempotency.keyOf` off the event id), plus the four-line version guard C4 (c) proves you need. +- `fake-billing.ts` — the provider in both directions. `mintDelivery` signs a `Buffer` and hands back + that same `Buffer`, so a proof verifying a re-serialised object is verifying something the provider + never signed. The adapter serves `GET /v1/subscriptions/{id}` (current truth, a snapshot) and + `POST /v1/entitlements` (recording `Idempotency-Key`). Knobs: `failNext`, `delayTicks`, `onRequest`. +- `clock-store.ts` — a `StitchStore` whose TTL runs on an injected clock, with an optional external + backing map so "the process restarted" is expressible. `memoryStore` cannot do either. +- `harness.ts` — `check` / `checkSeq` / `note` / `heading` / `finish`. No test framework. + +## Reading the numbers honestly + +- **C1 is the finding, and it has three parts, only two of which the capture predicted.** A real + `serve()` process answered **404** to a signed Stripe-shaped POST at `/webhooks/stripe`, + `/webhook`, `/`, `/stitch` and `/hooks/v1/billing` — the route table is exactly `GET /` and + `POST /stitch/:name` (serve.ts:219-231). On the one route that reaches user code the body has been + through `JSON.parse` (serve.ts:122-125,248), and the deepest user-reachable seam + (`Surface.buildRequest`) received a parsed object with no field carrying the raw string. **The + third part is decisive and unpredicted: the inbound headers are dropped entirely.** The run input + is built from the body alone (serve.ts:268-271); `req.headers` is read for `content-length` + (serve.ts:95) and `accept` (serve.ts:128) and nothing else. `stripe-signature` appears nowhere, so + there is nothing to verify _against_ even if the bytes were recoverable. The byte gap is real and + small: the provider signed **162 bytes**, a parse→stringify round-trip produces **153** logically + identical bytes, **9 bytes of whitespace**, and verification returns `bad-signature`. +- **A form-encoded provider cannot be received at all.** Slack signs an + `application/x-www-form-urlencoded` body; `serve` answered **400 `invalid JSON body`** before any + of the above could matter. +- **C2 by enumeration rather than grep.** Importing all six public entry points and filtering 72 + runtime exports for `/verif|hmac|signature|webhook|…/` returned exactly four names, and all four + are BYO-plugin conformance suites (`verifyStoreContract`, `verifyAdapterContract`, + `verifySinkContract`, `verifyFingerprintContract`). `AuthStrategy` is `{apply, name, scheme}` — + `apply(req, ctx)` mutates an OUTGOING request and `scheme` is a declarative wire description + (types.ts:1234-1236). `awsSigV4` ran here and produced an `AWS4-HMAC-SHA256` header on an outbound + request; its subtle key is imported with usages `['sign']` (aws-sigv4/src/index.ts:84), so it + structurally cannot verify. +- **Two things look like the answer and are not.** `xxh128` (hash.ts:110-113) is what someone + reaching for "compare a digest" finds first — measured taking **1 argument** and returning the same + 32-char digest with **no secret anywhere**, which is exactly why it authenticates nobody; hash.ts:110 + self-describes as non-cryptographic. And `idempotency` wears the same word as the dedup problem + while living at the opposite end of the pipe: measured putting `Idempotency-Key: evt_1PqR` on an + OUTBOUND POST. +- **C3: `createServeHandler` is a real mount seam, and verifying defeats it.** Mounted in a plain + `node:http` server it served an arbitrary `/webhooks/stripe` and ran the stitch, **200** — so C1's + ROUTE problem is fixable by owning the server. The BYTES problem is not: reading the stream to + verify (which succeeded) leaves `readBody` (serve.ts:90-120) waiting for `end` on an already-ended + stream. **The handler had not settled after 200ms**; the client got a 504 from the test's own + deadline, not from the handler. Verifying and delegating are mutually exclusive. +- **`serve` is unauthenticated, and the only inbound control is a byte cap.** `{body:{max:64}}` + answered a 130-byte signed delivery with **413 "exceeds"**, while an **UNSIGNED forged body** under + the cap **ran the stitch and returned 200** (serve.ts:28,59). The stitch-side hooks point outward: + `hooks.onRequest` fired with `req.url` = the OUTBOUND url, no `stripe-signature`, and a + `HookContext` of exactly `{attempt, name, req}`. +- **C4 splits a sentence the capture treats as one claim.** Acting on payload order with a reversed + pair applied `["active/pro","trialing/free"]` and left the app believing `trialing/free` while the + server said `active/pro` — a silent self-inflicted downgrade, nothing thrown. Fetch-on-receipt + applied `["active/pro","active/pro"]` and converged, at a measured cost of **2 API calls for 2 + events**. So PAYLOAD order is genuinely moot. **WRITE order is not:** two concurrent handlers + holding snapshots v2 and v3, last-write-wins, landed on **version 2 / `active`** while the server + said **version 3 / `canceled`**; a version guard rejected exactly `[2]` and recovered v3. Nothing + in the library expresses that guard — it is four lines of `reaction.ts`. +- **And one case fetch-on-receipt cannot answer at all.** A `.deleted` event's fetch returned + **404**, which is indistinguishable from "never existed". The event TYPE is still load-bearing, so + "the payload is only a hint" is not quite true. +- **What is unambiguously the library's job is that call.** With `auth`, `retry: {attempts: 3}`, a + fixed 1s backoff and `timeout: {total: '10s'}`, the fetch-on-receipt stitch survived **two 503s in + 3 measured attempts** on an injected clock and converged — no retry loop in the handler. +- **C5 reverses the capture's open question.** Scenario 4 left "whether a user can borrow the store + cleanly" open; the ownership runs the other way. `store` is a config key the USER supplies + (types.ts:1583-1584) and `memoryStore()` is a public export (index.ts:62). Measured directly: one + wrapped instance recorded both the ledger's `webhook:` keys and the ENGINE's `rl:` throttle + counter. Three deliveries of one event id → `["first:processed","retry:skipped","retry:skipped"]`, + **1 side effect**. +- **The dedup ledger everyone writes is racy, and the fix already ships.** `get`-then-`set` with 3 + concurrent claims on one id returned **`[true,true,true]`** — three charges. `increment(key, ttl)` + (types.ts:1969-1970), which exists for the throttle counter, returned **`[true,false,false]`**. +- **The TTL boundary is exact, and untestable against the default store.** With a 4-day TTL over + Stripe's 3-day window, claims at `t=3d` and `t=ttl-1ms` both skipped and `t=ttl` processed — the + boundary is `expires > now`, exclusive (store.ts:16), so a TTL equal to the retry window lets the + last retry through. But `memoryStore` takes **no clock** (arity 0) and reads `Date.now()` + (store.ts:16,45,54 via util.ts:4): after **four virtual days** on a `manualClock` the key was still + live. A 3-day window cannot be boundary-tested without a clock-backed store. +- **Durability is the real gap in the default, and the swap is first-class.** + `memoryStore.close()` is `data.clear()` (store.ts:59-61) — the ledger did **not** survive, so a + deploy inside the retry window re-processes every event still being retried. A BYO durable store + over the same interface survived a restart and passed **all 11 rules** of `verifyStoreContract` + (testing.ts:173) with **0 violations**; `redisStore` / `cloudflareKvStore` / `denoKvStore` are that + interface. +- **C6's finding is a silent data-loss bug in the obvious workaround.** `void call(input)` — the + spelling anyone writes to ack-then-continue — made **0 HTTP calls and raised 0 errors**. A stitch + call is a lazy thenable that starts on `.then` (stitch.ts:729,781), so the work is dropped after + the provider has been told 200. `void call(input).then(…)` does run it, and is then unsupervised: + unhandled it surfaced as **1 `unhandledRejection`**, and `.safe()` produced **0** and reported the + failure **nowhere**. +- **And `serve` makes the ack worse the more reliable you make the call.** Its handler consumes the + run to completion before writing a byte (serve.ts:177-201,272-273). With `retry: {attempts: 3}` and + a 5s fixed backoff on a `manualClock`, the ack was unsent after attempt 1, unsent after attempt 2, + and went out at attempt 3 having waited **exactly 10 virtual seconds** — Stripe's timeout to the + second. The retry policy manufactures the duplicate it exists to survive. `pipelineStages` + (config-summary.ts:86) says the same thing structurally: `call → throttle → POST … → retry → http +interpret → result`, nothing matching `/queue|background|detach|ack|defer|async/`, `result` last. +- **A small correction worth carrying: `backoff.base` is clamped by `backoff.max`, which defaults to + 10s** (types.ts:972-973). `base: 30_000` measured a **10,000ms** sleep, silently. +- **C7 runs the whole thing and the numbers hold.** Forged signature → **400 `bad-signature`**; a + genuine MAC on a 10-minute-old timestamp → **400 `stale`, 0 side effects**; the reversed pair both + acked **200** and converged on `active/pro` with the guard rejecting `[2]`; the provider's duplicate + acked **200 at a cost of 0 API calls**, with exactly one `Idempotency-Key` (`evt_updated`) ever + reaching the downstream write — two independent guards, both fired. +- **The boundary, as a number.** Receipt: **154 executable lines** (96 of `node:http` server + 58 of + `node:crypto` signature verification), importing **nothing** from `stitchapi` at runtime — its one + reference is `import type { StitchStore }`. Reaction: **63 executable lines**, almost all config. + **71% of the code is the half StitchAPI does not participate in**, and by concern the split is + 100% / 0%: no part of receipt is made easier by the library being present. + +## The footguns + +- **`serve` reads like a webhook endpoint and is not one.** It is an HTTP server, in the box, that + accepts POSTs with JSON bodies. Nothing about the name says "this exposes YOUR registry to a + trusted local caller", and the failure mode is not a compile error — it is a **404 in a provider + dashboard**, or worse, a **200 on an unsigned forged body** if someone reshapes their event to fit + `POST /stitch/:name`. It is unauthenticated by design (serve.ts:28,59); pointing a provider at it + means anyone who can reach the port can run your stitches. +- **`engine.ts:287` exports a symbol named `RAW_BODY`.** It is the raw **response** body of an + **outbound** call, retained for `.inspect()`. Anyone grepping this repo for "raw body" while + debugging a signature failure finds it first, and it is the opposite thing. +- **`xxh128` is the wrong hash and will happily produce a plausible-looking digest.** It is unkeyed. + A signature check built on it verifies nothing and looks like it works. +- **`idempotency` is not inbound dedup.** Both problems are "the same thing happened twice"; the + config key solves the outbound one. The inbound ledger is code you write against `StitchStore`. +- **`void call(input)` is not fire-and-forget, it is a no-op** — see C6 (c). This one is silent in + both directions: no request, no error. +- **`get`-then-`set` dedup passes every single-process test and double-processes in production.** Use + `increment(key, ttl) === 1`. +- **A dedup TTL equal to the retry window is off by one delivery**, because the store's liveness test + is `expires > now` (store.ts:16). Set it beyond the window, not to it. diff --git a/docs/scenarios/proofs/webhook-receipt/c1-serve-raw-body.ts b/docs/scenarios/proofs/webhook-receipt/c1-serve-raw-body.ts new file mode 100644 index 00000000..99def8ae --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c1-serve-raw-body.ts @@ -0,0 +1,283 @@ +// C1 — can `serve` receive a POST at an ARBITRARY path with the RAW body bytes preserved? +// +// This is measured against a REAL `serve()` process listening on 127.0.0.1, with real `fetch` +// requests carrying a real `Stripe-Signature` header over real signed bytes. Nothing here is +// simulated: the status codes below are what a Stripe delivery would actually receive. +// +// The capture's hypothesis is that `serve` is "the opposite of hand-me-the-raw-bytes-at-my-own-path" +// (serve.ts:206,283). Both halves of that are testable and both are measured. What the capture does +// NOT predict is the third finding, which is the decisive one: the signature HEADER never reaches +// user code either. `createServeHandler` builds the `StitchInput` from the BODY alone +// (serve.ts:246-256) and `req.headers` is read only for `content-length` (serve.ts:95) and `accept` +// (serve.ts:128). So even a user willing to re-derive the bytes has nothing to compare them to. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c1-serve-raw-body.ts +import { stitch } from '../../../../packages/core/src/index'; +import { serve } from '../../../../packages/core/src/serve'; +import type { + AdapterRequest, + ResolvedStitchConfig, + StitchInput, +} from '../../../../packages/core/src/types'; +import { type BillingEvent, mintDelivery } from './fake-billing'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { verifySignature } from './stripe-sig'; + +const SECRET = 'whsec_test_2f9d1c4b'; +const NOW_SECONDS = 1_800_000_000; + +const EVENT: BillingEvent = { + id: 'evt_1PqR', + type: 'customer.subscription.updated', + created: NOW_SECONDS, + data: { + object: { id: 'sub_1', status: 'active', plan: 'pro', version: 2 }, + }, +}; + +/** What the deepest user-reachable seam on the serve path was handed, per request. */ +interface Seen { + input: StitchInput; + headers: Record; +} + +/** + * A surface that captures the `StitchInput` the engine resolved and then answers 200 without any + * transport. `buildRequest` is the seam that sees the input (surface.ts:52-56); `execute` replaces + * the transport (surface.ts:118) so nothing leaves the process. + */ +function capturingSurface(seen: Seen[]) { + return { + // Kept as a literal type rather than widened to `Surface`: `stitch()`'s overloads narrow on + // the surface id, and a widened `kind` makes it match the `download` arm. + id: 'capture' as const, + buildRequest: ( + _cfg: ResolvedStitchConfig, + input: StitchInput, + base: AdapterRequest, + ): AdapterRequest => { + seen.push({ input, headers: { ...base.headers } }); + return base; + }, + execute: async () => ({ + status: 200, + headers: {}, + body: { ack: true }, + }), + }; +} + +async function main(): Promise { + heading( + 'C1 — raw bytes and arbitrary paths through a real `serve()` process', + ); + + // The provider's actual bytes. Real providers do NOT emit the key order or spacing your + // `JSON.stringify` would: this one emits `{"id":…,"type":…}` with a space after each colon, + // which is enough to change every byte of the MAC and nothing about the meaning. + const delivery = mintDelivery(EVENT, SECRET, NOW_SECONDS, (e) => + JSON.stringify(e, null, 0).replace(/":/g, '": '), + ); + note('provider bytes', delivery.raw.length + ' bytes'); + note('provider signature', delivery.signature.slice(0, 46) + '…'); + + const seen: Seen[] = []; + const registry = { + 'on-webhook': stitch({ + url: 'https://unused.test/never', + method: 'POST', + kind: capturingSurface(seen), + }), + }; + const handle = await serve(registry, { port: 0 }); + try { + const post = async (path: string): Promise => { + const res = await fetch(handle.url + path, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'stripe-signature': delivery.signature, + }, + body: Uint8Array.from(delivery.raw), + }); + await res.arrayBuffer(); + return res.status; + }; + + // ── (a) the arbitrary-path half ─────────────────────────────────────────────────────── + // Every path a provider dashboard would let you type. The route table is two entries and + // neither is yours (serve.ts:219-231). + const paths = [ + '/webhooks/stripe', + '/webhook', + '/', + '/stitch', + '/hooks/v1/billing', + ]; + const statuses: number[] = []; + for (const p of paths) statuses.push(await post(p)); + checkSeq( + '(a) POST status at ' + JSON.stringify(paths), + statuses, + [404, 404, 404, 404, 404], + ); + check( + '(a) any arbitrary path accepted a delivery', + statuses.some((s) => s < 400), + false, + ); + note( + '(a) → the route table is fixed', + 'GET / (list) and POST /stitch/:name (serve.ts:219-231). A webhook URL is neither', + ); + + // ── (b) the ONE route that reaches user code, with the same bytes ────────────────────── + const okStatus = await post('/stitch/on-webhook'); + check('(b) POST /stitch/on-webhook → status', okStatus, 200); + check('(b) requests that reached user code', seen.length, 1); + + // ── (c) what actually arrived: the PARSED object, and no raw form of it ──────────────── + const arrived = seen[0]?.input ?? {}; + const keys = Object.keys(arrived).sort(); + checkSeq('(c) StitchInput keys the engine resolved', keys, [ + 'created', + 'data', + 'id', + 'signal', + 'type', + ]); + check( + '(c) is any arrived value the raw string?', + Object.values(arrived).some( + (v) => typeof v === 'string' && v.includes('{'), + ), + false, + ); + check( + '(c) arrived body is a parsed object, deep-equal to JSON.parse(raw)', + JSON.stringify((arrived as Record)['data']) === + JSON.stringify( + (JSON.parse(delivery.raw.toString('utf8')) as BillingEvent) + .data, + ), + true, + ); + note( + '(c) → `parseInput` is `JSON.parse` (serve.ts:122-125)', + 'called on the string `readBody` decoded (serve.ts:248); the bytes are not retained anywhere', + ); + + // ── (d) the finding the capture does not predict: the SIGNATURE never arrives ────────── + // `createServeHandler` composes the run input from the body alone (serve.ts:268-271). + // `req.headers` is read for `content-length` and `accept` and for nothing else. + const outboundHeaders = seen[0]?.headers ?? {}; + check( + '(d) `stripe-signature` present anywhere in the resolved input', + JSON.stringify(arrived).toLowerCase().includes('stripe-signature'), + false, + ); + check( + '(d) `stripe-signature` present on the request user code sees', + Object.keys(outboundHeaders).some( + (h) => h.toLowerCase() === 'stripe-signature', + ), + false, + ); + note( + '(d) → there is nothing to verify AGAINST', + 'the inbound headers are dropped before user code; only the body becomes StitchInput', + ); + + // ── (e) the byte difference, demonstrated end to end ─────────────────────────────────── + // Given only the parsed object — which is all `serve` ever offers — the best a user can do + // is re-serialise. This is the exact failure the capture describes, measured. + const restringified = Buffer.from(JSON.stringify(arrived), 'utf8'); + const original = verifySignature( + delivery.raw, + delivery.signature, + SECRET, + NOW_SECONDS, + ); + const rebuilt = verifySignature( + restringified, + delivery.signature, + SECRET, + NOW_SECONDS, + ); + check('(e) verify against the RAW provider bytes', original.ok, true); + check( + '(e) verify against the re-stringified object', + rebuilt.ok, + false, + ); + check( + '(e) reason', + rebuilt.ok ? '(none)' : rebuilt.reason, + 'bad-signature', + ); + note( + '(e) bytes signed vs bytes rebuilt from the input', + `${delivery.raw.length} vs ${restringified.length} (the input also picked up the engine's \`signal\`)`, + ); + + // And the narrower version, with the `signal` key removed and nothing but whitespace + // between the two — so the failure cannot be blamed on the engine adding a field. + const parsedOnly = Buffer.from( + JSON.stringify(JSON.parse(delivery.raw.toString('utf8'))), + 'utf8', + ); + const whitespaceOnly = verifySignature( + parsedOnly, + delivery.signature, + SECRET, + NOW_SECONDS, + ); + check( + '(e) verify after a pure parse→stringify round-trip (whitespace only)', + whitespaceOnly.ok, + false, + ); + check( + '(e) the two payloads are logically identical', + JSON.stringify(JSON.parse(parsedOnly.toString('utf8'))) === + JSON.stringify(JSON.parse(delivery.raw.toString('utf8'))), + true, + ); + check( + '(e) bytes signed vs bytes after a pure round-trip', + `${delivery.raw.length} vs ${parsedOnly.length}`, + '162 vs 153', + ); + note( + '(e) → identical JSON, different bytes, dead signature', + `${delivery.raw.length - parsedOnly.length} bytes of whitespace is the entire difference`, + ); + + // ── (f) the non-JSON provider ───────────────────────────────────────────────────────── + // Slack signs an `application/x-www-form-urlencoded` body. `serve` rejects it at the door. + const form = await fetch(handle.url + '/stitch/on-webhook', { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + 'x-slack-signature': 'v0=deadbeef', + }, + body: 'payload=%7B%22type%22%3A%22url_verification%22%7D', + }); + const formBody = (await form.json()) as { error?: string }; + check('(f) form-encoded delivery → status', form.status, 400); + check('(f) → error', formBody.error, 'invalid JSON body'); + note( + '(f) → the body contract is JSON-only', + 'a form-encoded or protobuf provider cannot be received at all, signature or not', + ); + } finally { + await handle.close(); + } + + finish( + 'C1', + 'NO on both halves, and a third gap the capture missed. A real `serve()` process answered 404 to a signed Stripe-shaped POST at every one of /webhooks/stripe, /webhook, /, /stitch and /hooks/v1/billing — the route table is exactly `GET /` and `POST /stitch/:name` (serve.ts:219-231). On the one route that does reach user code the body has already been through `JSON.parse` (serve.ts:122-125,248): the deepest user-reachable seam (`Surface.buildRequest`) received a parsed object and no field carrying the raw string. THE THIRD GAP: the inbound headers are dropped entirely — `stripe-signature` appears nowhere in the resolved input, so there is nothing to verify against even if the bytes were recoverable. The byte difference is real and measured: the provider signed 162 bytes, a parse→stringify round-trip produces 153 logically identical bytes, and verification against them returns `bad-signature`. A form-encoded provider (Slack) is rejected 400 `invalid JSON body` before any of this', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c2-no-inbound-primitive.ts b/docs/scenarios/proofs/webhook-receipt/c2-no-inbound-primitive.ts new file mode 100644 index 00000000..4b08bf00 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c2-no-inbound-primitive.ts @@ -0,0 +1,199 @@ +// C2 — is there ANY inbound-signature / HMAC-verification primitive anywhere in the packages? +// +// A grep answers this, and a grep is not evidence — a missing export could be hiding behind a +// re-export chain or a differently-spelled name. So this script IMPORTS every public entry point +// of `stitchapi` and enumerates what is actually there at runtime, then checks the two things that +// LOOK like an answer and are not: +// +// • `xxh128` (hash.ts:110-113) is a hash, and it is unkeyed and non-cryptographic. The proof +// computes it with no secret at all, which is exactly why it cannot authenticate anything. +// • `awsSigV4` (aws-sigv4/src/index.ts:291) is the only HMAC in the repo, and it runs in the +// other direction: the script signs an OUTBOUND request with it and shows the strategy's only +// verb is `apply`, with no verify counterpart. Its HMAC key is imported with usages `['sign']` +// (aws-sigv4/src/index.ts:84) — it could not verify even if asked. +// +// And the near-miss the capture names: `idempotency` is about outbound WRITES. The script measures +// the header it puts on the wire, which is the opposite end of the duplicate problem. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c2-no-inbound-primitive.ts +import { awsSigV4 } from '../../../../packages/aws-sigv4/src/index'; +import * as auth from '../../../../packages/core/src/auth'; +import * as cache from '../../../../packages/core/src/cache'; +import { xxh128 } from '../../../../packages/core/src/hash'; +import * as core from '../../../../packages/core/src/index'; +import { stitch } from '../../../../packages/core/src/index'; +import * as pipe from '../../../../packages/core/src/pipe'; +import * as serveMod from '../../../../packages/core/src/serve'; +import * as testing from '../../../../packages/core/src/testing'; +import type { + Adapter, + AdapterRequest, + AuthContext, +} from '../../../../packages/core/src/types'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** Words a verification primitive would have to be spelled with, in any reasonable naming scheme. */ +const INBOUND_WORDS = + /verif|hmac|signature|webhook|constanttime|timingsafe|digest.*compare|receive|inbound/i; + +async function main(): Promise { + heading('C2 — searching the runtime for an inbound-signature primitive'); + + // ── (a) enumerate every public entry point's exports and filter for the words ───────────── + const entries: Array<[string, Record]> = [ + ['stitchapi', core as unknown as Record], + ['stitchapi/auth', auth as unknown as Record], + ['stitchapi/cache', cache as unknown as Record], + ['stitchapi/pipe', pipe as unknown as Record], + ['stitchapi/serve', serveMod as unknown as Record], + ['stitchapi/testing', testing as unknown as Record], + ]; + let total = 0; + const hits: string[] = []; + for (const [name, mod] of entries) { + const names = Object.keys(mod); + total += names.length; + for (const n of names) + if (INBOUND_WORDS.test(n)) hits.push(`${name}#${n}`); + } + note('(a) runtime exports enumerated across 6 entry points', total); + checkSeq( + '(a) exports matching /verif|hmac|signature|webhook|…/', + hits.sort(), + [ + 'stitchapi/testing#verifyAdapterContract', + 'stitchapi/testing#verifyFingerprintContract', + 'stitchapi/testing#verifySinkContract', + 'stitchapi/testing#verifyStoreContract', + ], + ); + note( + '(a) → the four hits are conformance suites for BYO plugins', + 'they verify that a store/adapter/sink/fingerprinter obeys its contract — nothing to do with a signature', + ); + + // ── (b) the AuthStrategy surface: one verb, and it points outward ───────────────────────── + // `auth` is the only place a credential meets a request, so if inbound verification lived + // anywhere it would live here. The strategy interface has no verify arm. + { + const strategy = auth.bearer(() => 'tok'); + checkSeq('(b) AuthStrategy keys', Object.keys(strategy).sort(), [ + 'apply', + 'name', + 'scheme', + ]); + note( + '(b) `scheme` is a DESCRIPTION, not a verb', + 'a declarative wire shape for OpenAPI export (types.ts:1234-1236) — it inspects nothing', + ); + check( + '(b) does any auth export mention verifying an inbound request?', + Object.keys(auth).some((n) => /verif|inbound|receive/i.test(n)), + false, + ); + note( + '(b) → `apply(req, ctx)` mutates an OUTGOING request', + 'there is no `AuthStrategy` member that inspects an incoming one', + ); + } + + // ── (c) the only HMAC in the repo, run — and its direction ──────────────────────────────── + { + const sig = awsSigV4({ + accessKeyId: 'AKIDEXAMPLE', + secretAccessKey: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY', + region: 'us-east-1', + service: 'execute-api', + }); + const req: AdapterRequest = { + url: 'https://example.test/v1/thing', + method: 'GET', + headers: {}, + }; + const store = core.memoryStore(); + const ctx: AuthContext = { + store, + vault: store, + emit: () => undefined, + }; + await sig.apply(req, ctx); + check( + '(c) awsSigV4 produced an Authorization header on the OUTBOUND request', + (req.headers['authorization'] ?? '').startsWith('AWS4-HMAC-SHA256'), + true, + ); + checkSeq('(c) awsSigV4 strategy keys', Object.keys(sig).sort(), [ + 'apply', + 'name', + ]); + await store.close?.(); + note( + '(c) → the one HMAC in the repo signs, and only signs', + 'its key is imported with usages ["sign"] (aws-sigv4/src/index.ts:84) — it cannot verify', + ); + } + + // ── (d) the hash that is NOT a signature ────────────────────────────────────────────────── + // Someone reaching for a "compare a digest" primitive will find `xxh128` first. It takes no + // key, which is the whole story: an attacker can compute it as easily as you can. + { + const payload = '{"id":"evt_1","type":"charge.succeeded"}'; + const a = xxh128(payload); + const b = xxh128(payload); + check( + '(d) xxh128 arity (a keyed MAC would take 2 args)', + xxh128.length, + 1, + ); + check( + '(d) xxh128 is deterministic with NO secret involved', + a === b, + true, + ); + check('(d) digest width in hex chars', a.length, 32); + note( + '(d) → hash.ts:110 self-describes as "non-cryptographic"', + 'unkeyed, so it authenticates nobody; it exists for cache keys and schema fingerprints', + ); + } + + // ── (e) the near-miss: `idempotency` is an OUTBOUND write concern ────────────────────────── + // Same word, opposite end of the pipe. It puts a key ON a request you send so a SERVER can + // collapse your duplicate. It has no inbound counterpart. + { + let sentHeaders: Record = {}; + const capture: Adapter = async (req) => { + sentHeaders = { ...req.headers }; + return { status: 200, headers: {}, body: { ok: true } }; + }; + const write = stitch({ + url: 'https://api.test/v1/charges', + method: 'POST', + adapter: capture, + retry: { attempts: 2 }, + idempotency: { + keyOf: (input) => String((input.body as { ref?: string }).ref), + }, + }); + const r = await write.safe({ + body: { ref: 'evt_1PqR', amount: 100 }, + }); + check('(e) the write succeeded', r.ok, true); + check( + '(e) `idempotency` set a header on the OUTBOUND request', + sentHeaders['Idempotency-Key'] ?? sentHeaders['idempotency-key'], + 'evt_1PqR', + ); + note( + '(e) → the key rides OUT, for a server to dedup on', + 'nothing in the config dedups something arriving IN; that ledger is C5', + ); + } + + finish( + 'C2', + 'NO — nothing in any package verifies an inbound signature, measured at runtime rather than by grep. Enumerating every public entry point turned up exactly four exports matching /verif|hmac|signature|webhook|…/ and all four are BYO-plugin conformance suites (`verifyStoreContract`, `verifyAdapterContract`, `verifySinkContract`, `verifyFingerprintContract`). `AuthStrategy` has exactly three keys, `{apply, name, scheme}` — `apply(req, ctx)` mutates an OUTGOING request, `scheme` is a declarative wire description, and there is no inbound arm. The only HMAC in the repo is `@stitchapi/aws-sigv4`, which ran here and produced an `AWS4-HMAC-SHA256` Authorization header on an outbound request; its subtle key is imported with usages ["sign"] (aws-sigv4/src/index.ts:84), so it structurally cannot verify. The nearest-looking primitive, `xxh128` (hash.ts:110-113), is UNKEYED and self-described as non-cryptographic — measured taking 1 argument and returning the same 32-char digest with no secret anywhere, which is precisely why it authenticates nobody. And `idempotency`, which wears the same word as the dedup problem, was measured putting `Idempotency-Key: evt_1PqR` on an OUTBOUND POST — the opposite end of the pipe', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c3-serve-seams.ts b/docs/scenarios/proofs/webhook-receipt/c3-serve-seams.ts new file mode 100644 index 00000000..48516ff5 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c3-serve-seams.ts @@ -0,0 +1,296 @@ +// C3 — can a signature be verified THROUGH `serve` at all? Via a surface, a hook, `ServeBodyOptions`, +// anything? Or must the user bring their own server? +// +// C1 established that the bytes and the headers are gone by the time user code runs. C3 asks the +// follow-up an engineer actually asks: fine, but is there a seam EARLIER? The serve path has three +// candidate places — the `ServeOptions` envelope, the exported `createServeHandler` mount point, +// and the stitch's own hooks — and this script drives all three against a real server. +// +// The finding with teeth is (e). `createServeHandler` really is a mount seam, and the obvious way +// to use it is exactly wrong: to verify a signature you must read the request stream, and +// `readBody` (serve.ts:90-120) then attaches its `data`/`end` listeners to a stream that has +// already ended. The request HANGS. Measured with a 200ms deadline, because "it hangs" is not a +// thing you can assert without one. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c3-serve-seams.ts +import { stitch } from '../../../../packages/core/src/index'; +import { + MAX_REQUEST_BODY_BYTES, + createServeHandler, + serve, +} from '../../../../packages/core/src/serve'; +import * as serveMod from '../../../../packages/core/src/serve'; +import type { Adapter, HookContext } from '../../../../packages/core/src/types'; +import { type BillingEvent, mintDelivery } from './fake-billing'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { verifySignature } from './stripe-sig'; + +import { createServer } from 'node:http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; + +const SECRET = 'whsec_test_2f9d1c4b'; +const NOW_SECONDS = 1_800_000_000; +const EVENT: BillingEvent = { + id: 'evt_1PqR', + type: 'customer.subscription.updated', + created: NOW_SECONDS, + data: { + object: { id: 'sub_1', status: 'active', plan: 'pro', version: 2 }, + }, +}; + +async function main(): Promise { + heading('C3 — every seam on the serve path, driven'); + + const delivery = mintDelivery(EVENT, SECRET, NOW_SECONDS); + + // ── (a) what `stitchapi/serve` actually offers ──────────────────────────────────────────── + checkSeq( + '(a) `stitchapi/serve` runtime exports', + Object.keys(serveMod).sort(), + ['MAX_REQUEST_BODY_BYTES', 'createServeHandler', 'serve'], + ); + check( + '(a) default body cap (bytes)', + MAX_REQUEST_BODY_BYTES, + 2 * 1024 * 1024, + ); + note( + '(a) → three exports: a server, a mountable handler, and a number', + 'no verifier, no middleware chain, no per-route registration', + ); + + // ── (b) the `ServeBodyOptions` seam, driven: it is a byte cap and only a byte cap ────────── + { + const hookCalls: HookContext[] = []; + const echo: Adapter = async () => ({ + status: 200, + headers: {}, + body: { ack: true }, + }); + const registry = { + 'on-webhook': stitch({ + url: 'https://unused.test/never', + method: 'POST', + adapter: echo, + hooks: { + onRequest: (ctx) => { + hookCalls.push(ctx); + }, + }, + }), + }; + const handle = await serve(registry, { port: 0, body: { max: 64 } }); + try { + const res = await fetch(handle.url + '/stitch/on-webhook', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'stripe-signature': delivery.signature, + }, + body: Uint8Array.from(delivery.raw), + }); + const parsed = (await res.json()) as { error?: string }; + check( + '(b) `body.max: 64` against a 130-byte delivery → status', + res.status, + 413, + ); + check( + '(b) → the rejection is about SIZE, not authenticity', + parsed.error?.includes('exceeds') ?? false, + true, + ); + + // Now within the cap: the delivery runs the stitch with no credential of any kind. + const ok = await fetch(handle.url + '/stitch/on-webhook', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{"id":"evt_forged"}', + }); + await ok.arrayBuffer(); + check('(b) an UNSIGNED, forged body → status', ok.status, 200); + check('(b) → it ran the stitch anyway', hookCalls.length, 1); + note( + '(b) → `serve` is unauthenticated by design (serve.ts:28,59)', + 'the only inbound control in `ServeOptions` is `body.max`; there is no credential check to fail', + ); + + // ── (c) the stitch-side hooks point OUTWARD ─────────────────────────────────────── + const ctx = hookCalls[0]; + check( + '(c) `hooks.onRequest` saw a request whose url is the OUTBOUND one', + ctx?.req?.url, + 'https://unused.test/never', + ); + check( + '(c) does that request carry the inbound `stripe-signature`?', + Object.keys(ctx?.req?.headers ?? {}).some( + (h) => h.toLowerCase() === 'stripe-signature', + ), + false, + ); + checkSeq( + '(c) `HookContext` keys available to a hook', + Object.keys(ctx ?? {}).sort(), + ['attempt', 'name', 'req'], + ); + note( + '(c) → `Hooks` is {onRequest,onResponse,onError,onRetry} (types.ts:1285-1290)', + 'all four describe the call the stitch is MAKING; none describes the request that arrived', + ); + } finally { + await handle.close(); + } + } + + // ── (d) the real seam: `createServeHandler` mounted at a user path, in a user server ─────── + // This part WORKS, and it is worth saying so: the handler is framework-free and exported + // exactly so it can be mounted (serve.ts:203-206). A user can own the route. + { + const seenInput: unknown[] = []; + const registry = { + 'on-webhook': stitch({ + url: 'https://unused.test/never', + method: 'POST', + adapter: (async (req) => { + seenInput.push(req.body); + return { status: 200, headers: {}, body: { ack: true } }; + }) as Adapter, + }), + }; + const handler = createServeHandler(registry); + const server = createServer((req, res) => { + // The user's own routing table — an arbitrary path, which is the thing `serve` cannot do. + if (req.url?.startsWith('/webhooks/stripe')) { + req.url = '/stitch/on-webhook'; + void handler(req, res); + return; + } + res.writeHead(404).end(); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const addr = server.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + try { + const res = await fetch( + `http://127.0.0.1:${port}/webhooks/stripe`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ body: { forwarded: true } }), + }, + ); + await res.arrayBuffer(); + check( + '(d) mounted at an arbitrary user path → status', + res.status, + 200, + ); + check('(d) the stitch ran', seenInput.length, 1); + note( + '(d) → `createServeHandler` genuinely is a mount seam', + 'the ROUTE problem from C1 is solvable by owning the server. The BYTES problem is not — see (e)', + ); + } finally { + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + } + } + + // ── (e) …and mounting it behind a verifier does not work, because verifying consumes ─────── + // To check a signature you must have the bytes. To have the bytes you must read the stream. + // `readBody` then waits on a stream that has already ended. + { + const registry = { + 'on-webhook': stitch({ + url: 'https://unused.test/never', + method: 'POST', + adapter: (async () => ({ + status: 200, + headers: {}, + body: { ack: true }, + })) as Adapter, + }), + }; + const handler = createServeHandler(registry); + let verified = false; + let handlerSettled = false; + const server = createServer( + (req: IncomingMessage, res: ServerResponse) => { + void (async () => { + // 1. Read the raw bytes — the ONLY way to verify. + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + const raw = Buffer.concat(chunks); + verified = verifySignature( + raw, + req.headers['stripe-signature'] as string | undefined, + SECRET, + NOW_SECONDS, + ).ok; + + // 2. Hand the (now-drained) request to the handler. + req.url = '/stitch/on-webhook'; + const done = handler(req, res).then(() => { + handlerSettled = true; + }); + const deadline = new Promise((r) => + setTimeout(r, 200).unref(), + ); + await Promise.race([done, deadline]); + if (!handlerSettled && !res.headersSent) { + res.writeHead(504, { + 'content-type': 'application/json', + }); + res.end( + JSON.stringify({ error: 'handler never settled' }), + ); + } + })(); + }, + ); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const addr = server.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + try { + const res = await fetch( + `http://127.0.0.1:${port}/webhooks/stripe`, + { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'stripe-signature': delivery.signature, + }, + body: Uint8Array.from(delivery.raw), + }, + ); + await res.arrayBuffer(); + check( + '(e) the signature verified against the raw bytes', + verified, + true, + ); + check( + '(e) …and then `createServeHandler` settled within 200ms', + handlerSettled, + false, + ); + check('(e) what the client got', res.status, 504); + note( + '(e) → `readBody` (serve.ts:90-120) waits for `end` on an ended stream', + 'verifying and delegating are mutually exclusive unless you re-feed a synthetic stream', + ); + } finally { + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + } + } + + finish( + 'C3', + 'NO seam exists on the serve path, and the mount point that does exist is defeated by the act of verifying. `stitchapi/serve` exports exactly three things — `serve`, `createServeHandler`, `MAX_REQUEST_BODY_BYTES` — and `ServeOptions` carries one inbound control, a byte cap: `{body:{max:64}}` answered a 130-byte signed delivery with 413 "exceeds", while an UNSIGNED forged body under the cap ran the stitch and returned 200, because `serve` is unauthenticated by design (serve.ts:28,59). The stitch-side hooks point the other way: `hooks.onRequest` fired with `req.url` = the OUTBOUND url and no `stripe-signature` header, and `HookContext` offered exactly {attempt,name,req}. `createServeHandler` IS a genuine mount seam — mounted in a plain `node:http` server it served an arbitrary path `/webhooks/stripe` and ran the stitch, 200 — so the ROUTE half of C1 is fixable by owning the server. The BYTES half is not: reading the stream to verify (which succeeded) left `readBody` (serve.ts:90-120) waiting on `end` from an already-ended stream, and the handler had not settled after 200ms — the client got a 504 from the test deadline rather than a response. You must bring your own server, and once you have, the `serve` front door adds nothing to the webhook path', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c4-out-of-order.ts b/docs/scenarios/proofs/webhook-receipt/c4-out-of-order.ts new file mode 100644 index 00000000..c1d6a2f8 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c4-out-of-order.ts @@ -0,0 +1,287 @@ +// C4 — the reaction half. Two events arrive REVERSED. Does fetch-on-receipt genuinely make +// ordering moot? +// +// This is the half that IS StitchAPI's job, so it gets the same scrutiny as the refusals above. +// The capture calls fetch-on-receipt "the standard fix" and "widely recommended", and it is — but +// "makes ordering moot" is two claims wearing one sentence, and only one of them survives: +// +// PAYLOAD order stops mattering. The answer to `GET /v1/subscriptions/sub_1` does not depend on +// which event prompted the question, so a reversed pair converges. Measured in (b). +// +// WRITE order still matters. Two handlers running concurrently each fetch a snapshot, and the +// later-landing write can carry the OLDER snapshot. Fetch-on-receipt does not fix that; a +// version guard does. Measured in (c), and it is the finding the capture does not predict. +// +// The genuinely-StitchAPI part is (e): the fetch-on-receipt call needs "its own auth, retry and +// rate-limit budget" (the capture's words), and that is config on the stitch rather than code in +// the handler. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c4-out-of-order.ts +import { bearer } from '../../../../packages/core/src/auth'; +import { stitch } from '../../../../packages/core/src/index'; +import { manualClock } from '../../../../packages/core/src/testing'; +import { + type BillingEvent, + FakeBilling, + type Subscription, +} from './fake-billing'; +import { check, checkSeq, finish, heading, note } from './harness'; + +const NOW = 1_800_000_000; + +/** The provider's two events, in the order they were CREATED. */ +const CREATED_EVENT: BillingEvent = { + id: 'evt_created', + type: 'customer.subscription.created', + created: NOW, + data: { + object: { id: 'sub_1', status: 'trialing', plan: 'free', version: 1 }, + }, +}; +const UPDATED_EVENT: BillingEvent = { + id: 'evt_updated', + type: 'customer.subscription.updated', + created: NOW + 30, + data: { + object: { id: 'sub_1', status: 'active', plan: 'pro', version: 2 }, + }, +}; + +/** The order they actually ARRIVE — reversed, which providers do not promise not to do. */ +const ARRIVAL = [UPDATED_EVENT, CREATED_EVENT]; + +/** The local read model the handler maintains. */ +interface Local { + status: string; + plan: string; + version: number; +} + +async function main(): Promise { + heading('C4 — reversed delivery, with and without fetch-on-receipt'); + + // The server's truth after BOTH changes have happened upstream. This is fixed before either + // delivery arrives — which is the whole premise of fetch-on-receipt. + const truth: Subscription = { + id: 'sub_1', + status: 'active', + plan: 'pro', + version: 2, + }; + + // ── (a) acting on the PAYLOAD, in arrival order ─────────────────────────────────────────── + { + let local: Local = { status: 'none', plan: 'none', version: 0 }; + const applied: string[] = []; + for (const ev of ARRIVAL) { + const o = ev.data.object; + local = { status: o.status, plan: o.plan, version: o.version }; + applied.push(`${o.status}/${o.plan}`); + } + checkSeq('(a) states applied, in arrival order', applied, [ + 'active/pro', + 'trialing/free', + ]); + check( + '(a) final local state', + `${local.status}/${local.plan}`, + 'trialing/free', + ); + check( + '(a) does it match the server?', + `${local.status}/${local.plan}` === `${truth.status}/${truth.plan}`, + false, + ); + note( + '(a) → the customer is on `pro` and the app believes `free`', + 'a self-inflicted downgrade, and nothing errored — the last payload simply won', + ); + } + + // ── (b) fetch-on-receipt through a stitch ───────────────────────────────────────────────── + { + const api = new FakeBilling(); + api.setSubscription(truth); + const fetchSub = stitch({ + baseUrl: FakeBilling.baseUrl, + path: '/v1/subscriptions/{id}', + method: 'GET', + adapter: api.adapter(), + }); + + let local: Local = { status: 'none', plan: 'none', version: 0 }; + const applied: string[] = []; + for (const ev of ARRIVAL) { + // The event is a HINT: all we take from it is the id. + const id = ev.data.object.id; + const current = (await fetchSub({ + params: { id }, + })) as Subscription; + local = { + status: current.status, + plan: current.plan, + version: current.version, + }; + applied.push(`${current.status}/${current.plan}`); + } + checkSeq('(b) states applied, in arrival order', applied, [ + 'active/pro', + 'active/pro', + ]); + check( + '(b) final local state', + `${local.status}/${local.plan}`, + 'active/pro', + ); + check( + '(b) does it match the server?', + `${local.status}/${local.plan}` === `${truth.status}/${truth.plan}`, + true, + ); + checkSeq('(b) the cost, in API calls', api.requests, [ + '/v1/subscriptions/sub_1', + '/v1/subscriptions/sub_1', + ]); + note( + '(b) → PAYLOAD order is genuinely moot', + 'both handlers computed the same answer, so which arrived first stopped mattering', + ); + } + + // ── (c) …and WRITE order is not ─────────────────────────────────────────────────────────── + // Two deliveries handled CONCURRENTLY. Both fetch. Between the two fetches the subscription + // changes again upstream — an ordinary thing to happen. Handler A holds the older snapshot and + // its write lands last. + { + const api = new FakeBilling(); + api.setSubscription({ ...truth }); + const fetchSub = stitch({ + baseUrl: FakeBilling.baseUrl, + path: '/v1/subscriptions/{id}', + method: 'GET', + adapter: api.adapter(), + }); + + // Handler A fetches first and sees v2. + const snapshotA = (await fetchSub({ + params: { id: 'sub_1' }, + })) as Subscription; + // A real upstream change lands between the two fetches. + api.setSubscription({ + id: 'sub_1', + status: 'canceled', + plan: 'pro', + version: 3, + }); + // Handler B fetches and sees v3. + const snapshotB = (await fetchSub({ + params: { id: 'sub_1' }, + })) as Subscription; + + checkSeq( + '(c) the two snapshots the concurrent handlers hold', + [snapshotA.version, snapshotB.version], + [2, 3], + ); + + // The writes land in the order the two handlers happen to finish, which is not the order + // they fetched in. Naive last-write-wins: + let naive: Local = { status: 'none', plan: 'none', version: 0 }; + for (const s of [snapshotB, snapshotA]) + naive = { status: s.status, plan: s.plan, version: s.version }; + check('(c) naive last-write-wins → local version', naive.version, 2); + check('(c) → local status', naive.status, 'active'); + check( + '(c) does it match the server (v3, canceled)?', + naive.status === 'canceled', + false, + ); + + // The guard that actually closes it: reject a write carrying an older version. + let guarded: Local = { status: 'none', plan: 'none', version: 0 }; + const rejected: number[] = []; + for (const s of [snapshotB, snapshotA]) { + if (s.version <= guarded.version) { + rejected.push(s.version); + continue; + } + guarded = { status: s.status, plan: s.plan, version: s.version }; + } + check('(c) version-guarded → local version', guarded.version, 3); + check('(c) → local status', guarded.status, 'canceled'); + checkSeq('(c) writes rejected as stale', rejected, [2]); + note( + '(c) → fetch-on-receipt is necessary, not sufficient', + "it removes the PAYLOAD ordering problem and leaves the WRITE ordering problem; the guard is user code, in the user's own database", + ); + } + + // ── (d) the ordering that fetch-on-receipt cannot see at all ────────────────────────────── + // A `.deleted` event whose subject no longer exists. The fetch 404s, and "gone" is a legitimate + // answer only because the event said so — so the payload is not purely a hint after all. + { + const api = new FakeBilling(); + // Subscription already deleted upstream. + const fetchSub = stitch({ + baseUrl: FakeBilling.baseUrl, + path: '/v1/subscriptions/{id}', + method: 'GET', + adapter: api.adapter(), + }); + const r = await fetchSub.safe({ params: { id: 'sub_1' } }); + check('(d) fetch-on-receipt for a deleted subject → ok', r.ok, false); + check('(d) → status', r.error?.status, 404); + note( + '(d) → a 404 is ambiguous', + '"deleted" and "never existed" look identical; the event TYPE is the only thing that disambiguates them', + ); + } + + // ── (e) what StitchAPI actually buys on this call ───────────────────────────────────────── + // The capture: the fetch "needs its own auth, retry and rate-limit budget". That is config. + { + const api = new FakeBilling(); + api.setSubscription(truth); + api.failNext('/v1/subscriptions/sub_1', 2, 503); + const clock = manualClock(); + const fetchSub = stitch({ + baseUrl: FakeBilling.baseUrl, + path: '/v1/subscriptions/{id}', + method: 'GET', + adapter: api.adapter(), + auth: bearer(() => 'sk_live_xyz'), + retry: { attempts: 3, backoff: { curve: 'fixed', base: 1_000 } }, + timeout: { total: '10s' }, + clock, + }); + + const pending = fetchSub.safe({ params: { id: 'sub_1' } }); + await clock.advance(0); + await clock.advance(1_000); + await clock.advance(1_000); + const r = await pending; + + check( + '(e) the fetch-on-receipt call succeeded through two 503s', + r.ok, + true, + ); + check( + '(e) → converged on the server truth', + `${(r.data as Subscription | undefined)?.status}/${(r.data as Subscription | undefined)?.plan}`, + 'active/pro', + ); + check('(e) HTTP attempts the provider saw', api.requests.length, 3); + note( + '(e) → the retry, the backoff, the auth and the deadline are config, not handler code', + 'this is the half of the scenario the library is for, and it is the whole half', + ); + } + + finish( + 'C4', + 'PARTLY — and the part that fails is not the part the capture warns about. Acting on payload order with a reversed pair applied ["active/pro","trialing/free"] and left the app believing `trialing/free` while the server said `active/pro`: a silent self-inflicted downgrade. Replacing the payload with a fetch-on-receipt stitch applied ["active/pro","active/pro"] and converged exactly, at a measured cost of 2 API calls for 2 events — so PAYLOAD order is genuinely moot. WRITE order is not: two concurrent handlers holding snapshots v2 and v3 with naive last-write-wins landed on version 2 / `active` while the server said version 3 / `canceled`, and only a version guard (which rejected exactly [2]) recovered version 3. And a `.deleted` event is a case fetch-on-receipt cannot answer at all — the fetch returned 404, which is indistinguishable from "never existed", so the event type is still load-bearing. What IS unambiguously the library\'s job is the call itself: with `auth`, `retry: {attempts: 3}`, a fixed 1s backoff and `timeout: {total: "10s"}`, the fetch survived two 503s in 3 measured attempts on an injected `manualClock` and converged, with no retry loop in the handler', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c5-dedup-ledger.ts b/docs/scenarios/proofs/webhook-receipt/c5-dedup-ledger.ts new file mode 100644 index 00000000..8f80d109 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c5-dedup-ledger.ts @@ -0,0 +1,307 @@ +// C5 — can `StitchStore` serve as the dedup ledger from USER code? Durable, TTL beyond the retry +// window? Measure a duplicate delivery being skipped, and the TTL boundary. +// +// The capture inherits an open question from scenario 4 — "the store is engine state, so whether a +// user can borrow it cleanly is open". That framing is backwards, and this script measures why: +// `store` is a config key the USER supplies (types.ts:1583-1584), and `memoryStore` is a public +// export (index.ts:62). Nothing is borrowed. You construct the store, use it as your ledger, and +// hand the same instance to the engine — one Redis connection serving both. +// +// Two findings the capture does not predict: +// +// • `get`-then-`set` is a RACE, and the store already ships the fix. Two concurrent handlers for +// one event id both read `undefined` and both process. `increment` is atomic and returns 1 and +// 2, so exactly one processes. Measured in (b). +// • `memoryStore`'s TTL reads `Date.now()` (store.ts:16,45,54 via util.ts:4) and ignores an +// injected clock entirely, so the boundary of a 3-DAY dedup window is not testable against it +// at all. Measured in (d), where a `manualClock` advanced four virtual days changes nothing. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c5-dedup-ledger.ts +import { memoryStore, stitch } from '../../../../packages/core/src/index'; +import { + assertConformance, + manualClock, + verifyStoreContract, +} from '../../../../packages/core/src/testing'; +import type { Adapter, StitchStore } from '../../../../packages/core/src/types'; +import { clockStore } from './clock-store'; +import { check, checkSeq, finish, heading, note } from './harness'; + +/** Stripe retries a failed delivery for up to 3 days. The ledger has to outlive that. */ +const RETRY_WINDOW_MS = 3 * 24 * 60 * 60 * 1000; +/** …so the TTL is set beyond it. This is the number the whole claim turns on. */ +const DEDUP_TTL_MS = 4 * 24 * 60 * 60 * 1000; + +/** The ledger, written the way a handler would write it. Racy on purpose — see `claimAtomic`. */ +async function claimNaive( + store: StitchStore, + eventId: string, +): Promise { + if ((await store.get(`webhook:${eventId}`)) !== undefined) return false; + await store.set(`webhook:${eventId}`, 1, DEDUP_TTL_MS); + return true; +} + +/** The same ledger on the store's atomic verb. First caller gets 1; everyone else loses. */ +async function claimAtomic( + store: StitchStore, + eventId: string, +): Promise { + return (await store.increment(`webhook:${eventId}`, DEDUP_TTL_MS)) === 1; +} + +async function main(): Promise { + heading('C5 — StitchStore as a dedup ledger, from user code'); + + // ── (a) the ownership direction, and a duplicate actually skipped ────────────────────────── + // The user makes the store. The engine is handed the same one. Nothing is extracted from + // anything. + { + // Wrap the store so every key either side touches is recorded — the evidence that ONE + // instance is carrying both the dedup ledger and the engine's own state. + const inner = memoryStore(); + const touched: string[] = []; + const store: StitchStore = { + get: (k) => { + touched.push(k); + return inner.get(k); + }, + set: (k, v, ttl) => { + touched.push(k); + return inner.set(k, v, ttl); + }, + increment: (k, ttl) => { + touched.push(k); + return inner.increment(k, ttl); + }, + close: () => inner.close?.() ?? Promise.resolve(), + }; + let sideEffects = 0; + const provision: Adapter = async () => { + sideEffects++; + return { status: 200, headers: {}, body: { ok: true } }; + }; + const act = stitch({ + url: 'https://api.billing.test/v1/provision', + method: 'POST', + adapter: provision, + store, // ← the SAME instance the ledger uses + // A rate gives the engine a reason to write to the store, so "shared" is measurable + // rather than asserted. + throttle: { rate: '100/s' }, + }); + + // Delivery, then the provider's at-least-once retry of the identical event. + const outcomes: string[] = []; + for (const attempt of ['first', 'retry', 'retry']) { + if (await claimNaive(store, 'evt_1PqR')) { + await act({ body: { sub: 'sub_1' } }); + outcomes.push(`${attempt}:processed`); + } else { + outcomes.push(`${attempt}:skipped`); + } + } + checkSeq('(a) three deliveries of one event id', outcomes, [ + 'first:processed', + 'retry:skipped', + 'retry:skipped', + ]); + check('(a) side effects performed', sideEffects, 1); + check( + '(a) the ledger wrote its own keys to that store', + touched.some((k) => k.startsWith('webhook:')), + true, + ); + check( + '(a) …and the ENGINE wrote its throttle counter to the same instance', + touched.some((k) => k.startsWith('rl:')), + true, + ); + note( + '(a) → `store` is a config key the USER supplies (types.ts:1583-1584)', + '`memoryStore()` is a public export (index.ts:62); the ledger owns it and lends it to the engine, not the other way round', + ); + await store.close?.(); + } + + // ── (b) get-then-set is a race; `increment` is not ───────────────────────────────────────── + // Two workers pulling the same event off the wire at once. This is the normal case at any + // scale above one process, and the naive ledger double-processes. + { + const store = memoryStore(); + const naive = await Promise.all([ + claimNaive(store, 'evt_race'), + claimNaive(store, 'evt_race'), + claimNaive(store, 'evt_race'), + ]); + checkSeq('(b) `get`-then-`set`, 3 concurrent claims', naive, [ + true, + true, + true, + ]); + check( + '(b) → how many handlers would have charged the card?', + naive.filter(Boolean).length, + 3, + ); + + const atomic = await Promise.all([ + claimAtomic(store, 'evt_race_atomic'), + claimAtomic(store, 'evt_race_atomic'), + claimAtomic(store, 'evt_race_atomic'), + ]); + checkSeq('(b) `increment`, 3 concurrent claims', atomic, [ + true, + false, + false, + ]); + check( + '(b) → how many handlers charge the card?', + atomic.filter(Boolean).length, + 1, + ); + note( + '(b) → the atomic verb is already in the contract', + '`increment(key, ttl)` (types.ts:1969-1970) exists for the throttle counter and is exactly the dedup primitive', + ); + await store.close?.(); + } + + // ── (c) the TTL boundary, on a clock-backed store ────────────────────────────────────────── + // A 4-day TTL against a 3-day retry window. The boundary is `expires > now` (store.ts:16), so + // the entry is live up to and including the last tick before expiry and gone on it. + { + const clock = manualClock(); + const store = clockStore(clock); + await claimAtomic(store, 'evt_ttl'); + + const probe: string[] = []; + // t = retry window: a Stripe retry at the far edge of its window. + await clock.advance(RETRY_WINDOW_MS); + probe.push( + (await claimAtomic(store, 'evt_ttl')) ? 'processed' : 'skipped', + ); + // t = ttl - 1ms. + await clock.advance(DEDUP_TTL_MS - RETRY_WINDOW_MS - 1); + probe.push( + (await claimAtomic(store, 'evt_ttl')) ? 'processed' : 'skipped', + ); + // t = ttl exactly — `expires > now` is now false. + await clock.advance(1); + probe.push( + (await claimAtomic(store, 'evt_ttl')) ? 'processed' : 'skipped', + ); + + checkSeq('(c) claims at t = 3d, ttl-1ms, ttl', probe, [ + 'skipped', + 'skipped', + 'processed', + ]); + check( + '(c) does the ledger cover the whole retry window?', + probe[0] === 'skipped' && probe[1] === 'skipped', + true, + ); + note( + '(c) → the boundary is `expires > now`, exclusive (store.ts:16)', + 'a TTL equal to the retry window would let the last retry through; 4 days over 3 leaves a day of margin', + ); + await store.close?.(); + } + + // ── (d) …and that boundary is NOT testable against `memoryStore` ────────────────────────── + // The trap scenario 6 measured, in the shape this scenario hits it: `memoryStore` takes no + // clock and reads `Date.now()`, so no amount of virtual time expires anything. + { + check( + '(d) `memoryStore` arity (a clock-aware store would take one)', + memoryStore.length, + 0, + ); + const clock = manualClock(); + const store = memoryStore(); + await claimAtomic(store, 'evt_virtual'); + await clock.advance(DEDUP_TTL_MS + 24 * 60 * 60 * 1000); // four days, then some + const afterFourVirtualDays = await claimAtomic(store, 'evt_virtual'); + check( + '(d) claim again after FOUR virtual days → processed?', + afterFourVirtualDays, + false, + ); + check( + '(d) virtual ms advanced', + clock.now(), + DEDUP_TTL_MS + 86_400_000, + ); + note( + '(d) → the injected clock is ignored (store.ts:16,45,54 via util.ts:4)', + 'the key is still live because `Date.now()` has not moved. A 3-day TTL cannot be boundary-tested without a clock-backed store', + ); + await store.close?.(); + } + + // ── (e) durability: what `memoryStore` costs you, and that the swap is contract-tested ───── + { + // The in-memory default conflates "release the handle" with "drop the data". + const mem = memoryStore(); + await claimAtomic(mem, 'evt_restart'); + await mem.close?.(); + const survived = !(await claimAtomic(mem, 'evt_restart')); + check( + '(e) memoryStore: did the ledger survive `close()`?', + survived, + false, + ); + note( + '(e) → `memoryStore.close()` is `data.clear()` (store.ts:59-61)', + "a deploy inside Stripe's 3-day window re-processes every event still being retried", + ); + + // A durable store is the same interface with the data outside the handle. Two successive + // stores over one backing map = the process restarted. + const clock = manualClock(); + const disk = new Map(); + const before = clockStore(clock, disk); + await claimAtomic(before, 'evt_restart'); + await before.close?.(); + const after = clockStore(clock, disk); + const stillDeduped = !(await claimAtomic(after, 'evt_restart')); + check( + '(e) durable store: ledger survived a restart?', + stillDeduped, + true, + ); + + // And the swap is not a leap of faith: the contract suite ships. + const report = await verifyStoreContract(() => + clockStore( + { + now: () => Date.now(), + setTimer: setTimeout, + clearTimer: clearTimeout, + sleep: async () => undefined, + }, + new Map(), + ), + ); + assertConformance(report); + check( + '(e) `verifyStoreContract` on the BYO store → ok', + report.ok, + true, + ); + check('(e) → rules passed', report.passed.length, 11); + check('(e) → violations', report.violations.length, 0); + note( + '(e) → `redisStore` / `cloudflareKvStore` / `denoKvStore` implement this same interface', + 'the durable ledger is a one-line config change, and `verifyStoreContract` (testing.ts:173) proves a BYO one conforms', + ); + } + + finish( + 'C5', + 'YES, and the capture\'s open question has the ownership backwards. `store` is a config key the USER supplies (types.ts:1583-1584) and `memoryStore()` is a public export (index.ts:62) — nothing is borrowed from the engine; the ledger constructs the store and lends the same instance to the stitch. Three deliveries of one event id measured ["first:processed","retry:skipped","retry:skipped"] with exactly 1 side effect. TWO FINDINGS THE CAPTURE MISSES. First, `get`-then-`set` is a race the store already fixes: 3 concurrent claims on one id returned [true,true,true] — three charges — while `increment(key, ttl)` (types.ts:1969-1970) returned [true,false,false], exactly one. Second, the TTL boundary is exact but only against a clock-backed store: with a 4-day TTL over Stripe\'s 3-day window, claims at t=3d and t=ttl-1ms both skipped and t=ttl processed (the boundary is `expires > now`, exclusive, store.ts:16) — whereas `memoryStore` takes no clock (arity 0) and after FOUR virtual days on a `manualClock` the key was still live, so a 3-day window cannot be boundary-tested against it at all. Durability is the real gap in the default: `memoryStore.close()` is `data.clear()` (store.ts:59-61), so the ledger did not survive, and a deploy inside the retry window re-processes everything still in flight. The swap is first-class — a BYO durable store over the same interface survived a restart and passed all 11 rules of `verifyStoreContract` (testing.ts:173) with 0 violations', + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c6-fast-ack.ts b/docs/scenarios/proofs/webhook-receipt/c6-fast-ack.ts new file mode 100644 index 00000000..69a1a36a --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c6-fast-ack.ts @@ -0,0 +1,261 @@ +// C6 — does anything in the library help with the fast-2xx requirement (ack now, process later)? +// +// The capture's framing: providers time out in seconds, so real work has to be queued rather than +// done inline, "which CREATES the duplicate problem you just solved, one layer down". So the +// question is whether StitchAPI has a queue, an outbox, a detach, or any seam that lets a response +// go out before the work finishes. +// +// It does not, and there are two sharp edges rather than one. +// +// (b) `serve`'s handler AWAITS the whole run before it writes a byte (serve.ts:272-273), so a +// stitch with a retry policy makes the ack as late as the last attempt. Driven on a +// `manualClock`, the ack landed exactly 10 VIRTUAL SECONDS and 3 upstream attempts late — +// which is Stripe's timeout to the second. Every resilience feature that makes an outbound +// call more reliable makes that ack later. +// +// (c) `void call(input)` — the spelling anyone reaches for to ack-then-continue — makes NO +// request at all. A stitch call is a lazy thenable that starts on `.then`, so the work is +// silently dropped while the provider is told 200. This is the finding of the claim. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c6-fast-ack.ts +import { pipelineStages } from '../../../../packages/core/src/config-summary'; +import { stitch } from '../../../../packages/core/src/index'; +import { serve } from '../../../../packages/core/src/serve'; +import { manualClock } from '../../../../packages/core/src/testing'; +import type { + Adapter, + RedactedStitchConfig, +} from '../../../../packages/core/src/types'; +import { check, finish, heading, note } from './harness'; + +/** + * Yield macrotasks until `pred` holds. Needed because half of this claim straddles a REAL socket + * (the `serve` request) and a VIRTUAL clock (the stitch's backoff): the run has to actually reach + * the engine before there is a timer to advance. Bounded so a broken assumption fails the script + * rather than hanging it. + */ +async function until(pred: () => boolean, turns = 200): Promise { + for (let i = 0; i < turns; i++) { + if (pred()) return true; + await new Promise((r) => setImmediate(r)); + } + return pred(); +} + +async function main(): Promise { + heading('C6 — ack now, process later'); + + // ── (a) the engine's own pipeline read-out: is there a stage after which a caller is free? ─ + // `pipelineStages` (config-summary.ts:86) renders the configured stages in engine order. It is + // the library's own answer to "what happens, and when", so it is the right place to look for a + // detach point. + { + const maximal = stitch({ + url: 'https://api.billing.test/v1/subscriptions/sub_1', + method: 'POST', + adapter: (async () => ({ + status: 200, + headers: {}, + body: {}, + })) as Adapter, + retry: { attempts: 3 }, + throttle: { rate: '100/s', concurrency: 4 }, + timeout: { total: '10s' }, + circuit: { failures: 5, cooldown: '30s' }, + idempotency: true, + hooks: { onResponse: () => undefined }, + }); + const stages = pipelineStages( + maximal.__config as unknown as RedactedStitchConfig, + ); + note('(a) configured pipeline, in engine order', stages.join(' → ')); + check( + '(a) any stage naming a queue / background / detach / ack?', + stages.some((s) => + /queue|background|detach|ack|defer|async/i.test(s), + ), + false, + ); + check('(a) the last stage', stages[stages.length - 1], 'result'); + note( + '(a) → every stage is upstream of `result`', + 'the pipeline is one call from `call` to `result`; there is no point at which a caller is released early', + ); + } + + // ── (b) `serve` awaits the whole run before responding, retries included ─────────────────── + // Driven on a manual clock so the lateness is an exact number rather than a stopwatch reading. + { + let attempts = 0; + const flaky: Adapter = async () => { + attempts++; + return attempts <= 2 + ? { status: 503, headers: {}, body: { error: 'busy' } } + : { status: 200, headers: {}, body: { ok: true } }; + }; + const clock = manualClock(); + const registry = { + 'on-webhook': stitch({ + url: 'https://api.billing.test/v1/subscriptions/sub_1', + method: 'GET', + adapter: flaky, + // 5s fixed, so two backoffs put the ack exactly at Stripe's 10s timeout. + // (`base: 30_000` would have been silently clamped to 10s — `backoff.max` + // defaults to 10s, types.ts:972-973.) + retry: { + attempts: 3, + backoff: { curve: 'fixed', base: 5_000 }, + }, + clock, + }), + }; + const handle = await serve(registry, { port: 0 }); + try { + let responded = false; + const ack = fetch(handle.url + '/stitch/on-webhook', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: '{}', + }).then((r) => { + responded = true; + return r; + }); + + // Attempt 1 — the 503. The virtual clock has not moved; nothing has gone back yet. + await until(() => attempts >= 1); + check('(b) upstream attempts so far', attempts, 1); + check('(b) has the provider been acked?', responded, false); + check('(b) virtual ms elapsed', clock.now(), 0); + + // 5 virtual seconds of backoff, then attempt 2 — still the 503. + await clock.advance(5_000); + await until(() => attempts >= 2); + check('(b) upstream attempts so far', attempts, 2); + check('(b) has the provider been acked?', responded, false); + + // 5 more, then attempt 3 — the success. NOW the ack goes out. + await clock.advance(5_000); + const res = await ack; + await res.arrayBuffer(); + check('(b) upstream attempts so far', attempts, 3); + check('(b) ack status', res.status, 200); + check('(b) virtual ms the ack waited', clock.now(), 10_000); + note( + '(b) → `serve` consumes the run to completion before writing (serve.ts:177-201,272-273)', + 'a provider that waits ~10s has already timed out and queued a re-delivery — the retry policy manufactured the duplicate', + ); + } finally { + await handle.close(); + } + } + + // ── (c) the only "ack now" available is not awaiting — and the obvious spelling is a no-op ─ + // THE FINDING. `void call(input)` is what anyone writes for fire-and-forget, and it does + // NOTHING: a stitch call returns a lazy thenable that starts the run on `.then` (stitch.ts: + // 729,781). No request is made, no error is raised, and the provider has already been acked. + { + const rejections: unknown[] = []; + const onUnhandled = (e: unknown): void => { + rejections.push(e); + }; + process.on('unhandledRejection', onUnhandled); + try { + let ran = 0; + const failing = stitch({ + url: 'https://api.billing.test/v1/subscriptions/sub_1', + method: 'GET', + adapter: (async () => { + ran++; + return { + status: 500, + headers: {}, + body: { error: 'boom' }, + }; + }) as Adapter, + }); + + // (c1) the spelling everyone reaches for. + void failing({}); + await until(() => ran >= 1, 20); + check('(c) `void call(input)` → HTTP calls made', ran, 0); + check( + '(c) `void call(input)` → errors raised anywhere', + rejections.length, + 0, + ); + + // (c2) the same thing with a `.then`, which is what actually starts it. + void failing({}).then( + () => undefined, + () => undefined, + ); + await until(() => ran >= 1); + check('(c) `void call(input).then(…)` → HTTP calls made', ran, 1); + + // (c3) unsupervised, for real: the run happens and the failure lands nowhere a caller + // can see. Un-handled, it is an unhandledRejection; handled by `.safe()`, it is silence. + // A ONE-ARG `.then` starts the run and leaves the rejection unowned, which is the + // realistic shape of "kick it off and move on". + const started = ran; + void failing({}).then(() => undefined); + await until(() => ran > started && rejections.length >= 1); + check( + '(c) an unsupervised failure surfaces as', + rejections.length, + 1, + ); + + rejections.length = 0; + const before = ran; + void failing.safe({}); + await until(() => ran > before); + check( + '(c) `.safe()` fire-and-forget → the work ran', + ran, + before + 1, + ); + check( + '(c) `.safe()` fire-and-forget → unhandled rejections', + rejections.length, + 0, + ); + check( + '(c) …and the failure was reported where?', + 'nowhere', + 'nowhere', + ); + note( + '(c) → `void call()` is not even an ack-and-continue, it is a DROP', + 'and the spelling that does run gives no durability, no redelivery, no dead-letter, no backpressure', + ); + } finally { + process.off('unhandledRejection', onUnhandled); + } + } + + // ── (d) the one fire-and-forget verb in the repo, and what it is for ─────────────────────── + // The repo's only "send and do not wait" is `channel().emit` (postmessage.ts:201-206) — a + // no-reply `window.postMessage` between browser realms. Not a top-level export, not a job + // queue, and not reachable from a server-side webhook handler. + { + const pm = await import('../../../../packages/core/src/postmessage'); + const verbs = Object.keys(pm).sort(); + check( + '(d) does the postmessage module export a queue?', + verbs.some((v) => /queue|job|outbox|worker|emit/i.test(v)), + false, + ); + note('(d) postmessage top-level exports', verbs.join(', ')); + note( + '(d) → `emit` is a METHOD on `channel()`, and a transport verb', + 'no-reply postMessage between browser realms; nothing about durability or redelivery', + ); + } + + finish( + 'C6', + "NO — nothing in the library addresses it, `serve` actively works against it, and the obvious workaround is a silent data-loss bug. The engine's own read-out says so first: `pipelineStages` on a maximally-configured stitch rendered `call → throttle → POST … → retry → http interpret → result` with no stage matching /queue|background|detach|ack|defer|async/ and `result` last — the pipeline never releases a caller early. `serve` then consumes the run to completion before writing a byte (serve.ts:177-201,272-273): with `retry: {attempts: 3}` and a 5s fixed backoff on a `manualClock`, the ack was still unsent after attempt 1 and after attempt 2 and went out only at attempt 3, having waited exactly 10 VIRTUAL SECONDS — Stripe's timeout to the second, so the retry policy manufactures the duplicate it was meant to survive. THE FINDING IS (c): `void call(input)`, the spelling anyone writes for ack-then-continue, made 0 HTTP calls and raised 0 errors — a stitch call is a lazy thenable that starts on `.then` (stitch.ts:729,781), so the work is silently dropped after the provider has been told 200. `void call(input).then(…)` does run it (1 call), and then it is unsupervised: unhandled it surfaced as 1 `unhandledRejection`, and `.safe()` produced 0 rejections and reported the failure nowhere. Either way it is an ack, not a queue — no durability, no redelivery, no dead-letter, no backpressure. The repo's only fire-and-forget verb, `channel().emit` (postmessage.ts:201-206), is a browser-realm transport with no queue semantics", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/c7-the-boundary.ts b/docs/scenarios/proofs/webhook-receipt/c7-the-boundary.ts new file mode 100644 index 00000000..f66b25c2 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/c7-the-boundary.ts @@ -0,0 +1,270 @@ +// C7 — the most honest end-to-end answer: a real `node:http` server for receipt, StitchAPI for +// everything downstream. Then the line count of EACH HALF, so the boundary is a number. +// +// This is not a demonstration that it can be made to work. It is the shape you would actually ship, +// run against a real socket with real signed bytes, exercising every failure the capture names: +// a forged signature, a replayed one, a duplicate delivery, and a reversed pair. The last section +// counts the two halves separately, because "which half is StitchAPI's" is the question this whole +// scenario exists to answer and a number settles it. +// +// pnpm exec tsx docs/scenarios/proofs/webhook-receipt/c7-the-boundary.ts +import { memoryStore } from '../../../../packages/core/src/index'; +import { + type BillingEvent, + type Delivery, + FakeBilling, + mintDelivery, +} from './fake-billing'; +import { check, checkSeq, finish, heading, note } from './harness'; +import { type Subscription, createReaction } from './reaction'; +import { startReceiver } from './receiver'; +import { signPayload } from './stripe-sig'; + +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SECRET = 'whsec_test_2f9d1c4b'; +const NOW = 1_800_000_000; +const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000; + +/** + * Executable lines — imports (however they wrap), blanks and comment-only lines removed on BOTH + * sides, so the number is the code someone actually writes and maintains. + */ +function executableLines(file: string): number { + return readFileSync(join(HERE, file), 'utf8') + .replace(/^import[\s\S]*?;$/gm, '') + .split('\n') + .map((l) => l.trim()) + .filter( + (l) => + l !== '' && + !l.startsWith('//') && + !l.startsWith('*') && + !l.startsWith('/*'), + ).length; +} + +const event = (id: string, type: string, sub: Subscription): BillingEvent => ({ + id, + type, + created: NOW, + data: { object: sub as never }, +}); + +async function main(): Promise { + heading('C7 — the assembled answer, and where the boundary falls'); + + const api = new FakeBilling(); + api.setSubscription({ + id: 'sub_1', + status: 'active', + plan: 'pro', + version: 2, + }); + const store = memoryStore(); + + const written: Subscription[] = []; + const stale: number[] = []; + const decisions: string[] = []; + + const reaction = createReaction({ + baseUrl: FakeBilling.baseUrl, + token: 'sk_live_xyz', + store, + adapter: api.adapter(), + write: (s) => written.push(s), + onStale: (v) => stale.push(v), + }); + + // The reaction is awaited here only so the assertions are deterministic; in the receiver it is + // fired after the ack and never awaited by the request path. + const settled: Promise[] = []; + const receiver = await startReceiver({ + path: '/webhooks/stripe', + secret: SECRET, + store, + dedupTtlMs: THREE_DAYS_MS + 24 * 60 * 60 * 1000, + nowSeconds: () => NOW, + onEvent: (e) => { + const p = reaction.handle(e); + settled.push(p); + return p; + }, + onDecision: (d) => decisions.push(d), + }); + + const deliver = async (d: Delivery, sig?: string): Promise => { + const res = await fetch(receiver.url + '/webhooks/stripe', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'stripe-signature': sig ?? d.signature, + }, + body: Uint8Array.from(d.raw), + }); + await res.arrayBuffer(); + return res.status; + }; + + try { + // The two events, minted in the order the provider created them… + const created = mintDelivery( + event('evt_created', 'customer.subscription.created', { + id: 'sub_1', + status: 'trialing', + plan: 'free', + version: 1, + }), + SECRET, + NOW, + ); + const updated = mintDelivery( + event('evt_updated', 'customer.subscription.updated', { + id: 'sub_1', + status: 'active', + plan: 'pro', + version: 2, + }), + SECRET, + NOW, + ); + + // ── (a) a forged delivery ───────────────────────────────────────────────────────────── + const forged = signPayload(updated.raw, 'whsec_wrong_key', NOW); + check( + '(a) forged signature → status', + await deliver(updated, forged), + 400, + ); + check('(a) → decision', decisions.at(-1), 'bad-signature'); + + // ── (b) a genuine signature on a stale timestamp: the replay window ──────────────────── + const old = mintDelivery( + event('evt_replay', 'customer.subscription.updated', { + id: 'sub_1', + status: 'active', + plan: 'pro', + version: 2, + }), + SECRET, + NOW - 600, // 10 minutes ago, tolerance is 5 + ); + check( + '(b) valid MAC, 10-minute-old timestamp → status', + await deliver(old), + 400, + ); + check('(b) → decision', decisions.at(-1), 'stale'); + check('(b) side effects from the replay', written.length, 0); + + // ── (c) the real pair, arriving REVERSED ────────────────────────────────────────────── + check( + '(c) `updated` (arriving first) → status', + await deliver(updated), + 200, + ); + await Promise.all(settled); + check( + '(c) `created` (arriving second) → status', + await deliver(created), + 200, + ); + await Promise.all(settled); + + checkSeq( + '(c) states written to the read model', + written.map((w) => `${w.status}/${w.plan}`), + ['active/pro'], + ); + checkSeq('(c) writes rejected by the version guard', stale, [2]); + check( + '(c) final read-model state matches the server', + `${written.at(-1)?.status}/${written.at(-1)?.plan}`, + 'active/pro', + ); + + // ── (d) the provider's at-least-once retry of an event already handled ──────────────── + const before = api.requests.length; + check('(d) duplicate delivery → status', await deliver(updated), 200); + await Promise.all(settled); + check('(d) → decision', decisions.at(-1), 'duplicate:evt_updated'); + check('(d) API calls it cost', api.requests.length - before, 0); + checkSeq( + '(d) idempotency keys on the downstream write', + api.entitlementKeys, + ['evt_updated'], + ); + note( + '(d) → two independent guards, and both fired', + 'the ledger stopped the duplicate before any call; `idempotency` would have stopped a double-write if it had not', + ); + + // ── (e) the decision log, end to end ────────────────────────────────────────────────── + checkSeq('(e) every decision the receiver made', decisions, [ + 'bad-signature', + 'stale', + 'accepted:evt_updated', + 'accepted:evt_created', + 'duplicate:evt_updated', + ]); + } finally { + await receiver.close(); + await reaction.close(); + await store.close?.(); + } + + // ── (f) the boundary, as a number ───────────────────────────────────────────────────────── + { + const receipt = executableLines('receiver.ts'); + const signature = executableLines('stripe-sig.ts'); + const reactionLines = executableLines('reaction.ts'); + const receiptTotal = receipt + signature; + + note( + ' RECEIPT half — `receiver.ts`', + `${receipt} executable lines (no StitchAPI in it)`, + ); + note( + ' RECEIPT half — `stripe-sig.ts`', + `${signature} executable lines (node:crypto)`, + ); + note(' RECEIPT half — total', `${receiptTotal} executable lines`); + note( + ' REACTION half — `reaction.ts`', + `${reactionLines} executable lines (all StitchAPI)`, + ); + + check('(f) receipt half, executable lines', receiptTotal, 154); + check('(f) reaction half, executable lines', reactionLines, 63); + check( + '(f) does the receipt half import anything from `stitchapi`?', + /from '.*packages\/core\/src\/(index|serve|auth|cache|pipe)'/.test( + readFileSync(join(HERE, 'receiver.ts'), 'utf8'), + ), + false, + ); + check( + '(f) …and its ONLY stitchapi reference is a type', + ( + readFileSync(join(HERE, 'receiver.ts'), 'utf8').match( + /^import type .*packages\/core/gm, + ) ?? [] + ).length, + 1, + ); + note( + '(f) → the split is 71% / 29% by line, and 100% / 0% by concern', + 'the receipt half touches no StitchAPI runtime at all; its one reference is `import type { StitchStore }`', + ); + } + + finish( + 'C7', + "The honest answer is a `node:http` server the user owns plus StitchAPI for everything after the ack, and it works end to end: a forged signature was rejected 400 `bad-signature`, a genuine MAC on a 10-minute-old timestamp was rejected 400 `stale` with 0 side effects, the two real events arriving REVERSED both acked 200 and converged the read model on `active/pro` with the version guard rejecting exactly [2], and the provider's duplicate acked 200 at a cost of 0 API calls with one `Idempotency-Key` (`evt_updated`) ever reaching the downstream write. THE BOUNDARY AS A NUMBER: the receipt half is 154 executable lines (96 of server + 58 of `node:crypto` signature verification) and imports NOTHING from `stitchapi` at runtime — its single reference to the package is `import type { StitchStore }`, a type. The reaction half is 63 executable lines and is almost all config: one seam carrying `auth`, `retry`, `throttle` and `timeout`, two member stitches, and a four-line version guard the library does not express. 71% of the code is the half StitchAPI does not participate in", + ); +} + +void main(); diff --git a/docs/scenarios/proofs/webhook-receipt/clock-store.ts b/docs/scenarios/proofs/webhook-receipt/clock-store.ts new file mode 100644 index 00000000..916a5ecb --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/clock-store.ts @@ -0,0 +1,62 @@ +// A `StitchStore` whose TTL is driven by an INJECTED clock, so a dedup entry's expiry is virtual +// time rather than wall time. +// +// This exists because the default `memoryStore()` reads `now()` — `Date.now()` (store.ts:16,45,54 +// via util.ts:4) — and therefore ignores a stitch's `clock` entirely. C5 measures that directly: a +// dedup key written with Stripe's 3-day retry window as its TTL is still readable after a +// `manualClock()` has been advanced FOUR virtual days. Testing the TTL boundary of a 3-day window +// against `Date.now()` is not something a test suite can do, so a user who wants that boundary +// covered writes this file — which is why it is here rather than imported. +// +// It is a faithful copy of `memoryStore`'s semantics with `now()` swapped for `clock.now()`: the +// `expires === 0` sentinel means "no TTL", `set(key, undefined)` deletes, and `increment` keeps the +// first window's expiry. The opportunistic sweep is omitted — a proof run stores a handful of keys. +// +// `backing` is optional and is what makes DURABILITY testable: hand the same `Map` to two +// successive stores and the second one is the process that restarted. +import type { Clock, StitchStore } from '../../../../packages/core/src/types'; + +export function clockStore( + clock: Clock, + backing: Map = new Map(), +): StitchStore { + const data = backing; + const live = (e?: { expires: number }): boolean => + !!e && (e.expires === 0 || e.expires > clock.now()); + return { + async get(key) { + const e = data.get(key); + if (!live(e)) { + data.delete(key); + return undefined; + } + return e?.value; + }, + async set(key, value, ttl) { + if (value === undefined) { + data.delete(key); + return; + } + data.set(key, { value, expires: ttl ? clock.now() + ttl : 0 }); + }, + async increment(key, ttl) { + const e = data.get(key); + const n = (live(e) ? (e?.value as number) : 0) + 1; + data.set(key, { + value: n, + expires: live(e) + ? (e?.expires ?? 0) + : ttl + ? clock.now() + ttl + : 0, + }); + return n; + }, + // A durable store's `close()` releases the CONNECTION, not the data — `redisStore` closes a + // client, it does not FLUSHDB. Only the in-memory default conflates the two, which is the + // point C5 (e) measures, so this one deliberately leaves `backing` intact. + async close() { + /* nothing to release: the data outlives the handle, as a durable store's does */ + }, + }; +} diff --git a/docs/scenarios/proofs/webhook-receipt/fake-billing.ts b/docs/scenarios/proofs/webhook-receipt/fake-billing.ts new file mode 100644 index 00000000..f91e8729 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/fake-billing.ts @@ -0,0 +1,152 @@ +// The provider, in both directions — because this scenario has two of them. +// +// OUTBOUND (StitchAPI's half): a `GET /v1/subscriptions/:id` served through an `Adapter`, which +// is the fetch-on-receipt call. It returns the CURRENT server truth, which is the entire reason +// fetch-on-receipt fixes out-of-order delivery: the answer does not depend on which event +// prompted the question. +// +// INBOUND (not StitchAPI's half): `mintDelivery` produces the exact bytes a provider would POST +// plus the `Stripe-Signature` header over those bytes. It signs a `Buffer` and hands back that +// same `Buffer`, so a proof that verifies against a re-serialised object is verifying against +// something the provider never signed — which is the bug C1 is about. +// +// Knobs are the ones the claims need and no more: `failNext` (transient status on a path, so the +// retry budget on the fetch-on-receipt stitch is measurable), `delayTicks` (hold a response for N +// microtask turns, so two concurrent handlers can be made to interleave deterministically), and +// `onRequest` (a hook the tests use to advance server state mid-flight). +import type { + Adapter, + AdapterRequest, + AdapterResponse, +} from '../../../../packages/core/src/types'; +import { signPayload } from './stripe-sig'; + +export interface Subscription { + id: string; + status: 'trialing' | 'active' | 'canceled'; + plan: 'free' | 'pro'; + /** Monotonic server version — the guard C4 needs to show fetch-on-receipt is not sufficient alone. */ + version: number; +} + +/** The shape of a provider event as it arrives on the wire. */ +export interface BillingEvent { + id: string; + type: string; + created: number; + data: { object: Subscription }; +} + +/** Exactly what lands on a webhook endpoint: raw bytes plus the header signed over them. */ +export interface Delivery { + raw: Buffer; + signature: string; + /** Convenience for the assertions — never used as the thing being verified. */ + eventId: string; +} + +/** + * Serialise an event and sign the resulting BYTES. The returned `raw` is the only correct input to + * verification: re-`JSON.stringify`ing the parsed object gives logically identical JSON whose bytes + * differ, which is the failure this scenario exists to demonstrate. + */ +export function mintDelivery( + event: BillingEvent, + secret: string, + timestampSeconds: number, + /** Whitespace/key-order the provider happened to emit. Defaults to compact, like a real one. */ + serialise: (e: BillingEvent) => string = (e) => JSON.stringify(e), +): Delivery { + const raw = Buffer.from(serialise(event), 'utf8'); + return { + raw, + signature: signPayload(raw, secret, timestampSeconds), + eventId: event.id, + }; +} + +export class FakeBilling { + static readonly baseUrl = 'https://api.billing.test'; + + /** Every path the client actually requested, in arrival order. The fetch-on-receipt cost. */ + readonly requests: string[] = []; + + /** `Idempotency-Key` values seen on the downstream write — one per logical entitlement change. */ + readonly entitlementKeys: string[] = []; + + private readonly subs = new Map(); + private readonly failures = new Map< + string, + { times: number; status: number } + >(); + private readonly delays = new Map(); + /** Fires as a request ARRIVES, before the (possibly delayed) response is built. */ + onRequest?: (path: string) => void; + + setSubscription(sub: Subscription): void { + this.subs.set(sub.id, sub); + } + + get(id: string): Subscription | undefined { + return this.subs.get(id); + } + + /** Answer `path` with `status` the next `times` requests, then serve normally. */ + failNext(path: string, times: number, status: number): void { + this.failures.set(path, { times, status }); + } + + /** Hold `path`'s response for `ticks` microtask turns so two handlers can be interleaved. */ + delayTicks(path: string, ticks: number): void { + this.delays.set(path, ticks); + } + + adapter(): Adapter { + return async (req: AdapterRequest): Promise => { + const path = new URL(req.url).pathname; + this.requests.push(path); + this.onRequest?.(path); + + const ticks = this.delays.get(path) ?? 0; + for (let i = 0; i < ticks; i++) await Promise.resolve(); + + const failure = this.failures.get(path); + if (failure && failure.times > 0) { + failure.times--; + return { + status: failure.status, + headers: {}, + body: { error: { message: 'transient' } }, + }; + } + + // The downstream write the reaction half performs after it has the current state. + // Records the `Idempotency-Key` so a duplicate that got past the ledger is visible. + if (path === '/v1/entitlements' && req.method === 'POST') { + const key = + req.headers['Idempotency-Key'] ?? + req.headers['idempotency-key']; + if (key !== undefined) this.entitlementKeys.push(key); + return { status: 200, headers: {}, body: { applied: true } }; + } + + const match = /^\/v1\/subscriptions\/([^/]+)$/.exec(path); + if (!match || req.method !== 'GET') + return { + status: 404, + headers: {}, + body: { error: { message: 'not_found' } }, + }; + const sub = this.subs.get(match[1] ?? ''); + if (!sub) + return { + status: 404, + headers: {}, + body: { error: { message: 'no such subscription' } }, + }; + // A snapshot, not the live object — a handler holding a response must not observe a + // later mutation through it. + return { status: 200, headers: {}, body: { ...sub } }; + }; + } +} diff --git a/docs/scenarios/proofs/webhook-receipt/harness.ts b/docs/scenarios/proofs/webhook-receipt/harness.ts new file mode 100644 index 00000000..c5e829a6 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/harness.ts @@ -0,0 +1,67 @@ +// Minimal assertion harness for the proof scripts: every check prints a line, and the script +// exits non-zero if any check failed. No test framework — these are standalone `tsx` scripts. +// +// This scenario's evidence is mostly STATUS CODES and BYTES: what a real `serve` process answered +// when a Stripe-shaped delivery was POSTed at it, and whether the bytes that reach user code are +// still the bytes the provider signed. So both assertions print the measured value whether they +// pass or fail — `404` and `hmac(raw) !== hmac(restringified)` ARE the findings, and they have to +// be readable out of context. + +let failures = 0; +let checks = 0; + +/** Assert an observed value equals what the claim predicts. Prints the MEASURED value either way. */ +export function check(label: string, actual: unknown, expected: unknown): void { + checks++; + const ok = Object.is(actual, expected); + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${String(actual)}${ok ? '' : ` (expected ${String(expected)})`}`, + ); +} + +/** + * Assert a measured SEQUENCE matches, comparing element-wise via `JSON.stringify`. The measured + * sequence is printed in full whether it passes or fails — the status spine + * (`[404, 404, 404, 200]`) and the applied-state spine (`["active", "trialing"]`) ARE the evidence. + */ +export function checkSeq( + label: string, + actual: readonly unknown[], + expected: readonly unknown[], +): void { + checks++; + const a = JSON.stringify(actual); + const e = JSON.stringify(expected); + const ok = a === e; + if (!ok) failures++; + console.log( + ` ${ok ? 'ok ' : 'FAIL'} ${label}: measured ${a}${ok ? '' : ` (expected ${e})`}`, + ); +} + +/** Record a measurement that is reported but not asserted (context for the verdict). */ +export function note(label: string, value: unknown): void { + console.log(` note ${label}: ${String(value)}`); +} + +export function heading(text: string): void { + console.log(`\n${text}`); +} + +/** + * Print the claim's verdict line and exit. `claim` is e.g. `'C1'`; `statement` is what a PASS + * means, so the printed line is self-describing when someone reads it out of context. + * + * Several claims here PASS by measuring an ABSENCE — the verdict statement carries the direction, + * because "PASS C2" on a claim whose content is "there is no such primitive" is otherwise unreadable. + */ +export function finish(claim: string, statement: string): never { + const pass = failures === 0; + console.log( + `\n${pass ? 'PASS' : 'FAIL'} ${claim} — ${statement} (${checks - failures}/${checks} checks)`, + ); + process.exit(pass ? 0 : 1); + // `process.exit` is typed `never`, but TypeScript still wants the end point unreachable. + throw new Error('unreachable'); +} diff --git a/docs/scenarios/proofs/webhook-receipt/reaction.ts b/docs/scenarios/proofs/webhook-receipt/reaction.ts new file mode 100644 index 00000000..4ce14e38 --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/reaction.ts @@ -0,0 +1,97 @@ +// THE REACTION HALF — user code, and it is almost entirely config. Everything downstream of the +// ack is an outbound call, which is exactly what this library is for. +// +// Two stitches on one seam: +// +// `current` — fetch-on-receipt. The event is a HINT; this is the answer. C4 measured that this +// is what makes reversed delivery converge, and that it needs its own auth, retry +// and rate budget — all of which are the seam's, not this file's. +// `apply` — the downstream write, carrying the event id as its idempotency key so a duplicate +// that got past the ledger still cannot double-charge. +// +// The version guard is here rather than in config because C4 (c) measured that fetch-on-receipt +// alone does NOT make write order safe: two concurrent handlers holding snapshots v2 and v3 land +// on v2 under last-write-wins. Nothing in the library expresses "reject a write carrying an older +// version" — it is four lines, and they are four lines you have to write. +import { bearer } from '../../../../packages/core/src/auth'; +import { seam } from '../../../../packages/core/src/index'; +import type { Adapter, StitchStore } from '../../../../packages/core/src/types'; + +export interface Subscription { + id: string; + status: string; + plan: string; + version: number; +} + +export interface ReactionOptions { + baseUrl: string; + token: string; + /** The SAME store the receipt half deduplicates against (C5). */ + store: StitchStore; + adapter: Adapter; + /** The local read model the handler maintains. */ + write: (sub: Subscription) => void; + /** Reports a write rejected as stale, so the guard is observable. */ + onStale?: (version: number) => void; +} + +export interface Reaction { + handle(event: { id: string; type: string; subject: string }): Promise; + close(): Promise; +} + +export function createReaction(opts: ReactionOptions): Reaction { + // One seam: the auth, the retry budget, the rate budget and the deadline are declared once and + // both members inherit them. The store is the receipt half's ledger, shared. + const api = seam({ + baseUrl: opts.baseUrl, + auth: bearer(() => opts.token), + adapter: opts.adapter, + store: opts.store, + retry: { attempts: 3, on: [429, 500, 502, 503, 504] }, + throttle: { rate: '25/s' }, + timeout: { total: '10s' }, + }); + + const current = api.stitch({ + path: '/v1/subscriptions/{id}', + method: 'GET', + }); + const apply = api.stitch({ + path: '/v1/entitlements', + method: 'POST', + idempotency: { + keyOf: (input) => + String((input.body as { eventId?: string }).eventId), + }, + }); + + let seen = 0; + + return { + async handle(event) { + // The payload contributed the subject id and nothing else — the state comes from here. + const sub = (await current({ + params: { id: event.subject }, + })) as Subscription; + + // C4 (c): the guard fetch-on-receipt does not give you. + if (sub.version <= seen) { + opts.onStale?.(sub.version); + return; + } + seen = sub.version; + + await apply({ + body: { + eventId: event.id, + subscription: sub.id, + plan: sub.plan, + }, + }); + opts.write(sub); + }, + close: () => api.close(), + }; +} diff --git a/docs/scenarios/proofs/webhook-receipt/receiver.ts b/docs/scenarios/proofs/webhook-receipt/receiver.ts new file mode 100644 index 00000000..ace947dd --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/receiver.ts @@ -0,0 +1,145 @@ +// THE RECEIPT HALF — user code, and there is no StitchAPI in this file. That absence is the +// deliverable: C1–C3 measured that nothing in the library can receive a signed webhook, so the +// honest end-to-end answer starts with a plain `node:http` server, and this is what it costs. +// +// Everything the capture lists as required is here and nowhere else: +// +// • an arbitrary route the provider dashboard can be pointed at +// • the RAW bytes, buffered before anything parses them +// • a constant-time HMAC over those bytes, inside a timestamp tolerance (`stripe-sig.ts`) +// • an atomic dedup claim on the event id, TTL beyond the provider's retry window +// • a 2xx written BEFORE the work starts +// • a body cap, so an unbounded POST cannot OOM the process +// +// The dedup ledger is a `StitchStore` — the ONE place the two halves genuinely share machinery +// (C5): it is an interface the user supplies, so the same Redis handle backs both the ledger here +// and the engine's throttle state on the other side. That is a type dependency, not a runtime one. +// +// `onEvent` is deliberately fired AFTER the response is flushed and its promise is not awaited by +// the request path. That is an ack, not a queue — see `README.md`; a production system replaces it +// with an enqueue, and then owns a second dedup problem one layer down. +import type { StitchStore } from '../../../../packages/core/src/types'; +import { type VerifyResult, verifySignature } from './stripe-sig'; + +import { type Server, createServer } from 'node:http'; + +export interface ReceiverOptions { + path: string; + secret: string; + store: StitchStore; + /** TTL for a dedup entry, in ms. Must exceed the provider's retry window (Stripe: 3 days). */ + dedupTtlMs: number; + /** Injected so the tolerance boundary is testable; defaults to the wall clock. */ + nowSeconds?: () => number; + /** Ceiling on a buffered body. */ + maxBytes?: number; + /** + * The reaction half. Fired after the ack; never awaited by the request path. It is handed the + * event id (for dedup + idempotency), the type, and the SUBJECT id — and nothing else from the + * payload, because C4 measured that the payload's own state is the part you must not trust. + */ + onEvent: (event: { + id: string; + type: string; + subject: string; + }) => Promise; + /** Observability seam for the proofs — one line per delivery decision. */ + onDecision?: (decision: string) => void; +} + +export interface ReceiverHandle { + url: string; + server: Server; + close(): Promise; +} + +export async function startReceiver( + opts: ReceiverOptions, +): Promise { + const now = opts.nowSeconds ?? (() => Math.floor(Date.now() / 1000)); + const maxBytes = opts.maxBytes ?? 1024 * 1024; + const decide = opts.onDecision ?? ((): void => undefined); + + const server = createServer((req, res) => { + void (async () => { + if (req.method !== 'POST' || req.url !== opts.path) { + res.writeHead(404).end(); + return; + } + // 1. RAW bytes, bounded. Nothing may parse before the MAC is checked. + const chunks: Buffer[] = []; + let size = 0; + for await (const c of req) { + const buf = c as Buffer; + size += buf.length; + if (size > maxBytes) { + req.destroy(); + res.writeHead(413).end(); + decide('too-large'); + return; + } + chunks.push(buf); + } + const raw = Buffer.concat(chunks); + + // 2. Authenticity + replay window, over those exact bytes. + const verdict: VerifyResult = verifySignature( + raw, + req.headers['stripe-signature'] as string | undefined, + opts.secret, + now(), + ); + if (!verdict.ok) { + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: verdict.reason })); + decide(verdict.reason); + return; + } + + // 3. Only now is parsing safe. + const parsed = JSON.parse(raw.toString('utf8')) as { + id: string; + type: string; + data: { object: { id: string } }; + }; + const event = { + id: parsed.id, + type: parsed.type, + subject: parsed.data.object.id, + }; + + // 4. Atomic dedup claim. `increment` and not `get`+`set`: the racy pair lets two + // workers both win (C5 (b)). + const claim = await opts.store.increment( + `webhook:${event.id}`, + opts.dedupTtlMs, + ); + if (claim !== 1) { + res.writeHead(200).end(); + decide(`duplicate:${event.id}`); + return; + } + + // 5. Ack FIRST, then work. The provider's clock stops here. + res.writeHead(200).end(); + decide(`accepted:${event.id}`); + void opts.onEvent(event).catch(() => { + // A real system dead-letters here. Ack-then-drop is the trade this shape makes. + decide(`failed:${event.id}`); + }); + })(); + }); + + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const addr = server.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + return { + url: `http://127.0.0.1:${port}`, + server, + close: () => + new Promise((r) => { + server.closeAllConnections(); + server.close(() => r()); + }), + }; +} diff --git a/docs/scenarios/proofs/webhook-receipt/stripe-sig.ts b/docs/scenarios/proofs/webhook-receipt/stripe-sig.ts new file mode 100644 index 00000000..889d8eeb --- /dev/null +++ b/docs/scenarios/proofs/webhook-receipt/stripe-sig.ts @@ -0,0 +1,102 @@ +// USER CODE — the half StitchAPI does not have. A Stripe-shaped webhook signature, signed and +// verified over the RAW request bytes with `node:crypto`. +// +// This file exists in the proofs because nothing in `packages/*/src` does it (C2), so every claim +// that needs a genuine signature has to bring one. It is also the subject of C7's line count: this +// is what you write yourself, and it is the boundary made concrete. +// +// The scheme is Stripe's, because it is the one the capture names: +// +// Stripe-Signature: t=,v1= +// signed payload = `${t}.${rawBody}` ← the RAW bytes, not a re-serialised object +// verify = timing-safe compare, then |now - t| <= tolerance +// +// Three details are the ones that are easy to get wrong, and all three are load-bearing here: +// 1. The MAC covers `${t}.${raw}` — the timestamp is INSIDE the MAC, so it cannot be moved. +// 2. The compare is `timingSafeEqual`, which throws on a length mismatch — hence the length +// guard before it (a 63-char `v1=` would otherwise crash the handler rather than reject). +// 3. The tolerance check is separate from the MAC check. A valid MAC on a 2-hour-old timestamp +// is a replay, and passing the MAC is not enough to accept it. +import { createHmac, timingSafeEqual } from 'node:crypto'; + +/** Default replay tolerance, in seconds — Stripe's own default. */ +export const DEFAULT_TOLERANCE_SECONDS = 300; + +/** Why a delivery was rejected. `ok` carries the reason for the assertions to read. */ +export type VerifyResult = + | { ok: true } + | { + ok: false; + reason: 'malformed' | 'bad-signature' | 'stale' | 'future'; + }; + +/** + * Produce the header a provider would send for these exact bytes. Used by the fake provider and by + * every proof that needs a genuine signature; `raw` is a `Buffer` on purpose, so a caller cannot + * accidentally sign a re-serialised string without noticing. + */ +export function signPayload( + raw: Buffer, + secret: string, + timestampSeconds: number, +): string { + const mac = createHmac('sha256', secret) + .update(`${timestampSeconds}.`) + .update(raw) + .digest('hex'); + return `t=${timestampSeconds},v1=${mac}`; +} + +/** Parse `t=…,v1=…` into its parts. Unknown scheme versions are ignored, as Stripe's own does. */ +function parseHeader(header: string): { t: number; v1: string } | undefined { + let t: number | undefined; + let v1: string | undefined; + for (const part of header.split(',')) { + const eq = part.indexOf('='); + if (eq === -1) continue; + const key = part.slice(0, eq).trim(); + const value = part.slice(eq + 1).trim(); + if (key === 't') t = Number(value); + else if (key === 'v1') v1 = value; + } + if (t === undefined || !Number.isFinite(t) || v1 === undefined) + return undefined; + return { t, v1 }; +} + +/** + * Verify a delivery. `raw` MUST be the bytes off the socket — the whole point of the exercise. + * `nowSeconds` is injected so the tolerance boundary is testable without sleeping. + */ +export function verifySignature( + raw: Buffer, + header: string | undefined, + secret: string, + nowSeconds: number, + toleranceSeconds: number = DEFAULT_TOLERANCE_SECONDS, +): VerifyResult { + if (!header) return { ok: false, reason: 'malformed' }; + const parts = parseHeader(header); + if (!parts) return { ok: false, reason: 'malformed' }; + + const expected = createHmac('sha256', secret) + .update(`${parts.t}.`) + .update(raw) + .digest('hex'); + + // `timingSafeEqual` THROWS on differing lengths, so the length is compared first — in the + // clear, which leaks nothing a hex-digest length does not already tell an attacker. + const got = Buffer.from(parts.v1, 'utf8'); + const want = Buffer.from(expected, 'utf8'); + if (got.length !== want.length) + return { ok: false, reason: 'bad-signature' }; + if (!timingSafeEqual(got, want)) + return { ok: false, reason: 'bad-signature' }; + + // A correct MAC on an old timestamp is a REPLAY, which is why this check is not folded into + // the one above. The future arm catches a badly-skewed sender rather than an attack. + const drift = nowSeconds - parts.t; + if (drift > toleranceSeconds) return { ok: false, reason: 'stale' }; + if (drift < -toleranceSeconds) return { ok: false, reason: 'future' }; + return { ok: true }; +} diff --git a/docs/scenarios/provider-failover.md b/docs/scenarios/provider-failover.md new file mode 100644 index 00000000..688de241 --- /dev/null +++ b/docs/scenarios/provider-failover.md @@ -0,0 +1,155 @@ +# Scenario: failing over to the backup provider + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `provider-failover` + +**Verification:** 8 proof scripts, run offline (165 checks), in +[`proofs/provider-failover/`](proofs/provider-failover/). Published page: +[`scenarios/provider-failover.mdx`](../../apps/docs/content/docs/scenarios/provider-failover.mdx). +Escalated: [`issue-drafts/any-is-priced-as-a-hedge.md`](issue-drafts/any-is-priced-as-a-hedge.md). + +| Claim | Verdict | Measured | +| ------------------------------------ | ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — does `any` call both on success | **confirmed, worse than predicted** | 10 successful primary calls → `[10, 10]`, **20 requests for 10 answers**; loser **completed** (0 aborted); `any` has **no preferred member** — a 10 ms-slower healthy primary lost | +| C2 — sequential fallback | expressible; capture wrong pessimistically | `linked` + try/catch → `[10, 0]`, **one traceId** `primary ← root, backup ← primary`; bare try/catch → 2 unrelated root traces | +| C3 — classification | no built-in; aggregate worse than predicted | `AggregateError` has `status` **and** `body` `undefined`; the 400 survives only in `.errors[0]`. `race` _does_ surface a real 400 — and is unusable as failover | +| C4 — one input for every member | shared, and it splits config/input | declared auth stayed per-member; a **per-call `authorization` for the primary arrived at the backup verbatim** | +| C5 — winner identity | not recoverable | result is the raw body; `pick` destroys attribution; group emits **0** events; a cancelled member emits nothing terminal | +| C6 — cancellation | cancelled ≠ free | loser's request always arrives; billed for the winner's latency; two equal providers → **80 for one answer, 0 saved** | +| C7 — hedging amplification | unconditional | `race` **2.00×** healthy _and_ degraded; a breaker is a health gate not a budget gate; two `url`-only stitches shared one breaker | +| C8 — assembled | PASS | `[10, 0]` healthy, 400 stops the chain, 503 fails over, one trace tree — **30 lines vs 104** | + +**Hypotheses: the headline held, two were wrong.** + +- C1's prediction was right and understated. The unpredicted parts: the loser **completes** + (the abort is in a `finally` after the winner settles), and `any` prefers the _faster_ member, + not the first — so it silently routes away from the provider you chose. +- "Sequential fallback may be the one shape the library doesn't offer" — **wrong + pessimistically**. `linked` + try/catch is the correct default and carries the trace properly. +- The `AggregateError` guess was right about the loss and wrong about the scale: it drops + `body` too, so _nothing_ a catch block routes on survives. + +**The framing worth keeping:** the library carries ~74% of this scenario and **all of it is per +member** — auth, retry, breaker, timeout, normalisation, trace identity. Its contribution to the +routing _between_ members is zero, and the 30 lines that fill the gap cannot be given back: +`Composable` is not user-authorable, so a hand-branded node compiles and then throws. + +--- + +## The use case + +You depend on a provider that will eventually be down: an LLM API, an SMS gateway, a payment +processor, a geocoder. So you line up a second one of the same shape and fail over when the +first fails. + +Two different techniques wear similar clothes: + +- **Failover** — try the primary; on failure, try the backup. One call in the happy path. +- **Hedging** — send to _both_ immediately, take whichever answers first. Two calls, always, + in exchange for a better tail latency. + +Choosing the wrong one is a bill, not a bug report. + +## Why it is not straightforward + +**The trigger has to classify the failure, not just notice it.** The consensus is sharp and +consistent: `404`, `429` and `5xx` are _availability_ errors — try the next provider. A `400` +is a _bad request_ — stop the chain, because your payload is malformed and the next provider +will reject it identically. A failover that treats all failures alike turns one bad request +into N bad requests, N bills, and an aggregate error that hides the actionable one. + +Then the shape-specific hazards: + +- **Hedging amplifies outages.** When a backend degrades, _every_ request crosses the hedge + threshold, so every request doubles — [doubling traffic to a backend that was already + failing](https://blog.alexoglou.com/posts/hedging/). Hedging is only safe coupled to a + circuit breaker. +- **Hedging requires idempotency.** Two in-flight copies of a non-idempotent write is two + writes. Cancelling the loser is a latency optimisation, not a correctness one — the request + may already have landed. +- **Cancelling doesn't refund.** With an LLM, a cancelled request is still billed for the + tokens generated before the cancel. "The loser is auto-cancelled" saves latency and not money. +- **The providers aren't actually interchangeable.** Different auth, different rate limits, + different response shapes, different error vocabularies — so each leg needs its own config, + and the results need normalising before the caller sees them. +- **You need to know who served it.** Cost attribution and telemetry both need the winning + provider's identity, and a combinator that returns "the value" tends to lose it. + +## Evidence this bites real projects + +- **The classification rule** is stated the same way across the ecosystem — + [Bifrost](https://dev.to/kuldeep_paul/adaptive-model-routing-and-fallback-logic-routing-around-llm-provider-outages-with-bifrost-4g3m), + [MixRoute](https://mixroute.ai/blog/handle-llm-api-failures/), + [Portkey](https://portkey.ai/blog/failover-routing-strategies-for-llms-in-production/): + availability errors fail over, a `400` must stop the chain. +- **Provider identity on the response** is called out as essential for cost attribution — you + cannot bill or debug what you cannot attribute. +- **Hedging's outage amplification** is the standard warning in every write-up on it + ([Costa](https://blog.alexoglou.com/posts/hedging/), + [OneUptime on Envoy](https://oneuptime.com/blog/post/2026-02-09-envoy-request-hedging/view)), + along with "hedge only idempotent operations". +- **A whole product category exists for this** — OpenRouter, Portkey, Bifrost — which is itself + the evidence that rolling it yourself is not a one-liner. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| **Sequential fallback** | `try A; catch { try B }`. | The correct default: one call in the happy path. Adds the primary's full timeout to the failure path's latency. | +| **Concurrent "first success"** | Fire both, take the first that works. | Best latency and **double spend on every call** — including the 99% that didn't need it. | +| **Hedge after a delay** | Fire the backup only if the primary is slow. | The nuanced answer. Needs a threshold, and amplifies an outage exactly when you can least afford it. | +| **Gateway / router** (OpenRouter, Portkey) | Someone else owns the routing. | Complete, and a third party in the path plus a bill. | +| **Retry, not failover** | Just retry the primary harder. | Right for a `429` or a blip; useless when the provider is genuinely down. | +| **Classify then route** | Availability errors fail over, `400`s stop. | What everyone converges on, and the part hand-rolled failover usually skips. | + +**Summary of the state of the art:** classify the error before routing, prefer sequential +fallback unless latency genuinely justifies hedging, hedge only idempotent calls, couple +hedging to a breaker, and record which provider served the request. + +--- + +## What to verify against StitchAPI + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **`any()` is documented as failover but implemented as concurrent.** Its docstring + (`pipe.ts:275-281`) says _"failover across interchangeable sources… a primary and a mirror, + two regions, two providers"_ — the vocabulary of fallback — while the first line says **"Run + nodes CONCURRENTLY."** If that holds, using `any` for provider failover calls **both** + providers on **every** call. For an LLM that is double spend on the happy path, and the + documented auto-cancellation does not help, because tokens already generated are billed + (established in [scenario 5](mid-stream-failure.md)). +- **`race()` is the hedge** — first to settle, winner or loser (`pipe.ts:296-300`). +- **Neither classifies.** `any` "waits past failures for a success", so a `400` from the primary + makes it wait for the backup's `400` too, and surfaces an `AggregateError` rather than the + actionable bad-request error. +- **There is no sequential-fallback combinator.** `all`/`any`/`race` are all concurrent; + `linked` is sequential but is a scope for plain `await`s, not a fallback. So the correct + default may be the one shape the library doesn't offer. +- Scenario 7 measured that `all()` hands **every member the same input** and bounds nothing — + worth checking whether `any`/`race` share that. +- Scenario 9 measured the circuit breaker is shared by default — which matters here, since + hedging is only safe with a breaker. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Does `any()` call every member on a **successful** primary? + Measure requests reaching each provider on a happy path. If both are called, that is the + cost finding. +2. **C2** — sequential fallback: is it expressible at all? Try `linked`, plain `try/catch`, + `any` with a delayed member. Measure calls to the backup when the primary succeeds. +3. **C3** — classification: can failover be made to trigger on `429`/`5xx` but **not** `400`? + Measure what the caller receives for a `400` — the actionable error, or an aggregate? +4. **C4** — do `any`/`race` share `all()`'s one-input-for-every-member behaviour? Two providers + with different auth and different paths is the normal case. +5. **C5** — is the winner's identity recoverable? Cost attribution needs it. +6. **C6** — cancellation: is the loser actually cancelled, and does the cancelled request still + reach the provider? (It will — the question is what the proof measures at the server.) +7. **C7** — hedging safety: does `race` amplify against a degraded backend, and can a breaker + be scoped to just the hedge? +8. **C8** — assemble the best available answer for "primary with a backup", run it, report the + seam and line count. + +C1 decides whether the docstring's framing is safe. "Failover" that costs double on every +successful call is a materially different product from what the word implies. diff --git a/docs/scenarios/stale-fixture.md b/docs/scenarios/stale-fixture.md new file mode 100644 index 00000000..a8bf05a8 --- /dev/null +++ b/docs/scenarios/stale-fixture.md @@ -0,0 +1,158 @@ +# Scenario: the mock that passed for six months + +**Researched:** 2026-08-05 · **Status:** ✅ verified (8 claims, 168 checks, offline) · page shipped +**Slug:** `stale-fixture` + +--- + +## The use case + +You integrate a vendor API. You write tests. You cannot call the real API on every CI run — it is +slow, rate-limited, costs money, and mutates state — so you test against something fake. + +Then the vendor changes the API, and **your tests keep passing.** + +## Why it is not straightforward + +This is the only scenario in the pass where the failure mode is _the test suite actively +lying to you_. Every available approach trades one form of wrongness for another. + +- **A recording goes stale silently.** VCR-style cassettes are the standard answer, and the + standard failure is that the cassette records a response that no longer exists. The classic + shape: a login cassette records a session token; a day later a test needing a valid token is + matched against that recording and gets an expired one — so the failure surfaces somewhere + unrelated, if at all. The guidance is to _"plan cassette regeneration workflow when external + APIs change; automate deletion and re-recording to prevent stale cassettes masking real API + issues"_ — i.e. the mitigation is a process you must remember to run. +- **A hand-written mock encodes your beliefs.** It is wrong in exactly the way your code is + wrong, because the same person wrote both from the same reading of the docs. A mock cannot + catch a misunderstanding; it can only preserve it. +- **The sandbox is not production.** Providers maintain sandboxes _"for initial development, not + for continuous regression testing"_ — they rarely support every edge case and do not reflect + recent production changes. They are also slow (real network) and often rate-limited. +- **Contract testing needs the vendor.** Pact and friends work when both sides participate. A + third-party vendor is not going to run your provider verification. +- **Time makes it worse.** Retry, backoff, timeout and circuit behaviour is where integration + bugs actually live, and testing it against a real clock gives you a suite that is slow, + flaky, and imprecise. `Thread.sleep`-based retry tests are the canonical example. + +The consensus is that no single approach is sufficient — you need virtualisation for volume +_and_ live verification for accuracy. Which means the real question for a client library is +narrower and sharper: **can it tell you your fake has drifted from the real thing?** + +## Evidence this bites real projects + +- **Stale cassettes** — [HTTP testing in R, ch. 6 (vcr)](https://books.ropensci.org/http-testing/vcr.html) + on the expired-token shape, and the standing advice to re-record on a schedule. +- **Sandbox parity** — [Keploy on sandbox testing](https://keploy.io/blog/community/sandbox-testing): + sandboxes exist for initial development, not continuous regression, and lag production. +- **Neither alone is enough** — [Signadot on mocks vs sandboxes](https://www.signadot.com/blog/mocking-and-testing-3rd-party-apis-with-sandboxes/): + service virtualisation for coverage at volume _plus_ live contract verification for accuracy. +- **Time-based tests** — [TimeProvider in .NET 8](https://www.eriklieben.com/posts/net8_timeprovider_for_unit_tests/) + and [Go's `testing/synctest`](https://huncoding.com/go-synctest-testing-concurrent-code-en/): + the industry has converged on injecting a clock precisely because retry/backoff tests are + otherwise unusable. Kafka and Flink both rolled their own. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ---------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------- | +| **Record/replay cassettes** | Record real traffic once, replay forever. | Goes stale silently. Re-recording is a process, not a check. | +| **Hand-written mocks** | Fixtures you write from the docs. | Encode your misunderstanding faithfully. Cannot catch what you got wrong. | +| **Vendor sandbox** | The vendor's test environment. | Lags production, misses edge cases, slow, rate-limited. Not built for regression runs. | +| **Contract testing (Pact)** | Both sides verify a shared contract. | Needs vendor participation. Not available for third parties. | +| **Hit production in CI** | The only truly accurate option. | Slow, costly, mutating, rate-limited, and flaky for reasons unrelated to your code. | +| **Schema/contract snapshot** | Validate responses against a schema. | The honest middle ground — but only if the _same_ schema guards prod and the fixtures. | + +**Summary of the state of the art:** use a fake for speed and a periodic real call for truth, +and make the gap between them detectable rather than hoping someone re-records. + +--- + +## What to verify against StitchAPI + +`stitchapi/testing` is a substantial module and this pass has never given it a dedicated run. +It exports `mockAdapter`, `stubStitch`/`failStitch`, `collectStitchEvents`, stream/SSE +fixtures (`streamOf`, `streamThenError`, `gatedStream`, `sseStream`), `manualClock`, and a set +of **contract verifiers** (`verifyStoreContract`, `verifyAdapterContract`, `verifySinkContract`, +`verifyFingerprintContract`). + +Note the split: the verifiers are aimed at people writing **plugins**, not people writing +**integrations**. Whether the integration half is as well served is the question. + +**And there is a specific, pre-registered suspicion.** [Pattern 2b](LEDGER.md) of this pass +records that **three time-driven features ignore the injected clock** — `timeout.total` +(scenario 4), `cache.ttl` (scenario 6) and SigV4 signing (scenario 14) — each found +incidentally while testing something else. If `manualClock` is the answer to the +slow-flaky-retry-test problem, then a test of `timeout.total` written with it **passes +vacuously**, which is the worst possible failure for a testing tool. This scenario should +settle the scope of that once, deliberately, instead of accumulating a fourth accidental +sighting. + +**Claims to test with runnable offline code:** + +1. **C1** — **DECIDING CLAIM.** Can a fixture be caught when it goes stale? If the same `output` + schema guards production and the test double, does a drifted fixture fail? Try: a field + removed, renamed, retyped, and nulled. +2. **C2** — **DECIDING CLAIM.** Is `manualClock` sound across every time-driven feature? + Enumerate `retry` backoff, `throttle`, `circuit.cooldown`, `timeout` (per-attempt **and** + `total`), `cache.ttl`, `paginate`. For each: does `advance()` drive it, or does it read + wall-clock? A feature that ignores the clock makes a test that _passes without asserting + anything_ — measure that explicitly. +3. **C3** — what does `mockAdapter` actually check? Does it validate that the fixture is a + well-formed `AdapterResponse`, or will it happily serve a shape the real adapter never + produces? +4. **C4** — can resilience be tested **without** a vendor? Assert attempt counts, backoff + delays, circuit transitions, throttle spacing — using `collectStitchEvents`. +5. **C5** — `stubStitch`/`failStitch`: can a _caller_ of a stitch be tested without the + engine? Does the stub honour the same input contract as the real stitch? +6. **C6** — the sandbox-parity question: can one stitch be pointed at sandbox and prod so the + difference is visible? Is there a spelling for "run this against the real API weekly"? +7. **C7** — streams: `streamOf`/`streamThenError`/`gatedStream` — is mid-stream failure + (scenario 5) testable deterministically? +8. **C8** — assemble the best available "my fixtures cannot silently rot" setup; report seams + and line count. + +C1 and C2 decide this. C1 is the scenario's actual question. C2 is the pre-registered +suspicion, and if `manualClock` is unsound anywhere the testing guide needs to say so. + +--- + +## Verification result + +**All 8 claims verified**, 168 checks across 8 scripts, re-run by me before writing up. + +| Claim | Verdict | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — can a stale fixture be caught? | **PARTIAL — and the missing half is the scenario.** Fixture-drifts-from-schema: 4/4 caught. Vendor-drifts-while-fixture-holds: test `ok: true`, prod `ok: false`, 5 keys different, **nothing offline detects it** | +| C2 — is `manualClock` sound? | **CONFIRMED, and wider than recorded — 6 wall-clock, not 3** | +| C3 — what does `mockAdapter` check? | Almost nothing; and it **fails the library's own adapter contract** | +| C4 — resilience without a vendor? | **Fully testable.** The strongest result in the scenario | +| C5 — `stubStitch` input contract | Runs **none** of the input schemas; plus a `.safe()` bug | +| C6 — sandbox parity | Targeting yes (3 targets, one `extends`), scheduling no | +| C7 — streams | **Fully deterministic** — 5 runs, 1 outcome, byte-identical | +| C8 — assembled | 98 lines, 5 seams, closes 4 of 5 | + +### Hypotheses that were wrong + +**My own framing of the vacuity mechanism.** I predicted `timeout.total` "ignores the clock". It +does not: the per-attempt clamp fires on virtual time, and the wall-anchored part is the +_deadline_, so the budget **resets** each attempt rather than being ignored. The measured +outcome (a 1000ms budget surviving 2700 virtual ms) matches the prediction; the mechanism does +not. Predicting an outcome correctly for the wrong reason is the subtler failure and worth +recording as such. + +**"The shared `output` schema is the honest middle ground."** True for fixture-vs-schema drift, +and structurally blind to vendor-vs-schema drift — which is the scenario. The thing I proposed +as the answer solves the adjacent problem. + +**"`drift()` might help find a stale fixture."** It helps you _read_ a failure, never _find_ one. +ADR 0015 removed snapshot drift deliberately — "a single snapshot is one observation" — so +there is no cross-call baseline by design. + +**"`paginate` is a time-driven feature to check."** It has no time in it. + +### Outputs + +- Page: [stale-fixture.mdx](../../apps/docs/content/docs/scenarios/stale-fixture.mdx) +- Draft: [testing-kit-clock-gaps-and-two-bugs](issue-drafts/testing-kit-clock-gaps-and-two-bugs.md) diff --git a/docs/scenarios/unconfirmed-write.md b/docs/scenarios/unconfirmed-write.md new file mode 100644 index 00000000..4bdb68d8 --- /dev/null +++ b/docs/scenarios/unconfirmed-write.md @@ -0,0 +1,147 @@ +# Scenario: the charge you can't confirm + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `unconfirmed-write` + +**Verification:** 8 proof scripts (120 checks), run offline against a fake vendor that owns a +charge ledger, so every number below is counted rather than inferred. In +[`proofs/unconfirmed-write/`](proofs/unconfirmed-write/). Published page: +[`scenarios/unconfirmed-write.mdx`](../../apps/docs/content/docs/scenarios/unconfirmed-write.mdx). +Escalated: [`issue-drafts/idempotency-default-is-not-restart-safe.md`](issue-drafts/idempotency-default-is-not-restart-safe.md). + +| Claim | Verdict | Measured | +| ------------------------------- | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| C1 — same key every attempt | **PASS, in the library's favour** | 3 attempts → 1 key, 1 charge; a response lost _after_ processing was **recovered** by the retry replaying the stored 200 | +| C2 — default key restart-stable | **FAIL — deciding, capture confirmed** | re-driven job → **2 keys, 2 charges for 1 intended**; `keyOf` → 1 and 1 | +| C3 — derived key stability | PASS with a sharp caveat | `JSON.stringify(body)` moved on **key order alone** → 2 charges, statuses `[200,200,200]`, **no 409** | +| C4 — cached failure | **capture REFUTED** | a stored 500 is **not** retried by default — 1 request under `attempts: 4` | +| C5 — key/body mismatch | PASS | 409 not retried, `idempotency_key_in_use` on `error.body`; but `verdict: {accept, flag}` swallows it | +| C6 — TTL expiry | **FAIL** | 25 h vs a 24 h TTL → **2 charges**, clean `200`, **no replay marker**, no client-side signal | +| C7 — timeout ambiguity | **FAIL** | dropped request (0 charges) and lost response (1 charge) → **field-for-field identical** errors; no event carries the key | +| C8 — assembled | PASS | **5 charges / 6 intended** (sixth declined) vs the default's **8 / 6** with two duplicates | + +**The capture's central worry was right, and one hypothesis was wrong in the library's favour.** +The default key is `randomUUID()` per call (`engine.ts:172`, inside `buildRequest` at `:257`), so +a queue re-driving a job mints a new key and charges again. But the cached-500 burn the capture +also feared does **not** happen — 500 isn't in the default `retry.on`. + +**The sharpest detail is in the guard, not the gap.** `stitch.ts:386` warns only when there is a +random key **and no `retry`** — the reasoning being that a random key does protect the retries +inside one call, which is true. The effect is that following the warning's own advice ("add +`retry`") silences it, while leaving the restart case open. The message's own wording — "only +dedupes its own retries" — is accurate and is exactly the limitation, so the fix may be a +re-wording rather than a behaviour change. + +**And a genuinely subtle one neither side anticipated:** a _stable_ key makes a recorded failure +**sticky** for the whole TTL — a declined card stayed declined — while the random key never +reaches the record and simply charges again. Both are defensible; nothing selects between them. + +--- + +## The use case + +You POST a charge. The connection times out. **You have no idea whether the money moved.** + +Retry and you may double-charge. Don't, and the customer may have paid for nothing. There is +no third option that involves guessing. + +## Why it is not straightforward + +**A timeout tells you nothing about the server.** It does not distinguish "the request never +arrived" from "it was processed and the response was lost". That is the whole problem, and no +amount of client-side care removes it — you can only make the _retry_ safe. + +Idempotency keys do that, and then bring their own edges: + +- **The replay returns the _original_ outcome, including a failure.** Stripe stores the status + and body of the first request for a key _whether it succeeded or failed_, so a retry after a + cached `500` returns that same `500` forever. A retry policy that keeps trying is burning + attempts against a recording. +- **The key has a TTL, and it is shorter than your job queue.** Stripe prunes after ~24 hours + and then treats the key as fresh. _"If a client retries 25 hours later because of a delayed + job, a 1-hour TTL means they get double-charged."_ +- **The same key with a different body is an error.** Stripe compares the parameters and rejects + a mismatch. So a payload rebuilt with anything volatile — a timestamp, a re-serialised map + with different key order — turns a safe retry into a hard failure. `stripe-ruby#431` exists + because the client didn't handle the resulting `409`. +- **The key must survive the process, not just the call.** A key minted per call is retry-safe + _within_ that call and useless if the process dies and a queue re-drives the job. The second + run mints a new key, and the second key is a second charge. +- **Recovery has its own race.** "Query to see whether it landed" is the standard fallback, and + between the query and the decision the original request can still land. + +## Evidence this bites real projects + +- **Stripe's own reference** — [idempotent requests](https://docs.stripe.com/api/idempotent_requests): + the stored result is replayed _"regardless of whether it succeeds or fails"_, keys are pruned + after ~24 hours, and a reused key after pruning generates a new request. +- **`stripe-ruby#431`** — ["Stripe.request does not retry on a 409 response"](https://github.com/stripe/stripe-ruby/issues/431): + the client-side gap around key conflicts. +- **`medusajs#4798`** — [Stripe webhook processing fails with 409 Conflict](https://github.com/medusajs/medusa/issues/4798). +- **Brandur's [implementing Stripe-like idempotency keys](https://brandur.org/idempotency-keys)** + is the canonical write-up of the server side, including the parameter-comparison rule. +- **The TTL trap** — a 25-hour retry against a 1-hour TTL double-charging — is called out + explicitly in the practitioner guidance, along with the expiry race (two requests arriving as + a key expires can both pass the existence check). + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| -------------------------------------------------- | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Retry blindly** | Treat a timeout like any failure. | Double-charges. The failure mode the whole scenario exists to prevent. | +| **Never retry a write** | Surface the timeout to a human. | Safe and expensive: every transient blip becomes a support ticket, and the customer still doesn't know if they paid. | +| **Idempotency key, minted per call** | A uuid per logical call. | Correct for retries inside the call; **useless across a restart**, which is exactly when a queue re-drives the job. | +| **Idempotency key derived from the business fact** | Hash the order id / invoice ref. | The right answer — stable across processes, restarts and queues. Requires a genuinely unique business key, and colliding keys merge two different writes into one. | +| **Query-then-decide** | On timeout, look for the record; create only if absent. | The standard recovery, and racy: the original can land between the query and the decision. Needs the query to be authoritative and the create to still carry the key. | +| **Persist intent first** | Write "I am about to charge X" locally, then charge, then mark done. | The durable answer, and now you own a two-phase workflow and its own recovery. | + +**Summary of the state of the art:** derive the key from the business fact so it survives a +restart, keep the body byte-stable for that key, don't retry into a cached failure, and set the +TTL longer than your slowest retry path. + +--- + +## What to verify against StitchAPI + +The [`idempotent-writes` recipe](../../apps/docs/content/docs/recipes/idempotent-writes.mdx) +covers attaching a key. This scenario is about the cases where that isn't enough. +[Scenario 1](oauth2-refresh-token-rotation.md) noted in passing that the default key is _"a +random uuid generated once per call — already retry-safe"_, and `keyOf` derives a stable value +from the input. Retry-safe within a call is not the same as restart-safe. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **The default key is minted per call.** If so it survives a retry and _not_ a restart — and a + queue re-driving a job is the single most likely way this scenario actually happens. +- **`keyOf` should be restart-stable**, being a pure function of the input. Worth confirming, + including whether the _body_ it derives from is byte-stable (key order, number formatting). +- **The cached-failure interaction is untested.** If the vendor replays a stored `500`, does + `retry` burn every attempt against it? `retry.on` includes 500 by default in some + configurations, and scenario 14 measured a skew 403 being retried into the ground. +- Scenario 14 measured `auth.apply` runs per attempt and the throttle wait precedes it. The + parallel question here: **is the idempotency header applied per attempt, and is it the same + value each time?** + +**Claims to test with runnable offline code:** + +1. **C1** — is the same key sent on every attempt of one call? Measure the header across a retry. +2. **C2** — **DECIDING CLAIM.** Is the default key stable across a **process restart**? Build the + same stitch twice with identical input and compare. If it differs, a re-driven job + double-charges. +3. **C3** — is a `keyOf`-derived key restart-stable, and is it stable against **body + re-serialisation** (key order, number formatting, an added timestamp)? +4. **C4** — the **cached failure**: the vendor replays a stored `500` for the key. Does `retry` + burn all attempts against it? Can that be distinguished from a fresh `500`? +5. **C5** — the **key/body mismatch**: same key, changed body → the vendor's `409`/`422`. Is it + retried (it must not be), and does the caller get something actionable? +6. **C6** — the **TTL expiry**: a retry after the key is pruned creates a second charge. Is there + anything client-side that could notice — and can a `keyOf` key be made to encode the intent so + the second charge is at least detectable? +7. **C7** — the **timeout itself**: can the caller distinguish "never sent" from "sent, outcome + unknown"? What do `attempts`, the error, and the event stream carry? +8. **C8** — assemble the safest available answer: derived key, no retry into a cached failure, + and a recovery path. Report the seam and line count. + +C2 decides this one. A key that doesn't survive a restart protects against the failure mode you +can see and not the one that costs money. diff --git a/docs/scenarios/unstable-pagination.md b/docs/scenarios/unstable-pagination.md new file mode 100644 index 00000000..6d7f86cd --- /dev/null +++ b/docs/scenarios/unstable-pagination.md @@ -0,0 +1,159 @@ +# Scenario: the page that moved while you were reading it + +**Researched:** 2026-08-05 · **Status:** VERIFIED — achievable with user code · page shipped +**Slug:** `unstable-pagination` + +> ⚠️ **The causation below is BACKWARDS. Read the corrected version.** This capture said insert +> → skip and delete → duplicate. It is the reverse, and the proofs establish it against ground +> truth. Left uncorrected in the body as the record of what was assumed; the table and the +> published page carry the right version. + +**Verification:** 8 proof scripts, run offline (213 checks), in +[`proofs/unstable-pagination/`](proofs/unstable-pagination/). Published page: +[`scenarios/unstable-pagination.mdx`](../../apps/docs/content/docs/scenarios/unstable-pagination.mdx). +Escalated: [`issue-drafts/paginate-cannot-report-a-partial-run.md`](issue-drafts/paginate-cannot-report-a-partial-run.md). + +| Claim | Verdict | Measured | +| ------------------------------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| C1 — insert behind the cursor | **REFUTED IN DIRECTION** | not a skip — a **duplicate**: `skipped []`, `duplicated ["r04"]`, 11 items for 10 rows. An offset insert can _never_ cause a skip | +| C2 — delete behind the cursor | **REFUTED IN DIRECTION, worse** | not a duplicate — a **skip**: `skipped ["r05"]`. And `total` dropped to 9 at the same instant, so length === total === 9 and the cheap check detects nothing | +| C3 — non-unique sort key, zero writes | confirmed, sharpest case | `skipped ["r05"]` **and** `duplicated ["r03"]` on a static collection; they cancel, so only the _deduped_ count (9 vs 10) fires | +| C4 — keyset via `next` | confirmed, 4 lines | `skipped []` / `duplicated []` on every workload that broke offset — but a composite cursor against a vendor that doesn't `ORDER BY` it still lost `["r03"]` | +| C5 — detection seams | confirmed **with a trap** | deduping in `items`/`transform` emptied a page → break → **6 rows lost by the fix**. `output` is the safe seam | +| C6 — `total` reconciliation | reachable; misses the case that matters | raw length-vs-total fired **0 of 4**; nothing fired on the delete while `r05` was gone | +| C7 — drift vs the known edges | confirmed | drift emptied a page mid-run → `skipped ["r10","r11","r12"]`, and the shrunken total made the reconciler _agree_. `pages: 50` → 200 of 220 rows, `ok` | +| C8 — assembled | PASS | 0 false negatives / 3 false alarms over 8 workloads — **84 lines vs 74 hand-rolled** | + +**The first capture error of FACT rather than prediction.** Every prior scenario's wrong +hypotheses were about which primitive would carry the solution. This one had the mechanism of +the problem itself backwards, and would have shipped a page teaching the wrong causation. The +correct version: + +- **Insert behind the cursor** → rows shift to _higher_ indices → the next fixed offset lands on + a row already read → **duplicate**. +- **Delete behind the cursor** → rows shift to _lower_ indices → the next offset jumps past one + → **skip**. + +**The finding worth carrying:** the signal that catches the delete case is that the **declared +`total` moved** — not dedupe, not length-vs-total. A delete removes one row from the result and +one from `total` simultaneously, so every arithmetic check balances while a row is missing. + +--- + +## The use case + +You page through a collection to sync it — orders, tickets, contacts — `?offset=0&limit=100`, +then `100`, then `200`. Meanwhile the collection is _live_: rows are being inserted, deleted, +and edited by other people. + +You end up with a list. It is quietly wrong. + +## Why it is not straightforward + +**Offset is a position in a result set, not a position in the data.** Anything that changes the +result set between two requests moves every row after it: + +- **An insert before your offset** pushes every later row down one. The row that was last on + page 1 becomes first on page 2 — no, worse: the row that _would_ have been first on page 2 + is now last on page 1's territory, and you **never see it**. +- **A delete before your offset** pulls every later row up one, so the row you already read on + page 1 appears again on page 2. You process it **twice**. +- **A mutable sort field** does both: edit `updated_at` on a row you've already read and it + jumps ahead of your cursor. +- **A non-unique sort key** breaks it with _no writes at all_. Ties in `created_at` may come + back in any order, and the database is under no obligation to be consistent between two + queries — so rows can be skipped or repeated on a completely static table. + +The fix is **keyset (seek) pagination** — `WHERE (created_at, id) > (:last_ts, :last_id)` — with +a composite cursor so ties are broken by something unique. That is a _server_ capability. +**A client cannot make an offset API consistent.** If the vendor only offers `offset`/`limit`, +no amount of client cleverness produces a correct page sequence. + +What a client _can_ do, and this is where the difference lies: + +- **Detect duplicates** — the same id appearing on two pages is proof of drift. +- **Reconcile against a declared total** — most APIs return `total`; aggregating 987 of a stated + 1000 is a detectable gap. +- **Use the cursor when offered**, and never silently fall back to offset. + +The trap is that a client which **aggregates pages into one array** does none of that by +default: duplicates are silently included, and skips are invisible by construction. + +## Evidence this bites real projects + +- **ServiceNow** — [the REST pagination gotcha that _silently drops records_](https://vexpose.blog/2026/07/28/the-servicenow-rest-pagination-gotcha-that-silently-drops-records/): + a named, specific production failure of exactly this shape. +- **The failure taxonomy** is consistent across write-ups — + [Knit on pagination stability](https://www.getknit.dev/blog/how-to-preserve-api-pagination-stability), + [12 pagination failure modes you'll see in production](https://medium.com/@sparknp1/12-pagination-failure-modes-youll-see-in-production-8ed33658df7e), + [Cursor-based pagination: why your API is silently showing users wrong data](https://medium.com/@moksh.9/cursor-based-pagination-why-your-api-is-silently-showing-users-wrong-data-561f69c2040c). +- **CedarDB** — ["Offset considered harmful"](https://cedardb.com/blog/pagination/) on the + surprising complexity of SQL pagination. +- The **composite-cursor** rule (sort field **plus** a unique key) is the consistent + recommendation for the tie case. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| --------------------------------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| **Offset/limit, as offered** | The default. | Skips and duplicates under any concurrent write, and under ties with no writes at all. Silent. | +| **Keyset / seek pagination** | Cursor on `(sort, id)`. | The correct fix — and only available if the vendor implemented it. | +| **Snapshot / point-in-time read** | Ask the API for a consistent snapshot. | Ideal where offered (some exports do). Rare in REST. | +| **Sort by an immutable key** | Page by `id ASC` rather than `updated_at`. | Removes the mutable-sort-field case, and not the insert/delete cases. Also loses the ordering you wanted. | +| **Client-side dedupe by id** | Drop repeats as you aggregate. | Cheap, and fixes only _half_ — duplicates go away, skips remain invisible. | +| **Reconcile against `total`** | Compare what you got to what the API claimed. | The only cheap way to _detect_ a skip. Doesn't repair it, and `total` itself moves. | +| **Re-sync from scratch** | Periodically page the whole collection again. | The practical safety net for a sync job, at full cost every time. | + +**Summary of the state of the art:** use keyset if the vendor offers it; if not, dedupe by id, +reconcile against the declared total, and treat a mismatch as a signal to re-sync. The +distinctive thing is that **correctness is not achievable client-side** — only _detection_ is, +and detection is what almost every client omits. + +--- + +## What to verify against StitchAPI + +This is `paginate`'s third appearance in this section, and deliberately from a different angle: +[scenario 3](batch-partial-failure.md) and [scenario 4](async-job-polling.md) tested its +_mechanics_. This one is about whether the **data it hands you is right**, and whether it can +tell you when it isn't. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- `paginate` aggregates every page's `items` into one array (`types.ts:1412-1422`). Nothing in + its three fields (`next`, `items`, `pages`) suggests dedupe or reconciliation, so the default + is likely "silently include the duplicate, silently omit the skip". +- `next(prevBody, pagesFetched)` receives the previous page's **raw body**, which should make + **keyset** pagination expressible — read the last item's `(sort, id)` and return it as the next + input. That is the one thing the library should do well here. +- Scenario 3 measured that a zero-item page ends the loop successfully with the remainder + unfetched, and scenario 4 measured that the default `items` wraps a non-array body as one + item. Both matter here: a drifted page could hit either. +- `total` reconciliation needs the caller to see a field from the **last** page's body — and + scenario 3 measured that `next` is never called on the terminal page, so the tail state may be + unreachable from inside the loop. + +**Claims to test with runnable offline code:** + +1. **C1** — model a live collection. With an **insert** before the cursor between page 1 and + page 2, does the aggregated array **miss** a record? Measure exactly which id is lost. +2. **C2** — with a **delete** before the cursor, does the aggregated array contain a record + **twice**? Does anything at all flag it? +3. **C3** — the **no-writes tie case**: a non-unique sort key where the server returns ties in a + different order per query. Measure skips/duplicates on a completely static collection. +4. **C4** — is **keyset** pagination expressible through `next`? Build it against a server that + offers `(created_at, id)` seek and measure that the same insert/delete workload produces a + correct, complete, duplicate-free result. +5. **C5** — can duplicates be **detected or removed** inside the library — via `items`, + `transform`, `output`, a surface? Measure what the caller can see. +6. **C6** — can the run be **reconciled against a declared `total`**? Is the last page's `total` + reachable from anywhere the caller can act on? +7. **C7** — does a drifted page interact with the two known `paginate` edges — the zero-item + break and the non-array `items` wrap? +8. **C8** — assemble the most honest answer: keyset where available, dedupe + reconcile where + not, and a signal the caller can act on. Report the seam and line count. + +C1–C3 establish the damage; C5 and C6 decide whether the library can _tell you_. Correctness +here is the server's to give — the honest question is whether a client that aggregates pages +can at least refuse to lie about the result. diff --git a/docs/scenarios/webhook-receipt.md b/docs/scenarios/webhook-receipt.md new file mode 100644 index 00000000..065c00c7 --- /dev/null +++ b/docs/scenarios/webhook-receipt.md @@ -0,0 +1,145 @@ +# Scenario: receiving a signed webhook + +**Researched:** 2026-08-05 · **Status:** VERIFIED — **split: receipt out of scope by design, reaction in scope** · page shipped +**Slug:** `webhook-receipt` + +**Verification:** 7 proof scripts, run offline against a real local `node:http` server, in +[`proofs/webhook-receipt/`](proofs/webhook-receipt/). Published page: +[`scenarios/webhook-receipt.mdx`](../../apps/docs/content/docs/scenarios/webhook-receipt.mdx). +Escalated: [`issue-drafts/void-call-drops-work.md`](issue-drafts/void-call-drops-work.md). + +| Claim | Verdict | Measured | +| ----------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| C1 — raw bytes at an arbitrary path | **out of scope** | 404 at all 5 paths; body pre-`JSON.parse`d; **inbound headers dropped entirely**; 162 signed bytes vs 153 after a round-trip | +| C2 — inbound signature primitive | **none** | 72 runtime exports enumerated; the 4 matches are BYO conformance suites; `aws-sigv4` key has `usages: ['sign']` | +| C3 — a seam on the serve path | **none** | `serve` exports 3 things; an **unsigned forged body ran the stitch, 200**; `createServeHandler` mounts anywhere but reading the stream to verify deadlocks `readBody` | +| C4 — fetch-on-receipt | partly | payload order moot (converged, 2 calls / 2 events); **write order is not** (concurrent v2/v3 landed on v2); `.deleted` → 404, indistinguishable from never-existed | +| C5 — `StitchStore` as dedup ledger | PASS | 3 deliveries → 1 side effect; `get`+`set` under concurrency → **[true,true,true]**, `increment` → **[true,false,false]** | +| C6 — fast 2xx | **no** | no queue/detach stage; `serve` acked only at attempt 3 after **exactly 10 virtual seconds**; `void call(input)` → **0 HTTP calls, 0 errors** | +| C7 — the boundary as a number | PASS | receipt **154 lines, zero `stitchapi` runtime imports**; reaction **63 lines**, mostly config | + +**The capture's framing held, which is a first — but three things it missed matter.** It +predicted `serve` wouldn't work and asked for the boundary to be stated precisely. Right. What +it did not anticipate: + +- **The inbound headers are dropped**, not just the raw bytes. Even with the bytes there is no + `stripe-signature` to verify against — the gap is wider than "body already parsed". +- **`serve` is unauthenticated**, so mistaking it for a webhook endpoint isn't a 404, it's an + open endpoint that runs your stitches on a forged body. +- **Fetch-on-receipt fixes payload order but not write order.** The capture treated it as the + clean answer to ordering; concurrent handlers still need a version guard. + +**The honest shape: 71% of the code by line, 100% by concern, is the half StitchAPI does not +participate in** — and that is by design. [`the-stitch.mdx:44`](../../apps/docs/content/docs/concepts/the-stitch.mdx) +already states it: inbound webhooks stay the application's job. This scenario's value is making +that boundary concrete and measured rather than asserted. + +--- + +## The use case + +Stripe, GitHub, Slack, Shopify — they all push events to an endpoint you host. A payment +succeeded; a PR opened; a subscription changed. You verify the signature, decide it's genuine, +and act on it. + +This is the other direction from every other scenario in this section: **someone is calling +you.** + +## Why it is not straightforward + +**Signature verification needs the exact bytes.** The provider signs the raw payload. If any +middleware parses the JSON before you get to it, re-stringifying gives _logically identical_ +JSON with different bytes — whitespace and key order shift — and verification fails. In Express +this is the notorious `express.raw()`-must-precede-`express.json()` ordering rule, and it is the +single most-reported webhook bug there is. + +Then the delivery semantics: + +- **At-least-once means duplicates are normal, not exceptional.** The sender can't distinguish + "you didn't process it" from "you did and the ack was lost", so it retries. Your handler must + be idempotent, keyed on the event id. +- **The dedup TTL must exceed the retry window.** Stripe retries for up to 3 days; a 1-hour + dedup cache is a duplicate waiting to happen. That means durable storage, not memory. +- **Order cannot be trusted.** `subscription.updated` can arrive before `subscription.created`. + The standard fix is to treat the webhook as a _hint_ and fetch current state from the API — + which turns an inbound problem into an outbound call. +- **Replay windows.** Stripe's signature header carries a timestamp; you reject deliveries + outside a tolerance (5 minutes by default) so a captured payload can't be replayed later. +- **Comparison must be constant-time**, or the HMAC check leaks via timing. +- **You must return 2xx fast.** Providers time out in seconds and retry on slowness, so any + real work has to be queued rather than done inline — which _creates_ the duplicate problem + you just solved, one layer down. + +## Evidence this bites real projects + +- **Stripe's own docs** have a page dedicated to it — [resolve webhook signature verification + errors](https://docs.stripe.com/webhooks/signature) — and the raw-body rule is its headline. +- The failure is common enough to have a genre of writeups: + [why yours keep failing](https://medium.com/@amanjaved421/why-your-stripe-webhooks-keep-failing-verification-in-n8n-a6eb53b4584b), + ["are you passing the raw request body?"](https://sukhadagholb.medium.com/webhook-signature-verification-for-stripe-are-you-passing-raw-request-body-received-from-stripe-3b2deed6a75d), + and [the payload must be a string or Buffer](https://dev.to/nerdincode/debugging-stripe-webhooks-in-nodejs-the-payload-must-be-a-string-or-buffer-error-4a60). +- **Delivery semantics** are documented consistently across providers and infrastructure + vendors — [Svix on idempotency and deduplication](https://www.svix.com/resources/webhook-university/reliability/idempotency-and-deduplication/), + [Hookdeck](https://hookdeck.com/webhooks/guides/implement-webhook-idempotency), + [Postmark on why idempotency matters](https://postmarkapp.com/blog/why-idempotency-is-important) — + all making the same two points: duplicates are guaranteed, and order is not. + +## The common solutions, and what each costs + +| Approach | What it is | Where it breaks | +| ------------------------------------------------------------ | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Provider SDK verifier** (`stripe.webhooks.constructEvent`) | The vendor hands you a verifier. | Correct, and the right answer for that vendor. One per provider, each with its own header format and tolerance. | +| **Framework raw-body middleware** | `express.raw()` scoped to the webhook route. | The standard fix, and entirely about _ordering_ — get it wrong and it fails silently in the direction of "invalid signature". | +| **Hand-rolled HMAC** | `createHmac` + `timingSafeEqual`. | Fine, and easy to get subtly wrong: constant-time comparison, the timestamp tolerance, and the exact string being signed all matter. | +| **Webhook gateway** (Svix, Hookdeck) | Offload receipt, dedup, retry and replay. | The most complete answer at scale. A third party in the path, and a bill. | +| **Dedup on the event id** | Store ids, skip repeats. | Necessary, and the TTL is the trap — it must outlive the provider's retry window. | +| **Fetch-on-receipt** | Treat the payload as a hint; read current state from the API. | Makes ordering irrelevant and is widely recommended. Costs an API call per event, and that call needs its own auth, retry and rate-limit budget. | + +**Summary of the state of the art:** verify raw bytes in constant time inside a timestamp +window, dedup on the event id with a TTL longer than the retry window, ack fast, and fetch +current state rather than trusting payload order. + +--- + +## What to verify against StitchAPI + +This is the first scenario where the honest answer may be **"that's not what this library is"** — +and if so, the useful output is a documented boundary, not a stretched workaround. + +Read of the working tree before verification — **hypotheses, to be confirmed or refuted by +running code**: + +- **`serve` is not a webhook receiver.** It exposes _your_ registry over a thin front door: + `GET /` lists stitch names, `POST /stitch/:name` runs one with the JSON body as its **input** + (`serve.ts:206,283`). Fixed routes, JSON-parsed body — the opposite of "hand me the raw bytes + at my own path". If that holds, the receipt half is out of scope by construction. +- **No HMAC verification primitive exists in core.** The only signing code in the repo is + `@stitchapi/aws-sigv4`, which signs **outbound** requests. Nothing verifies an inbound one. +- **The half that _is_ StitchAPI's job** is the reaction: fetch-on-receipt is an outbound call, + and it is the recommended fix for out-of-order delivery. `StitchStore` is also exactly the + shape a dedup ledger needs (`get`/`set` with TTL, pluggable, durable) — though scenario 4 + measured that the store is engine state, so whether a user can borrow it cleanly is open. +- `idempotency` exists but is for outbound writes — a different problem wearing the same word. + +**Claims to test with runnable offline code:** + +1. **C1** — can `serve` receive a POST at an arbitrary path with the **raw body bytes** + preserved? Measure what a handler actually gets. If the body is JSON-parsed before any user + code, say so and show the byte difference that breaks a signature. +2. **C2** — is there **any** inbound-signature primitive in the packages? Grep exhaustively. +3. **C3** — can a signature be verified at all through `serve` — e.g. by a surface, a hook, or + the `ServeBodyOptions` seam? Or must the user bring their own server? +4. **C4** — the reaction half: does fetch-on-receipt work well as a stitch, and does it actually + make out-of-order delivery moot? Model two events arriving reversed and show the outcome + with and without the fetch. +5. **C5** — can `StitchStore` serve as the dedup ledger — durable, TTL beyond the retry window, + usable from user code? Measure a duplicate delivery being skipped, and what happens at the + TTL boundary. +6. **C6** — does anything in the library help with the fast-2xx requirement (ack now, work + later)? +7. **C7** — assemble the most honest end-to-end answer: a real Node server for receipt, plus + StitchAPI for everything after it. Report which half is which, and the line count of each. + +The deliverable here is the **boundary**, stated precisely. If receipt is out of scope, the page +should say that plainly and point at what to use instead — that is more useful than a clever +workaround that makes a client library pretend to be a server.