diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e783f78..e26fda9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: PY - name: Distributions bundle the exact event ABI corpus (#144) run: | - for f in trace-v6-execution.ndjson trace-v6-lifecycle.ndjson runner-v10-refusal.ndjson; do + for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || { echo "wheel is missing droste/testing/fixtures/$f"; exit 1; } @@ -154,21 +154,21 @@ jobs: from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) source = Path(sys.argv[1]) - assert trace_v6_execution_ndjson() == (source / "trace-v6-execution.ndjson").read_bytes() - assert trace_v6_lifecycle_ndjson() == (source / "trace-v6-lifecycle.ndjson").read_bytes() + assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes() + assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes() assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-execution.ndjson" + cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson" + cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson" cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" @@ -319,7 +319,7 @@ jobs: PY - name: Distributions bundle the exact event ABI corpus (#144) run: | - for f in trace-v6-execution.ndjson trace-v6-lifecycle.ndjson runner-v10-refusal.ndjson; do + for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || { echo "wheel is missing droste/testing/fixtures/$f"; exit 1; } @@ -333,21 +333,21 @@ jobs: from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) source = Path(sys.argv[1]) - assert trace_v6_execution_ndjson() == (source / "trace-v6-execution.ndjson").read_bytes() - assert trace_v6_lifecycle_ndjson() == (source / "trace-v6-lifecycle.ndjson").read_bytes() + assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes() + assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes() assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-execution.ndjson" + cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson" + cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson" cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c76b23f..ee1c0ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,8 +80,8 @@ jobs: staging="droste-relay-$GITHUB_REF_NAME" mkdir -p "$staging/conformance" relay-dist cp src/droste/substrates/_relay/*.ts pyodide/README.md "$staging/" - cp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ + cp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$staging/conformance/" printf '%s %s\n' "$GITHUB_REF_NAME" "$GITHUB_SHA" > "$staging/DROSTE_VERSION" diff --git a/AGENTS.md b/AGENTS.md index 0714b90..49a5ad7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ evidence with that status rather than leaving the model to interpret prefixes. ## Trace ABI -- Every structured event is a strict Trace ABI v6 value. Stamp it exactly once +- Every structured event is a strict Trace ABI v7 value. Stamp it exactly once through `ExecutionContext`; do not emit raw or partially enveloped event dictionaries at host boundaries. - Treat every envelope/body, classification, and ordering change as an ABI @@ -189,7 +189,7 @@ evidence with that status rather than leaving the model to interpret prefixes. - A strict published event vocabulary/body change requires a Trace ABI bump and, when embedded in runner output, an atomic runner-protocol bump. Do not expand an old strict version in place or add a compatibility decoder in the engine. -- Trace ABI v6 usage is `resolved` only when both root and subcall scopes have +- Trace ABI v7 usage is `resolved` only when both root and subcall scopes have complete provider usage. Missing or malformed usage preserves any known counts, marks that scope `complete=false`, and makes terminal usage `kind="partial"`; never report conservative reservations as provider usage. @@ -239,7 +239,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial evidence rather than invalidating an otherwise valid envelope. - `reasoning_tokens` is a non-negative breakdown inside `completion_tokens`. Preserve it through internal usage copies, folds, and root/subcall - `ExecutionStats`, but do not add it to totals or Trace ABI v6 events. + `ExecutionStats`, but do not add it to totals or Trace ABI v7 events. Observation basis and reasoning usage are internal callback/accounting facts; do not add them to the public usage projections. @@ -272,7 +272,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial and settle conservatively rather than falling back to the start estimate. - `ExecutionStats` folds those same cache classes separately for root and - subcall scopes. Trace ABI v6 exposes them in every durable usage breakdown; + subcall scopes. Trace ABI v7 exposes them in every durable usage breakdown; complete scopes require the disjoint cache classes to fit inside inclusive input tokens, while partial scopes preserve independently validated counts. ModelRelay names the classes `cache_read_input_tokens` and diff --git a/UPGRADING.md b/UPGRADING.md index fcf7dbd..32bd56d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -13,6 +13,30 @@ consumers, and Pyodide-substrate integrations staging the Deno relay. ## Unreleased (post-0.21.1) +### Trace ABI v7 publishes answer-state checkpoints + +The new configurable `checkpoint` event carries the draft the engine currently +holds — `checkpoint_seq`, `draft`, `draft_chars`, `ready`, and an opaque +`payload` — after every executed step whose draft moved. A host that loses the +process (a watchdog kill, a crashed substrate) can render the last checkpoint +instead of nothing. It is retention-gated like every other content-bearing +event, so it reaches a terminal record only when named in +`TraceRetentionPolicy.retain`. + +`RLMConfig.checkpoint_payload_provider` is the optional callable that fills +`payload`. It returns any JSON object or `None`; the engine never inspects the +value and never schema-checks it. A provider that raises is reported through a +`RuntimeWarning` and the checkpoint carries `payload: null` — a checkpoint can +never fail a run. + +Hosts must accept Trace ABI 7 and update to the `trace_v7_*` conformance +fixtures (`droste.testing.trace_v7_lifecycle_ndjson` / +`trace_v7_execution_ndjson`, backing `trace-v7-lifecycle.ndjson` / +`trace-v7-execution.ndjson`). Scaffold manifests report `abis.trace: 7`, so +every manifest id changes; pinned ids must be re-derived. Strict v6 readers +reject the new event and the new version, so this is an atomic consumer +migration. + ## 0.21.1 (from 0.21.0) ### Streamed Responses preserve their terminal stop reason diff --git a/docs/architecture.md b/docs/architecture.md index d601ee3..8020424 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -293,14 +293,14 @@ exceptions use the same closed five-field shape as other preflight responses; run exceptions use the ordinary run response shape. Completed responses also carry the policy-resolved -[Trace ABI v6](trace-abi.md) `run_record`. Live events and terminal records use +[Trace ABI v7](trace-abi.md) `run_record`. Live events and terminal records use the same strict envelope and projection. Persistence remains a host I/O decision; the engine never opens a trace store. The Deno/Pyodide relay keeps its three process output concerns physically separate. fd1 carries exactly one unary response JSON line. A required `DROSTE_RELAY_EVENT_FD` names one inherited writable descriptor (fd3 by -convention) that carries canonical Trace ABI v6 NDJSON only. An external +convention) that carries canonical Trace ABI v7 NDJSON only. An external launcher also includes that number in `DENO_EXTRA_STDIO_FDS`, which Deno consumes at startup to register inherited descriptors above fd2. Passing an OS-level descriptor without this Deno marker leaves it unavailable to relay @@ -376,7 +376,7 @@ are versioned, each by a single integer: Protocol v9 embeds Trace ABI v5 and its transient cumulative `usage_progress` event at settled root/subcall boundaries. The bump prevents a v8 host from silently dropping the live accounting signal. - Protocol v10 embeds Trace ABI v6 and scaffold manifest v3, adding the + Protocol v10 embeds Trace ABI v7 and scaffold manifest v3, adding the required seventh `max_iterations` authorization field. The exact seven-field budget prevents a v9 host from silently submitting an unbounded loop contract. diff --git a/docs/budgets.md b/docs/budgets.md index 0b51df4..757214c 100644 --- a/docs/budgets.md +++ b/docs/budgets.md @@ -70,7 +70,7 @@ reservations fails loudly. ## Trace facts -Every mutation is a durable Trace ABI v6 `budget` event from +Every mutation is a durable Trace ABI v7 `budget` event from `source="budget_ledger"`: `reserve`, `commit`, `refund`, or `exhaust`. Mutation events carry `resource`, non-negative `amount`, and `call_id`. The terminal snapshot records the configured, consumed, and remaining vectors. diff --git a/docs/scaffold-manifest.md b/docs/scaffold-manifest.md index 116c76d..ed7d576 100644 --- a/docs/scaffold-manifest.md +++ b/docs/scaffold-manifest.md @@ -24,7 +24,7 @@ inference facts. "schema_version": 3, "engine": {"version": "0.10.6", "source_revision": null}, "abis": { - "kernel": 1, "capability": 1, "trace": 6, + "kernel": 1, "capability": 1, "trace": 7, "prompt_pack": 2, "provider": 4, "runner": 10 }, "prompt_pack": { diff --git a/docs/trace-abi.md b/docs/trace-abi.md index 7dc280a..9d2a5bd 100644 --- a/docs/trace-abi.md +++ b/docs/trace-abi.md @@ -1,4 +1,4 @@ -# Trace ABI v6 +# Trace ABI v7 Droste exposes one append-only event stream and one policy-resolved terminal `RunRecord`. The engine creates values; it does not choose a database or write @@ -15,7 +15,7 @@ does not change merely because a released fixture is added. ## Event envelope -Every event is a strict v6 value with these fields: +Every event is a strict v7 value with these fields: ```json { @@ -23,7 +23,7 @@ Every event is a strict v6 value with these fields: "seq": 1, "timestamp": "2026-07-14T05:00:00Z", "type": "progress", - "version": 6, + "version": 7, "persistence_class": "transient", "parent_run_id": "optional-parent", "depth": 0, @@ -48,7 +48,7 @@ The persistence class is exhaustive and fixed by event type: | Class | Event types | Rule | | --- | --- | --- | | `durable` | `usage`, `budget`, `policy`, `capability`, `done` | Always in the terminal record | -| `configurable` | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay` | Included only when named by `TraceRetentionPolicy.retain` | +| `configurable` | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay`, `checkpoint` | Included only when named by `TraceRetentionPolicy.retain` | | `transient` | `startup`, `progress`, `reasoning_delta`, `usage_progress` | Live delivery only; never in the terminal record | Retention governs the terminal record, not the live channel. `result` is @@ -56,7 +56,7 @@ always delivered once before `done`, even when it is not retained. `replay` is different: it is emitted only when the host explicitly selects replay retention. -## Exhaustive v6 bodies +## Exhaustive v7 bodies Every event body has a fixed top-level schema. Optional fields are marked `?`. Objects named below are JSON objects; all other types are primitive. @@ -75,6 +75,7 @@ Objects named below are JSON objects; all other types are primitive. | `repair` | `phase: "start"|"completion"|"failure"`, `kind: "missing_code"|"execution_error"|"terminal"`, `iteration`, `error?` only on failure | | `extract` | `phase: "start"|"completion"|"failure"`, `iteration`, `extract_error?` only on failure | | `result`, `replay` | `result: object` | +| `checkpoint` | `iteration: integer`, `checkpoint_seq: integer`, `draft: string`, `draft_chars: integer`, `ready: boolean`, `payload: object|null` | | `usage_progress` | `boundary: "root"|"subcall"`, `kind`, cumulative `root`, `subcall`, `unattributed`, and `total_tokens` | | `usage` | `kind: "resolved"|"partial"`, `root: object`, `subcall: object`, `unattributed: object`, `total_tokens: integer`, `wall_time_ms: integer` | | `budget` | `kind: "snapshot"|"mutation"`, `source: string`, plus the kind-specific fields below | @@ -118,7 +119,7 @@ partial observation preserves its reported counters and marks the affected scope incomplete. The last progress snapshot reconciles with terminal `usage` unless a later boundary has no numeric usage observation. -The budget body remains a discriminated event in Trace ABI v6. The terminal snapshot uses +The budget body remains a discriminated event in Trace ABI v7. The terminal snapshot uses `kind="snapshot"`, `source="budget_ledger"`, and `configured`, `consumed`, and `remaining` objects. The configured object includes the structural `max_iterations` ceiling; terminal `iterations` records how many iterations @@ -193,6 +194,24 @@ start and exactly one completion or failure once entered. Extract fallback does the same; a failure carries the typed `extract_error`. The canonical `result` still carries the unary-equivalent answer and `done` remains content-free. +## Answer-state checkpoints + +`checkpoint` publishes the answer state the engine holds right now, so a host +that loses the process still has the last draft it saw. The engine emits one +after every executed step whose draft moved, or whose host had something of its +own to add. `checkpoint_seq` starts at one and strictly increases within a +`run_id`; `draft_chars` always equals `len(draft)`. + +`payload` is opaque. Hosts fill it through `RLMConfig.checkpoint_payload_provider`, +a callable returning any JSON object or `None`. The engine carries the value +across without inspecting it and never schema-checks its contents, exactly as +the relay carries adapter `meta`. A provider that raises is reported and the +checkpoint carries `payload: null`: a checkpoint can never fail a run. + +The event is `configurable`, not `durable`. It carries draft content, so it +reaches a terminal record only when a host names it in +`TraceRetentionPolicy.retain`. + ## Terminal reconciliation Finalization emits resolved `usage`, `budget`, and `policy`; always delivers the @@ -211,12 +230,12 @@ and sdist. Python consumers load them through package resources: ```python from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) -execution_lines = trace_v6_execution_ndjson().splitlines() -event_lines = trace_v6_lifecycle_ndjson().splitlines() +execution_lines = trace_v7_execution_ndjson().splitlines() +event_lines = trace_v7_lifecycle_ndjson().splitlines() pre_admission_refusal = runner_v10_refusal_ndjson() ``` diff --git a/examples/pyodide-host/e2e_test.ts b/examples/pyodide-host/e2e_test.ts index 8910786..c5507d6 100644 --- a/examples/pyodide-host/e2e_test.ts +++ b/examples/pyodide-host/e2e_test.ts @@ -33,7 +33,7 @@ const RUNNER_REFUSAL_FIXTURE = new URL( import.meta.url, ); const TRACE_LIFECYCLE_FIXTURE = new URL( - "../../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); const TEST_BUDGET = { diff --git a/pyodide/README.md b/pyodide/README.md index fae960c..fea959b 100644 --- a/pyodide/README.md +++ b/pyodide/README.md @@ -243,7 +243,7 @@ The process has three independent output lanes: | Descriptor | Contract | |------------|----------| | fd1 | Exactly one unary response JSON line. Adapter-owned responses use its HostResponse schema; pre-adapter event-channel failures use the relay-level error above. | -| configured event descriptor (fd3 by convention) | Canonical Trace ABI v6 NDJSON only. | +| configured event descriptor (fd3 by convention) | Canonical Trace ABI v7 NDJSON only. | | fd2 | Diagnostics only; never parse or promote these bytes as events. | Drain fd2 and the event descriptor concurrently. A hard cancellation or @@ -466,7 +466,7 @@ Roughly three tiers of coverage: - **Extract-fallback failure rate is unknown.** When `max_iterations` is exhausted without `answer["ready"]`, one more LLM call tries to synthesize a best-effort answer; a failure there now surfaces as a structured - `extract_error` (result field + the Trace ABI v6 `extract` failure event) instead of + `extract_error` (result field + the Trace ABI v7 `extract` failure event) instead of silently falling back to raw loop output, but there's no data yet on how often that call actually fails or why. No retry has been added — that's a decision for once real failure data exists, not before. diff --git a/pyodide/event_channel_probe.ts b/pyodide/event_channel_probe.ts index 93e6d3a..eb9ab23 100644 --- a/pyodide/event_channel_probe.ts +++ b/pyodide/event_channel_probe.ts @@ -4,7 +4,7 @@ const mode = Deno.args[0]; const channel = eventChannelFromEnvironment(); const fixture = await Deno.readTextFile( new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ), ); diff --git a/pyodide/event_channel_test.ts b/pyodide/event_channel_test.ts index 508689a..b8a7d1c 100644 --- a/pyodide/event_channel_test.ts +++ b/pyodide/event_channel_test.ts @@ -8,7 +8,7 @@ import { import { isRlmEvent } from "../src/droste/substrates/_relay/events.ts"; const TRACE_LIFECYCLE_FIXTURE = new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); diff --git a/pyodide/events_test.ts b/pyodide/events_test.ts index 7728b8c..80867fb 100644 --- a/pyodide/events_test.ts +++ b/pyodide/events_test.ts @@ -48,6 +48,14 @@ const BODIES: Record> = { extract: { phase: "start", iteration: 1 }, result: { result: {} }, replay: { result: {} }, + checkpoint: { + iteration: 1, + checkpoint_seq: 1, + draft: "draft", + draft_chars: 5, + ready: false, + payload: null, + }, usage: { kind: "resolved", root: { @@ -161,7 +169,7 @@ function wire( run_id: "run-1", seq: 1, timestamp: "2026-07-14T00:00:00Z", - version: 6, + version: 7, persistence_class: persistence ?? PERSISTENCE_BY_TYPE[type], depth: 0, ...body, @@ -180,6 +188,52 @@ Deno.test("carries the real payload for a code event (live code streaming)", () ); }); +// `draft_chars` is Python's len() — code points. Comparing it against +// String.length (UTF-16 code units) drops every draft containing a non-BMP +// character, and keeps dropping them, because the draft keeps the character. +// A messaging corpus is full of emoji, so this is the common case, not an edge. +Deno.test("forwards a checkpoint whose draft is not ASCII", () => { + for ( + const draft of [ + "Sarah said 🎉", + "𝐛𝐨𝐥𝐝 math alphanumerics", + "𠮷野家", // astral CJK + "family: 👨‍👩‍👧‍👦", + "flag: 🇯🇵", + ] + ) { + const body = { + iteration: 1, + checkpoint_seq: 1, + draft, + // What droste actually stamps: len(draft) in Python. + draft_chars: [...draft].length, + ready: false, + payload: null, + }; + assert(isRlmEvent(wire("checkpoint", body)), `should forward: ${draft}`); + } +}); + +Deno.test("still rejects a checkpoint whose draft_chars is simply wrong", () => { + const draft = "Sarah said 🎉"; + for (const draft_chars of [[...draft].length + 1, draft.length + 5, 0]) { + assert( + !isRlmEvent( + wire("checkpoint", { + iteration: 1, + checkpoint_seq: 1, + draft, + draft_chars, + ready: false, + payload: null, + }), + ), + `should drop draft_chars=${draft_chars}`, + ); + } +}); + Deno.test("drops non-events: loader chatter, stray prints, empty lines", () => { for ( const noise of [ @@ -329,7 +383,7 @@ Deno.test("successful output beginning ERROR remains an output event", () => { Deno.test("Python and relay accept the same execution golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v6-execution.ndjson", + "../src/droste/testing/fixtures/trace-v7-execution.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); @@ -373,7 +427,7 @@ Deno.test("Python and relay accept the same execution golden NDJSON", async () = Deno.test("Python and relay accept the same lifecycle golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); @@ -551,6 +605,7 @@ Deno.test("vocabulary matches the engine's emitters", () => { [ "budget", "capability", + "checkpoint", "code", "done", "execution_error", diff --git a/src/droste/__init__.py b/src/droste/__init__.py index fe31564..0bf68aa 100644 --- a/src/droste/__init__.py +++ b/src/droste/__init__.py @@ -105,7 +105,13 @@ preflight_rlm, run_rlm, ) -from .loop.step import ReadyMetadataValidator, ReadyMetadataValidatorError +from .loop.step import ( + CheckpointPayloadProvider, + ExtractableWorkProbe, + ExtractContextProvider, + ReadyMetadataValidator, + ReadyMetadataValidatorError, +) from .policy import PolicyHints from .prompts.builder import SystemPromptBuilder from .prompts.pack import ( @@ -163,6 +169,9 @@ "RLMPreflight", "RLMResult", "RLM_PREFLIGHT_SCHEMA_VERSION", + "CheckpointPayloadProvider", + "ExtractableWorkProbe", + "ExtractContextProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", "extract_code_block", diff --git a/src/droste/execution/progress.py b/src/droste/execution/progress.py index e457491..63a571d 100644 --- a/src/droste/execution/progress.py +++ b/src/droste/execution/progress.py @@ -45,6 +45,7 @@ "extract", # discriminated terminal extraction lifecycle facts "result", # canonical unary-equivalent final result (without trajectory) "replay", # configurable replay input/output details + "checkpoint", # {checkpoint_seq, draft, draft_chars, ready, payload} answer state "usage_progress", # transient cumulative usage at a settled model boundary "usage", # durable resolved token/call accounting "budget", # durable configured/consumed budget facts @@ -147,6 +148,29 @@ def extract_event( return value +def checkpoint_event( + iteration: int, + checkpoint_seq: int, + draft: str, + *, + ready: bool, + payload: Any = None, +) -> dict[str, Any]: + """Build one answer-state checkpoint. + + ``payload`` is an opaque host value: the engine carries it across without + ever inspecting it, exactly as the relay carries adapter ``meta``.""" + return { + "type": "checkpoint", + "iteration": iteration, + "checkpoint_seq": checkpoint_seq, + "draft": draft, + "draft_chars": len(draft), + "ready": ready, + "payload": payload, + } + + # --- sinks ------------------------------------------------------------------- diff --git a/src/droste/execution/trace.py b/src/droste/execution/trace.py index e4781c4..9cea952 100644 --- a/src/droste/execution/trace.py +++ b/src/droste/execution/trace.py @@ -17,7 +17,7 @@ from typing import Any, Callable, Mapping from uuid import uuid4 -TRACE_ABI_VERSION = 6 +TRACE_ABI_VERSION = 7 class PersistenceClass(str, Enum): @@ -41,6 +41,7 @@ class PersistenceClass(str, Enum): "repair", "result", "replay", + "checkpoint", } ) TRANSIENT_EVENT_TYPES = frozenset({"startup", "progress", "reasoning_delta", "usage_progress"}) @@ -57,7 +58,7 @@ class PersistenceClass(str, Enum): EventFieldType = type | tuple[type, ...] EventBodySchema = tuple[Mapping[str, EventFieldType], Mapping[str, EventFieldType]] -# One exhaustive v6 table. The first mapping is required fields; the second is +# One exhaustive v7 table. The first mapping is required fields; the second is # optional fields. Nested broker/result values keep their own schema authority. EVENT_BODY_SCHEMAS: Mapping[str, EventBodySchema] = MappingProxyType( { @@ -113,6 +114,19 @@ class PersistenceClass(str, Enum): ), "result": ({"result": Mapping}, {}), "replay": ({"result": Mapping}, {}), + "checkpoint": ( + { + "iteration": int, + "checkpoint_seq": int, + "draft": str, + "draft_chars": int, + "ready": bool, + # Opaque host value. The envelope pins the container only; the + # contents are never inspected and never schema-checked. + "payload": (Mapping, _NONE_TYPE), + }, + {}, + ), "usage_progress": ( { "boundary": str, @@ -184,11 +198,11 @@ def _matches_field_type(value: Any, expected: EventFieldType) -> bool: def validate_event_body(event_type: str, body: Mapping[str, Any]) -> None: - """Validate one body against the exhaustive Trace ABI v6 table.""" + """Validate one body against the exhaustive Trace ABI v7 table.""" try: required, optional = EVENT_BODY_SCHEMAS[event_type] except KeyError as exc: - raise ValueError(f"event type {event_type!r} has no v5 body schema") from exc + raise ValueError(f"event type {event_type!r} has no v7 body schema") from exc missing = required.keys() - body.keys() if missing: raise ValueError(f"event {event_type!r} missing body fields: " + ", ".join(sorted(missing))) @@ -231,9 +245,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if event_type == "subcall": phase = body["phase"] if phase not in {"start", "progress", "completion", "failure"}: - raise ValueError("subcall phase is not recognized by Trace ABI v6") + raise ValueError("subcall phase is not recognized by Trace ABI v7") if body["operation"] not in {"llm_query", "llm_batch", "llm_batch_with_errors"}: - raise ValueError("subcall operation is not recognized by Trace ABI v6") + raise ValueError("subcall operation is not recognized by Trace ABI v7") if not body["call_id"]: raise ValueError("subcall call_id must not be empty") if body["iteration"] < 1: @@ -292,9 +306,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("subcall batch_count must be non-negative") elif event_type == "repair": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("repair phase is not recognized by Trace ABI v6") + raise ValueError("repair phase is not recognized by Trace ABI v7") if body["kind"] not in {"missing_code", "execution_error", "terminal"}: - raise ValueError("repair kind is not recognized by Trace ABI v6") + raise ValueError("repair kind is not recognized by Trace ABI v7") if body["iteration"] < 1: raise ValueError("repair iteration must be positive") error = body.get("error") @@ -310,7 +324,7 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("repair start/completion cannot carry error") elif event_type == "extract": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("extract phase is not recognized by Trace ABI v6") + raise ValueError("extract phase is not recognized by Trace ABI v7") if body["iteration"] < 1: raise ValueError("extract iteration must be positive") extract_error = body.get("extract_error") @@ -324,6 +338,13 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: ) elif extract_error is not None: raise ValueError("extract start/completion cannot carry extract_error") + elif event_type == "checkpoint": + if body["iteration"] < 1: + raise ValueError("checkpoint iteration must be positive") + if body["checkpoint_seq"] < 1: + raise ValueError("checkpoint checkpoint_seq must be positive") + if body["draft_chars"] != len(body["draft"]): + raise ValueError("checkpoint draft_chars must equal the draft length") elif event_type in {"usage", "usage_progress"}: if event_type == "usage_progress" and body["boundary"] not in {"root", "subcall"}: raise ValueError("usage_progress boundary must be 'root' or 'subcall'") @@ -408,7 +429,7 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if not required <= body.keys(): raise ValueError("budget mutation requires action, resource, and amount") if body["action"] not in {"reserve", "commit", "refund", "exhaust"}: - raise ValueError("budget mutation action is not recognized by Trace ABI v6") + raise ValueError("budget mutation action is not recognized by Trace ABI v7") if body["amount"] < 0: raise ValueError("budget mutation amount must be non-negative") else: @@ -465,10 +486,10 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: ScaffoldManifest.from_dict(scaffold_manifest) elif event_type == "policy": if body["outcome"] not in {"passed", "violated", "not_evaluated", "not_enforced"}: - raise ValueError("policy outcome is not recognized by Trace ABI v6") + raise ValueError("policy outcome is not recognized by Trace ABI v7") elif event_type == "done": if body["status"] not in {"success", "error", "cancelled"}: - raise ValueError("done status is not recognized by Trace ABI v6") + raise ValueError("done status is not recognized by Trace ABI v7") if body["iterations"] < 0: raise ValueError("done iterations must be non-negative") stdout_chars = body.get("stdout_chars") @@ -619,7 +640,7 @@ def as_dict(self) -> dict[str, Any]: def parse_event(value: RunEvent | Mapping[str, Any]) -> RunEvent: - """The one strict parser for Trace ABI v6 event values.""" + """The one strict parser for Trace ABI v7 event values.""" if isinstance(value, RunEvent): return value required = { @@ -860,6 +881,7 @@ class TraceRecorder: _events: list[RunEvent] = field(default_factory=list, init=False, repr=False) _terminal_record: RunRecord | None = field(default=None, init=False, repr=False) _started_monotonic: float | None = field(default=None, init=False, repr=False) + _last_checkpoint_seq: int = field(default=0, init=False, repr=False) _lock: Lock = field(default_factory=Lock, init=False, repr=False) def __post_init__(self) -> None: @@ -921,6 +943,14 @@ def _append_locked(self, event: Mapping[str, Any]) -> RunEvent: persistence_class=persistence_class_for(event_type), body=body, ) + if event_type == "checkpoint": + # ``checkpoint_seq`` is the emitter's answer-state ordinal, so the + # recorder — the one authority for a run_id — pins it strictly + # increasing the way it pins ``seq``. + checkpoint_seq = value.body["checkpoint_seq"] + if checkpoint_seq <= self._last_checkpoint_seq: + raise ValueError("checkpoint_seq must strictly increase within a run") + self._last_checkpoint_seq = checkpoint_seq self._events.append(value) return value diff --git a/src/droste/loop/__init__.py b/src/droste/loop/__init__.py index cec1231..6f6c9ed 100644 --- a/src/droste/loop/__init__.py +++ b/src/droste/loop/__init__.py @@ -1,6 +1,12 @@ from .code_extractor import extract_code_block from .rlm import RLMConfig, RLMPreflight, RLMResult, preflight_rlm, run_rlm -from .step import ReadyMetadataValidator, ReadyMetadataValidatorError +from .step import ( + CheckpointPayloadProvider, + ExtractableWorkProbe, + ExtractContextProvider, + ReadyMetadataValidator, + ReadyMetadataValidatorError, +) from .trajectory import IterationRecord __all__ = [ @@ -11,6 +17,9 @@ "RLMResult", "extract_code_block", "IterationRecord", + "CheckpointPayloadProvider", + "ExtractableWorkProbe", + "ExtractContextProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", ] diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index 1732d51..45df043 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -18,6 +18,7 @@ ) from ..execution.progress import ( EventCallback, + checkpoint_event, extract_event, iteration_start_event, llm_response_event, @@ -105,6 +106,10 @@ _EXTRACT_CODE_CHARS = 1000 _EXTRACT_OUTPUT_CHARS = 1500 _EXTRACT_SUMMARY_CHARS = 60000 +# Host observations ride an uncached extract call, so this is deliberately far +# tighter than the trajectory budget above: enough rows to answer from, not the +# whole ledger. +_EXTRACT_HOST_CONTEXT_CHARS = 8000 # Sentinel used in extract summaries for iterations that printed nothing; the # conversational nudge shown to the in-loop model must not read as real output. @@ -139,6 +144,30 @@ def _warn_environment_cleanup(cleanup_error: BaseException) -> None: ) +def _warn_checkpoint(reason: str, error: BaseException) -> None: + """Report a degraded checkpoint. A checkpoint never fails a run.""" + + detail = " ".join(str(error).split())[:1_000] + warnings.warn( + f"RLM checkpoint {reason}: {type(error).__name__}: {detail}", + RuntimeWarning, + stacklevel=3, + ) + + +def _checkpoint_payload(cfg: "RLMConfig") -> Any: + """Resolve the host's opaque checkpoint payload, never inspecting it.""" + + provider = cfg.checkpoint_payload_provider + if provider is None: + return None + try: + return provider() + except Exception as provider_error: + _warn_checkpoint("payload provider failed", provider_error) + return None + + ProgressCallback = Any @@ -543,6 +572,79 @@ def _has_extractable_work(answer: dict[str, Any], has_successful_step: bool) -> return has_successful_step +def _host_extract_context(cfg: "RLMConfig") -> str: + """Host-rendered observations to append to the terminal extract prompt. + + Bounded here rather than trusting the host: this rides on an extract call + that is deliberately uncached (`cache_anchors=None`), so every character is + paid at full prefill, and an unbounded host could spend the recovery call's + whole input budget describing what it found. + + A provider that raises contributes nothing, for the same reason the probe + does: a terminal recovery path must not be able to fail the run it is + recovering. + """ + + provider = cfg.extract_context_provider + if provider is None: + return "" + try: + rendered = provider() + except Exception as exc: # noqa: BLE001 - never let a provider end the run + warnings.warn( + f"RLM extract-context provider failed, continuing without it: {exc}", + RuntimeWarning, + stacklevel=2, + ) + return "" + if not isinstance(rendered, str) or not rendered.strip(): + return "" + return rendered[:_EXTRACT_HOST_CONTEXT_CHARS] + + +def _host_reports_extractable_work(cfg: "RLMConfig") -> bool: + """Whether the host says its own state holds work worth extracting. + + The engine's test above sees only what the engine owns. Generated code that + retrieves real data through a host accessor and then raises before printing + leaves no draft, no successful step, and no stdout — so the engine concludes + there is nothing to extract while the host is holding everything the answer + needed. Asking rather than inferring keeps the engine ignorant of what the + host's data layer is, which is the whole point of the boundary. + + A probe that raises is treated as "no": a terminal recovery path must not be + able to fail the run it is recovering. + """ + + probe = cfg.extractable_work_probe + if probe is None: + return False + try: + return bool(probe()) + except Exception as exc: # noqa: BLE001 - never let a probe end the run + warnings.warn( + f"RLM extractable-work probe failed, treating as no work: {exc}", + RuntimeWarning, + stacklevel=2, + ) + return False + + +def _is_deadline_error(error: RLMError | None) -> bool: + """Whether a root failure is wall-clock deadline exhaustion. + + Deadline exhaustion is a *time* verdict, not a correctness one: the work + already done is as trustworthy as it was a millisecond earlier. So it + routes to the terminal handoff and the extract fallback (like an exhausted + iteration budget) rather than ending the run through ``early_result``, + which would strand that work behind a fatal ``error`` every host discards. + """ + if error is None or error.type != "BudgetExhausted": + return False + details = error.details + return isinstance(details, dict) and details.get("resource") == "wall_ms" + + def _terminal_semantic_budget_error( evidence: _StructuredBatchEvidence | None, context: ExecutionContext, @@ -580,6 +682,7 @@ def _extract_final_answer( cfg: "RLMConfig", context: ExecutionContext, prompt_pack: PromptPack, + host_context: str = "", ) -> tuple[str, RLMError | None]: """One extract pass: when a terminal budget handoff occurs without answer['ready'], make one @@ -593,8 +696,13 @@ def _extract_final_answer( this" apart from "extraction failed, this is raw debug output," instead of both cases looking identical.""" try: + # Host observations sit between the trajectory and the sentinel, and are + # labelled as retrieved data rather than as something the code printed — + # they exist precisely because the code did not print them. + observations = f"\n\nData retrieved during the run:\n{host_context}" if host_context else "" history = ( _trajectory_summary(draft, trajectory) + + observations + f"\n\nUnable sentinel: {prompt_pack.unable_sentinel}" ) slots = PromptSlots( @@ -621,6 +729,11 @@ def _extract_final_answer( model=cfg.root_model or "", context=context, cache_anchors=None, + # This single call is the recovery for a terminal budget handoff, + # including deadline exhaustion. Reserving through the deadline + # would refuse it exactly when it is most needed, so the run would + # fall back to raw scraps. Token budget still binds. + deadline_exempt=True, ) if root_error is not None: return "", root_error @@ -799,6 +912,8 @@ def run_rlm( transcript_window: list[TranscriptWindowEntry] = [] previous_draft_snapshot = str(answer.get("content", "")) code = "" + checkpoint_seq = 0 + checkpoint_draft = "" def step_kwargs() -> dict[str, Any]: return dict( @@ -814,6 +929,32 @@ def step_kwargs() -> dict[str, Any]: ready_metadata_validator=ready_metadata_validator, ) + def emit_checkpoint() -> None: + """Publish answer state so a killed run stays renderable by a host. + + A checkpoint is worth emitting when the draft moved or the host has + something of its own to add; neither the host's provider nor the + emission itself may fail the run.""" + nonlocal checkpoint_seq, checkpoint_draft + draft = str(answer.get("content") or "") + payload = _checkpoint_payload(cfg) + if draft == checkpoint_draft and payload is None: + return + checkpoint_seq += 1 + checkpoint_draft = draft + try: + context.emit_event( + checkpoint_event( + iterations, + checkpoint_seq, + draft, + ready=bool(answer.get("ready")), + payload=payload, + ) + ) + except Exception as checkpoint_error: + _warn_checkpoint("dropped", checkpoint_error) + def early_result(run_error: RLMError | None) -> RLMResult: return finalize( answer_text=_best_answer(answer, last_output, last_response, last_execution_status), @@ -935,6 +1076,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr response, usage, root_error = call_live_root(messages) if root_error is not None: + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = response context.emit_event(llm_response_event(iterations, response)) @@ -962,6 +1107,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr message=root_error.message, ) ) + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = repair_response context.emit_event(llm_response_event(iterations, repair_response)) @@ -1015,6 +1164,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr last_execution_status = outcome.execution_status error = outcome.error answer_metadata = outcome.answer_metadata + emit_checkpoint() if outcome.error is None: has_successful_step = True trajectory.append( @@ -1085,6 +1235,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr ) ) trajectory.append(failed_record) + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = repair_response context.emit_event(llm_response_event(iterations, repair_response)) @@ -1104,6 +1258,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr last_execution_status = outcome.execution_status error = outcome.error answer_metadata = outcome.answer_metadata + emit_checkpoint() if outcome.error is None: code = repaired_code has_successful_step = True @@ -1163,6 +1318,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr terminal_handoff and error is not None and error.type != "IterationLimitExceeded" + and not _is_deadline_error(error) and not str(answer.get("content") or "").strip() ): context.emit_progress( @@ -1265,11 +1421,20 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr was_extracted = False extract_error: RLMError | None = None recovered_error: RLMError | None = None + # Host observations are supplied ONLY when the host is what made this + # extraction possible. When the engine can already see the work — a + # retained draft, or a step that ran to completion — the prompt stays + # byte-identical to what it was before this hook existed, so runs that + # already extract successfully keep their answers, their spend, and + # their benchmark scores. The behavior change is confined to runs that + # produce nothing today. + engine_sees_work = _has_extractable_work(answer, has_successful_step) + host_sees_work = not engine_sees_work and _host_reports_extractable_work(cfg) if ( not answer.get("ready") and terminal_handoff and trajectory - and _has_extractable_work(answer, has_successful_step) + and (engine_sees_work or host_sees_work) ): context.emit_progress("Loop ended unconfirmed: extracting best final answer...") context.emit_event(extract_event(iterations, "start")) @@ -1282,6 +1447,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr cfg, context, resolved_prompt_pack.pack, + host_context=_host_extract_context(cfg) if host_sees_work else "", ) if extracted: context.emit_event(extract_event(iterations, "completion")) diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index 74d3db7..e24d067 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -79,6 +79,28 @@ Sequence[str], ] +# Resolves the opaque value carried by each answer-state checkpoint. The engine +# never inspects what this returns and never schema-checks it: hosts use it to +# ferry their own answer-critical state (whatever that is) alongside the draft. +CheckpointPayloadProvider: TypeAlias = Callable[[], Any] + +# Answers "is there work here worth one terminal extract call?" for state the +# engine cannot see. The engine's own test — a retained draft, or a step that +# executed successfully — misses the case where generated code retrieved real +# data through a host accessor and then raised before printing any of it. The +# engine has no signal there (no draft, no successful step, no stdout), but the +# host does, and stays the only thing that knows what its own data layer did. +ExtractableWorkProbe: TypeAlias = Callable[[], bool] + +# Renders host-held observations for the terminal extract prompt. The engine +# builds that prompt from the draft and per-iteration code/stdout, which is +# everything it can see — but a REPL only surfaces what generated code chose to +# print, so a run that fetched rows and then raised leaves the extract pass +# looking at failing code and nothing else. This lets the host contribute what +# its accessors actually returned. Pre-rendered text: the engine bounds it and +# concatenates it, and never parses or inspects it. +ExtractContextProvider: TypeAlias = Callable[[], str] + class ReadyMetadataValidatorError(RuntimeError): """Trusted host validator failed instead of returning repairable violations.""" @@ -108,6 +130,13 @@ class RLMConfig: on_run_record: RunRecordCallback | None = None rollout: RolloutConfiguration = field(default_factory=RolloutConfiguration) checkpoint_requirements: ScaffoldRequirements | None = None + # Called immediately before each answer-state checkpoint. Returns a + # JSON-serializable value, or None for "nothing to add". A provider that + # raises is reported and the checkpoint carries a null payload: a + # checkpoint can never fail a run. + checkpoint_payload_provider: CheckpointPayloadProvider | None = None + extractable_work_probe: ExtractableWorkProbe | None = None + extract_context_provider: ExtractContextProvider | None = None @dataclass @@ -525,11 +554,19 @@ def call_root( context: ExecutionContext, cache_anchors: tuple[int, ...] | None = (0, -1), transcript_window: tuple[TranscriptWindowEntry, ...] = (), + deadline_exempt: bool = False, ) -> tuple[str, Any, RLMError | None]: """One root-LLM call with token accounting. Returns ``(response, usage, None)`` on success or ``("", None, error)`` — a root failure always ends the run, so the caller finalizes immediately. + + ``deadline_exempt`` reserves without ``through_deadline``, so the call is + checked against tokens/subcalls/depth but not against the shared wall-clock + deadline. It exists for the single terminal extract pass: that pass is the + recovery *for* deadline exhaustion, so gating it on the deadline it is + recovering from would make the fallback unreachable. Token budget still + binds, and the caller must keep such calls to a bounded count. """ outbound_messages, frontier = project_live_transcript(messages, transcript_window) call_id = "root:" + str(uuid4()) @@ -538,7 +575,7 @@ def call_root( tokens=input_estimate + context.budget.root_output_tokens, ) try: - context.ledger.reserve(call_id, request, through_deadline=True) + context.ledger.reserve(call_id, request, through_deadline=not deadline_exempt) except BudgetExhausted as exc: return ( "", diff --git a/src/droste/substrates/_relay/events.ts b/src/droste/substrates/_relay/events.ts index fc0afef..c4375a9 100644 --- a/src/droste/substrates/_relay/events.ts +++ b/src/droste/substrates/_relay/events.ts @@ -20,6 +20,7 @@ export const RLM_EVENT_TYPES = new Set([ "extract", // discriminated extract start/completion/failure "result", // canonical unary-equivalent final result "replay", // configurable replay details + "checkpoint", // {checkpoint_seq, draft, draft_chars, ready, payload} answer state "usage_progress", // cumulative usage at a settled root or subcall boundary "usage", // durable resolved-or-partial provider accounting "budget", // durable budget facts @@ -43,6 +44,7 @@ export const PERSISTENCE_BY_TYPE: Readonly> = { extract: "configurable", result: "configurable", replay: "configurable", + checkpoint: "configurable", usage: "durable", budget: "durable", policy: "durable", @@ -311,6 +313,26 @@ function validBody(type: string, body: Record): boolean { case "result": case "replay": return exactBody(body, ["result"]) && isObject(body.result); + case "checkpoint": + // `payload` is opaque: the container is pinned, the contents never are. + return exactBody(body, [ + "iteration", + "checkpoint_seq", + "draft", + "draft_chars", + "ready", + "payload", + ]) && integerField("iteration") && Number(body.iteration) >= 1 && + integerField("checkpoint_seq") && Number(body.checkpoint_seq) >= 1 && + stringField("draft") && integerField("draft_chars") && + // `draft_chars` is Python's len(): a COUNT OF CODE POINTS. JavaScript's + // String.length counts UTF-16 code units, so any non-BMP character + // (emoji, astral CJK) makes the two disagree and would drop the frame + // here — silently, and for every later checkpoint too, since the draft + // keeps the character. Spreading the string iterates code points. + Number(body.draft_chars) === [...String(body.draft)].length && + typeof body.ready === "boolean" && + (body.payload === null || isObject(body.payload)); case "usage_progress": return validUsageBody(body, true); case "usage": @@ -409,7 +431,7 @@ export function isRlmEvent(line: string): boolean { Number.isInteger(o.seq) && o.seq > 0 && typeof o.timestamp === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(o.timestamp) && - o.version === 6 && + o.version === 7 && o.persistence_class === PERSISTENCE_BY_TYPE[o.type] && Number.isInteger(o.depth) && o.depth >= 0 && (o.depth === 0 diff --git a/src/droste/substrates/_relay/relay.ts b/src/droste/substrates/_relay/relay.ts index b85f7f6..0ed5af3 100644 --- a/src/droste/substrates/_relay/relay.ts +++ b/src/droste/substrates/_relay/relay.ts @@ -455,7 +455,7 @@ function writeRelayEvent(obj: Record): void { depth: 1, seq: ++relaySeq, timestamp: new Date().toISOString(), - version: 6, + version: 7, persistence_class: "transient", }; eventChannel.writeFrame(JSON.stringify(event)); diff --git a/src/droste/testing/__init__.py b/src/droste/testing/__init__.py index 6bb5c0b..e6b468a 100644 --- a/src/droste/testing/__init__.py +++ b/src/droste/testing/__init__.py @@ -16,16 +16,16 @@ from .subcall_client import MockSubcallClient -def trace_v6_lifecycle_ndjson() -> bytes: - """Return the shared Trace ABI v6 lifecycle conformance corpus.""" +def trace_v7_lifecycle_ndjson() -> bytes: + """Return the shared Trace ABI v7 lifecycle conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v6-lifecycle.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v7-lifecycle.ndjson").read_bytes() -def trace_v6_execution_ndjson() -> bytes: - """Return the shared Trace ABI v6 response/code/output/error conformance corpus.""" +def trace_v7_execution_ndjson() -> bytes: + """Return the shared Trace ABI v7 response/code/output/error conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v6-execution.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v7-execution.ndjson").read_bytes() def runner_v10_refusal_ndjson() -> bytes: @@ -50,6 +50,6 @@ def runner_v10_refusal_ndjson() -> bytes: "require_ordered_terminal_events", "require_unknown_completion", "run_while_blocked", - "trace_v6_execution_ndjson", - "trace_v6_lifecycle_ndjson", + "trace_v7_execution_ndjson", + "trace_v7_lifecycle_ndjson", ] diff --git a/src/droste/testing/_trace_fixtures.py b/src/droste/testing/_trace_fixtures.py index c7e444c..409ebc2 100644 --- a/src/droste/testing/_trace_fixtures.py +++ b/src/droste/testing/_trace_fixtures.py @@ -22,7 +22,7 @@ def _ndjson(events: tuple[RunEvent, ...]) -> bytes: ) -def build_trace_v6_execution_ndjson() -> bytes: +def build_trace_v7_execution_ndjson() -> bytes: """Build the deterministic code/output/error projection corpus.""" started_at = datetime(2026, 7, 16, tzinfo=timezone.utc) diff --git a/src/droste/testing/fixtures/trace-v6-execution.ndjson b/src/droste/testing/fixtures/trace-v7-execution.ndjson similarity index 74% rename from src/droste/testing/fixtures/trace-v6-execution.ndjson rename to src/droste/testing/fixtures/trace-v7-execution.ndjson index 2794b1d..6ebe74b 100644 --- a/src/droste/testing/fixtures/trace-v6-execution.ndjson +++ b/src/droste/testing/fixtures/trace-v7-execution.ndjson @@ -1,9 +1,9 @@ -{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} diff --git a/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson b/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson similarity index 85% rename from src/droste/testing/fixtures/trace-v6-lifecycle.ndjson rename to src/droste/testing/fixtures/trace-v7-lifecycle.ndjson index 298ab44..834e9ea 100644 --- a/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson +++ b/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson @@ -1,67 +1,67 @@ -{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":6,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6","scaffold_manifest_version":3} -{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} -{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} -{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} -{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} -{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} -{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} -{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} -{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} -{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} -{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6"},"stdout_chars":13}} -{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6","scaffold_manifest_version":3,"stdout_chars":13} -{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} -{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} -{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7"},"stdout_chars":0}} -{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} -{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} -{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} -{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} -{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a0931838ab34bac30b7f49133a004b0a39c55f190efd91a136b5f084e258d111"},"stdout_chars":4097}} -{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a0931838ab34bac30b7f49133a004b0a39c55f190efd91a136b5f084e258d111","scaffold_manifest_version":3,"stdout_chars":4097} -{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} -{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} -{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7"},"stdout_chars":0}} -{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} -{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} -{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} -{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} -{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:113605ba678f547eab4a9dc81a5e63def14916bb51c50cbfc99ef0397590bf5d"},"stdout_chars":0}} -{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:113605ba678f547eab4a9dc81a5e63def14916bb51c50cbfc99ef0397590bf5d","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":7,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3} +{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} +{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} +{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} +{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} +{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} +{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} +{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} +{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} +{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} +{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9"},"stdout_chars":13}} +{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3,"stdout_chars":13} +{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} +{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} +{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} +{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} +{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} +{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} +{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775"},"stdout_chars":4097}} +{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775","scaffold_manifest_version":3,"stdout_chars":4097} +{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} +{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} +{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} +{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} +{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} +{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} +{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d"},"stdout_chars":0}} +{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d","scaffold_manifest_version":3,"stdout_chars":0} diff --git a/tests/test_answer_checkpoints.py b/tests/test_answer_checkpoints.py new file mode 100644 index 0000000..e1b7e0d --- /dev/null +++ b/tests/test_answer_checkpoints.py @@ -0,0 +1,263 @@ +"""Answer-state checkpoints (Trace ABI v7). + +All answer-critical state used to live only in the loop's memory until the +terminal result. A host that lost the process — a watchdog kill, a crashed +substrate — lost every draft with it. The `checkpoint` event publishes that +state as it moves, so a kill becomes a render rather than a recovery. + +The engine stays agnostic about what a host considers answer-critical: +`payload` is opaque, carried across without inspection or schema checks, and +nothing about a checkpoint may ever fail a run. +""" + +from __future__ import annotations + +import pytest + +from droste import RLMConfig, RunEvent, TraceRetentionPolicy, run_rlm +from droste.execution.progress import EVENT_TYPES, checkpoint_event +from droste.execution.trace import ( + TRACE_ABI_VERSION, + PersistenceClass, + TraceRecorder, + persistence_class_for, + select_retained_events, +) +from droste.protocols.llm_client import TokenUsage +from droste.testing import MockEnvironment, MockLLMClient, MockResponse, MockSubcallClient + + +def _reply(code: str) -> MockResponse: + return MockResponse( + text=f"```python\n{code}\n```", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2, exact=True), + ) + + +def _draft_reply(content: str) -> MockResponse: + return _reply(f"answer['content'] = {content!r}") + + +def _ready_reply(content: str) -> MockResponse: + return _reply(f"answer['content'] = {content!r}\nanswer['ready'] = True") + + +def _run(responses: list[MockResponse], **config_kwargs: object) -> tuple[object, list[dict]]: + events: list[dict] = [] + result = run_rlm( + question="q", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(**config_kwargs), # type: ignore[arg-type] + on_event=events.append, + ) + return result, [event for event in events if event["type"] == "checkpoint"] + + +def _valid_body(**overrides: object) -> dict[str, object]: + body: dict[str, object] = { + "type": "checkpoint", + "iteration": 1, + "checkpoint_seq": 1, + "draft": "partial answer", + "draft_chars": len("partial answer"), + "ready": False, + "payload": None, + } + body.update(overrides) + return body + + +# --- Layer 1: the wire contract ---------------------------------------------- + + +def test_trace_abi_is_version_seven_and_knows_checkpoint() -> None: + assert TRACE_ABI_VERSION == 7 + assert "checkpoint" in EVENT_TYPES + + +def test_checkpoint_is_retention_gated_content_not_a_durable_fact() -> None: + """A checkpoint carries the draft itself, so it must be selectable by the + same retention machinery as every other content-bearing event — never + durable, which would persist message-derived content unconditionally.""" + assert persistence_class_for("checkpoint") is PersistenceClass.CONFIGURABLE + + recorder = TraceRecorder(run_id="retention") + event = recorder.append(_valid_body()) + assert select_retained_events((event,), TraceRetentionPolicy()) == () + retained = select_retained_events( + (event,), TraceRetentionPolicy(retain=frozenset({"checkpoint"})) + ) + assert retained == (event,) + + +def test_checkpoint_envelope_is_strict() -> None: + recorder = TraceRecorder(run_id="strict-checkpoint") + + with pytest.raises(ValueError, match="draft_chars must equal the draft length"): + recorder.append(_valid_body(draft_chars=3)) + with pytest.raises(ValueError, match="checkpoint_seq must be positive"): + recorder.append(_valid_body(checkpoint_seq=0)) + with pytest.raises(ValueError, match="checkpoint iteration must be positive"): + recorder.append(_valid_body(iteration=0)) + with pytest.raises(TypeError, match="ready.*invalid type"): + recorder.append(_valid_body(ready="no")) + with pytest.raises(ValueError, match="missing body fields: draft"): + body = _valid_body() + del body["draft"] + recorder.append(body) + with pytest.raises(ValueError, match="unknown body fields: evidence"): + recorder.append(_valid_body(evidence=[])) + + +def test_checkpoint_seq_strictly_increases_within_a_run() -> None: + recorder = TraceRecorder(run_id="ordered-checkpoints") + recorder.append(_valid_body(checkpoint_seq=1)) + recorder.append(_valid_body(checkpoint_seq=4)) + + with pytest.raises(ValueError, match="strictly increase"): + recorder.append(_valid_body(checkpoint_seq=4)) + with pytest.raises(ValueError, match="strictly increase"): + recorder.append(_valid_body(checkpoint_seq=2)) + + # A different run owns its own ordinals. + other = TraceRecorder(run_id="other-run") + assert other.append(_valid_body(checkpoint_seq=1)).body["checkpoint_seq"] == 1 + + +def test_checkpoint_payload_is_carried_but_never_schema_checked() -> None: + """The envelope pins the container; the contents are the host's business. + Anything narrower would make droste learn what a host puts in there.""" + payload = { + "anything": [{"nested": 1}, None, "text"], + "the_engine": {"does": {"not": {"care": True}}}, + } + recorder = TraceRecorder(run_id="opaque-payload") + event = recorder.append(_valid_body(payload=payload)) + assert event.as_dict()["payload"] == payload + + assert recorder.append(_valid_body(checkpoint_seq=2, payload=None)).body["payload"] is None + with pytest.raises(TypeError, match="payload.*invalid type"): + recorder.append(_valid_body(checkpoint_seq=3, payload=["not", "an", "object"])) + + +def test_checkpoint_event_builder_derives_draft_chars() -> None: + assert checkpoint_event(2, 3, "abcd", ready=True, payload={"k": 1}) == { + "type": "checkpoint", + "iteration": 2, + "checkpoint_seq": 3, + "draft": "abcd", + "draft_chars": 4, + "ready": True, + "payload": {"k": 1}, + } + + +# --- Layer 2: the loop emission point and the payload hook -------------------- + + +def test_checkpoint_follows_each_executed_step_whose_draft_moved() -> None: + result, checkpoints = _run([_draft_reply("first draft"), _ready_reply("final answer")]) + + assert result.ready + assert [event["checkpoint_seq"] for event in checkpoints] == [1, 2] + assert [event["draft"] for event in checkpoints] == ["first draft", "final answer"] + assert [event["draft_chars"] for event in checkpoints] == [11, 12] + assert [event["ready"] for event in checkpoints] == [False, True] + assert [event["iteration"] for event in checkpoints] == [1, 2] + assert all(event["payload"] is None for event in checkpoints) + assert all(event["version"] == 7 for event in checkpoints) + assert all(event["persistence_class"] == "configurable" for event in checkpoints) + + +def test_repaired_code_checkpoints_its_own_draft() -> None: + """The repaired attempt is what actually ran; its draft must be published + too, or a kill right after a repair renders the pre-repair answer.""" + _, checkpoints = _run( + [ + _reply("answer['content'] = 'salvageable'\nraise ValueError('boom')"), + _ready_reply("repaired answer"), + ] + ) + + assert [event["draft"] for event in checkpoints] == ["salvageable", "repaired answer"] + assert [event["checkpoint_seq"] for event in checkpoints] == [1, 2] + assert [event["ready"] for event in checkpoints] == [False, True] + + +def test_unmoved_draft_without_a_payload_emits_nothing() -> None: + _, checkpoints = _run([_reply("print('no draft')"), _ready_reply("done")]) + + assert [event["draft"] for event in checkpoints] == ["done"] + + +def test_payload_alone_is_reason_enough_to_checkpoint() -> None: + """A host whose own state moved needs a checkpoint even when the draft + stood still — the draft is not the only answer-critical state.""" + payloads = iter([{"host": 1}, {"host": 2}]) + _, checkpoints = _run( + [_reply("print('no draft')"), _ready_reply("done")], + checkpoint_payload_provider=lambda: next(payloads), + ) + + assert [event["draft"] for event in checkpoints] == ["", "done"] + assert [event["payload"] for event in checkpoints] == [{"host": 1}, {"host": 2}] + + +def test_payload_provider_value_reaches_the_wire_unexamined() -> None: + payload = {"opaque": [{"to": "droste"}, 7, None], "totals": {"a": 1}} + _, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=lambda: payload, + ) + + assert [event["payload"] for event in checkpoints] == [payload] + + +def test_raising_payload_provider_reports_and_still_checkpoints() -> None: + def provider() -> dict[str, object]: + raise RuntimeError("host ledger exploded") + + with pytest.warns(RuntimeWarning, match="checkpoint payload provider failed"): + result, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=provider, + ) + + assert result.ready + assert result.answer == "answer" + assert [event["payload"] for event in checkpoints] == [None] + assert [event["draft"] for event in checkpoints] == ["answer"] + + +def test_unrepresentable_payload_drops_the_checkpoint_not_the_run() -> None: + """A payload droste cannot put on the wire is still the host's mistake to + fix, never a reason to lose a completed run.""" + with pytest.warns(RuntimeWarning, match="checkpoint dropped"): + result, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=lambda: object(), + ) + + assert result.ready + assert result.answer == "answer" + assert result.error is None + assert checkpoints == [] + + +def test_checkpoints_reach_a_run_record_only_when_retained() -> None: + result, checkpoints = _run([_ready_reply("answer")]) + assert checkpoints + assert all(event.type != "checkpoint" for event in result.run_record.events) + + retained, _ = _run( + [_ready_reply("answer")], + trace_retention=TraceRetentionPolicy( + retain=frozenset({"checkpoint"}), policy_id="checkpoint-retained" + ), + ) + checkpoint_events: list[RunEvent] = [ + event for event in retained.run_record.events if event.type == "checkpoint" + ] + assert [event.body["draft"] for event in checkpoint_events] == ["answer"] diff --git a/tests/test_event_vocabulary.py b/tests/test_event_vocabulary.py index a63531a..f8c3c1e 100644 --- a/tests/test_event_vocabulary.py +++ b/tests/test_event_vocabulary.py @@ -118,6 +118,7 @@ def test_run_rlm_event_stream_through_attached_sink() -> None: "progress", "code", "output", + "checkpoint", "usage", "budget", "policy", diff --git a/tests/test_runner_subcall_reporting.py b/tests/test_runner_subcall_reporting.py index 0a78df5..da35b27 100644 --- a/tests/test_runner_subcall_reporting.py +++ b/tests/test_runner_subcall_reporting.py @@ -253,7 +253,7 @@ def __init__(self, **kwargs: Any) -> None: "seed": 17, } assert manifest["abis"]["runner"] == 10 - assert manifest["abis"]["trace"] == 6 + assert manifest["abis"]["trace"] == 7 assert manifest["engine"]["source_revision"] == "commit-a" assert manifest["id"].startswith("sha256:") assert "trajectory" not in response @@ -295,7 +295,7 @@ def __init__(self, **kwargs: Any) -> None: ] assert len({event["call_id"] for event in live_subcalls}) == 1 assert live_subcalls[1]["checkpoint"] == {"tokens": 5, "subcalls": 1} - assert all(event["iteration"] == 1 and event["version"] == 6 for event in live_subcalls) + assert all(event["iteration"] == 1 and event["version"] == 7 for event in live_subcalls) assert capability["outcome"]["capability_id"]["operation"] == "llm_query" assert "params" not in capability["outcome"] assert "result" not in capability["outcome"] diff --git a/tests/test_subcall_input_capacity.py b/tests/test_subcall_input_capacity.py index fbc3cc9..fafea8c 100644 --- a/tests/test_subcall_input_capacity.py +++ b/tests/test_subcall_input_capacity.py @@ -349,7 +349,7 @@ def test_runner_preflight_records_declared_capacity() -> None: } -def test_runner_v5_is_refused_before_trace_v6_can_be_ignored() -> None: +def test_runner_v5_is_refused_before_trace_v7_can_be_ignored() -> None: response = run_worker( { "protocol_version": 5, diff --git a/tests/test_trace_abi.py b/tests/test_trace_abi.py index ec84f56..4e85920 100644 --- a/tests/test_trace_abi.py +++ b/tests/test_trace_abi.py @@ -33,10 +33,10 @@ MockResponse, MockSubcallClient, runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) -from droste.testing._trace_fixtures import build_trace_v6_execution_ndjson +from droste.testing._trace_fixtures import build_trace_v7_execution_ndjson from droste_runner.run import run as run_worker @@ -233,7 +233,7 @@ def test_usage_progress_emits_cumulative_role_boundaries_without_estimating() -> assert progress[1]["root"]["total_tokens"] == 10 assert progress[1]["subcall"]["total_tokens"] == 7 assert progress[1]["persistence_class"] == "transient" - assert progress[1]["version"] == 6 + assert progress[1]["version"] == 7 def test_concurrent_subcall_usage_progress_is_serialized_and_monotonic() -> None: @@ -582,7 +582,7 @@ def test_run_record_allows_repeated_durable_budget_mutations() -> None: ] -def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: +def test_parser_requires_v7_envelope_and_rejects_false_classification() -> None: with pytest.raises(ValueError, match="missing envelope fields"): parse_event({"type": "code", "iteration": 1, "code": "print(1)"}) @@ -592,7 +592,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "configurable", "depth": 0, "iteration": 1, @@ -607,7 +607,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 2, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "transient", "depth": 0, "engine_version": "0.10.6", @@ -624,7 +624,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "durable", "depth": 0, "code": "print(1)", @@ -638,7 +638,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T01:00:00+01:00", - "version": 6, + "version": 7, "persistence_class": "transient", "depth": 0, "status": "working", @@ -673,7 +673,7 @@ def test_event_bodies_reject_missing_unknown_and_wrong_primitive_fields() -> Non recorder.finish(terminal) -def test_trace_v6_budget_snapshot_requires_max_iterations() -> None: +def test_trace_v7_budget_snapshot_requires_max_iterations() -> None: recorder = TraceRecorder(run_id="strict-budget") budget = json.loads(json.dumps(_terminal()["budget"])) del budget["configured"]["max_iterations"] @@ -844,11 +844,11 @@ def test_trace_abi_v3_is_rejected_instead_of_dropping_cache_usage() -> None: ) -def _trace_v6_golden_runs() -> dict[str, list[RunEvent]]: +def _trace_v7_golden_runs() -> dict[str, list[RunEvent]]: runs: dict[str, list[RunEvent]] = {} previous_run_id: str | None = None closed: set[str] = set() - for line in trace_v6_lifecycle_ndjson().decode("utf-8").splitlines(): + for line in trace_v7_lifecycle_ndjson().decode("utf-8").splitlines(): event = parse_event(json.loads(line)) if event.run_id != previous_run_id: if event.run_id in closed: @@ -860,9 +860,9 @@ def _trace_v6_golden_runs() -> dict[str, list[RunEvent]]: return runs -def test_trace_v6_execution_fixture_is_canonical_and_exact() -> None: - fixture = trace_v6_execution_ndjson() - assert fixture == build_trace_v6_execution_ndjson() +def test_trace_v7_execution_fixture_is_canonical_and_exact() -> None: + fixture = trace_v7_execution_ndjson() + assert fixture == build_trace_v7_execution_ndjson() events = [parse_event(json.loads(line)) for line in fixture.decode("utf-8").splitlines()] root = events[:6] @@ -894,8 +894,8 @@ def _error_type(value: object) -> object: return value.get("type") if isinstance(value, Mapping) else None -def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: - runs = _trace_v6_golden_runs() +def test_trace_v7_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: + runs = _trace_v7_golden_runs() assert set(runs) == { "golden-success", "golden-recovered", @@ -928,7 +928,7 @@ def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> No "prompt_pack": 1, "provider": 4, "runner": 10, - "trace": 6, + "trace": 7, } assert dict(scaffold_manifest.body["budget"]) == dict(done["budget"]["configured"]) assert ( @@ -991,8 +991,8 @@ def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> No assert terminal["usage"]["subcall"]["cache_creation_tokens"] == 1 -def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: - events = [event for run in _trace_v6_golden_runs().values() for event in run] +def test_trace_v7_golden_corpus_covers_each_discriminated_lifecycle() -> None: + events = [event for run in _trace_v7_golden_runs().values() for event in run] subcalls = [event for event in events if event.type == "subcall"] repairs = [event for event in events if event.type == "repair"] extracts = [event for event in events if event.type == "extract"] @@ -1019,7 +1019,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert {event.body["boundary"] for event in usage_progress} == {"root", "subcall"} assert all(event.persistence_class is PersistenceClass.TRANSIENT for event in usage_progress) - output_limit = _trace_v6_golden_runs()["golden-output-limit"] + output_limit = _trace_v7_golden_runs()["golden-output-limit"] assert not any(event.type == "output" for event in output_limit) assert any( event.type == "execution_error" and event.body["error_type"] == "SandboxError" @@ -1043,7 +1043,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: event.body["message"] for event in output_limit if event.type == "execution_error" ) assert f"exceeded {output_manifest['sandbox']['output_chars']} characters" in output_error - cancelled = _trace_v6_golden_runs()["golden-cancelled"] + cancelled = _trace_v7_golden_runs()["golden-cancelled"] assert cancelled[-1].body["status"] == "error" assert any( event.type == "execution_error" and event.body["error_type"] == "CapabilityCallError" @@ -1057,7 +1057,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert cancelled[-1].body["usage"]["root"]["complete"] is False assert cancelled[-1].body["usage"]["subcall"]["complete"] is False - extract_failed = _trace_v6_golden_runs()["golden-extract-failed"] + extract_failed = _trace_v7_golden_runs()["golden-extract-failed"] result = next(event.body["result"] for event in extract_failed if event.type == "result") assert result["answer"] == f"Error: {result['error']['message']}" assert result["error"]["details"]["withheld_content"] == "retained evidence" diff --git a/tests/test_wall_clock_fallback.py b/tests/test_wall_clock_fallback.py new file mode 100644 index 0000000..bfa7c47 --- /dev/null +++ b/tests/test_wall_clock_fallback.py @@ -0,0 +1,346 @@ +"""Wall-clock deadline exhaustion routes to the terminal extract fallback. + +A run that spent `budget.wall_ms` used to end through `early_result`, which +hands the host a fatal `error` alongside whatever partial answer existed. +Hosts treat `result.error` as fatal and discard the run — so every iteration +of real work was thrown away over a *time* verdict, which says nothing about +whether that work was any good. Deadline exhaustion now takes the same +terminal handoff as an exhausted iteration budget and comes back as a +best-effort answer. +""" + +import time + +import pytest + +from droste import Budget, RLMConfig, run_rlm +from droste.exceptions import RLMError +from droste.execution import create_execution_context +from droste.loop.rlm import ( + _EXTRACT_HOST_CONTEXT_CHARS, + _extract_final_answer, + _is_deadline_error, +) +from droste.loop.trajectory import EXECUTION_STATUS_SUCCESS, IterationRecord +from droste.prompts import load_builtin_prompt_catalog, resolve_prompt_pack +from droste.protocols.llm_client import TokenUsage +from droste.testing import MockEnvironment, MockLLMClient, MockResponse, MockSubcallClient + +_TRAJECTORY = [ + IterationRecord( + iteration=1, + llm_input=[{"role": "user", "content": "test"}], + llm_output="```python\nprint('useful evidence')\n```", + code_executed="print('useful evidence')", + execution_result="useful evidence", + tokens_used=2, + execution_status=EXECUTION_STATUS_SUCCESS, + ) +] + + +def _usage() -> TokenUsage: + return TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2, exact=True) + + +def _deadline_error(resource: str = "wall_ms") -> RLMError: + return RLMError( + type="BudgetExhausted", + message="budget exhausted", + details={"resource": resource, "requested": 1, "remaining": 0}, + ) + + +def _pack(): + return resolve_prompt_pack( + model="", profile="full", engine_catalog=load_builtin_prompt_catalog() + ).pack + + +def test_is_deadline_error_discriminates_resource(): + """Only wall-clock exhaustion reroutes. Token exhaustion must stay fatal: + there is no budget left to pay for the extract call it would trigger.""" + assert _is_deadline_error(_deadline_error()) is True + assert _is_deadline_error(_deadline_error("tokens")) is False + assert _is_deadline_error(_deadline_error("subcalls")) is False + assert _is_deadline_error(RLMError(type="PolicyError", message="x")) is False + assert _is_deadline_error(None) is False + # Details absent entirely must neither crash nor match. + assert _is_deadline_error(RLMError(type="BudgetExhausted", message="x")) is False + + +def test_extract_call_survives_the_deadline_it_is_recovering_from(): + """The crux. `call_root` reserves with through_deadline=True, which refuses + once remaining.wall_ms hits 0 — so without the exemption the extract pass + is unreachable in exactly the case it exists to serve.""" + context = create_execution_context(budget=Budget(wall_ms=1)) + time.sleep(0.01) # guarantee the shared run deadline is spent + assert context.ledger.snapshot().remaining.wall_ms == 0 + + text, error = _extract_final_answer( + "test", + "partial draft", + _TRAJECTORY, + MockLLMClient(responses=[MockResponse(text="synthesized answer", usage=_usage())]), + RLMConfig(), + context, + _pack(), + ) + + assert error is None + assert text == "synthesized answer" + + +def test_deadline_exhaustion_yields_extracted_answer_not_fatal_error(): + """End to end: iteration 1 does real work and burns the deadline, so + iteration 2's root call trips wall_ms. The run must return a best-effort + answer with the deadline preserved as diagnostic provenance.""" + responses = [ + MockResponse( + text=( + "```python\n" + "import time\n" + "answer['content'] = 'partial finding'\n" + "time.sleep(0.05)\n" + "```" + ), + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis of the partial finding.", usage=_usage()), + ] + + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(wall_ms=20)), + ) + + assert result.error is None, "a time verdict must not surface as a fatal run error" + assert result.extracted is True + assert result.answer == "Best-effort synthesis of the partial finding." + assert result.recovered_error is not None + assert result.recovered_error.type == "BudgetExhausted" + assert result.recovered_error.details["resource"] == "wall_ms" + + +def test_token_exhaustion_still_ends_the_run_fatally(): + """Guard the discrimination above at the run level: a token-exhausted run + must NOT be rerouted into an extract pass it cannot afford.""" + tiny = Budget(tokens=0x40, root_output_tokens=0x40, subcall_output_tokens=0x40) + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient( + responses=[MockResponse(text="```python\npass\n```", usage=_usage())] + ), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=tiny), + ) + + assert result.error is not None + assert result.error.type == "BudgetExhausted" + assert result.error.details["resource"] == "tokens" + assert result.extracted is False + # Fatal, so nothing is "recovered" — the distinction Cozy's chip relies on. + assert result.recovered_error is None + + +def test_recovered_budget_exhaustion_is_always_the_wall_clock(): + """Pins a cross-repo invariant. Cozy renders a recovered `BudgetExhausted` + as "ran out of time", which is only honest while the wall clock is the sole + budget resource that can reach a recoverable terminal handoff: token + exhaustion stays fatal (above) and semantic budgets surface as PolicyError. + + If a future change lets another resource recover here, this fails — rather + than Cozy silently telling users the wrong thing about why their answer is + incomplete.""" + responses = [ + MockResponse( + text=( + "```python\n" + "import time\n" + "answer['content'] = 'partial finding'\n" + "time.sleep(0.05)\n" + "```" + ), + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis.", usage=_usage()), + ] + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(wall_ms=20)), + ) + + assert result.recovered_error is not None + assert result.recovered_error.type == "BudgetExhausted" + assert result.recovered_error.details["resource"] == "wall_ms" + + +# --- The host knows about work the engine cannot see --------------------- +# +# Generated code that retrieves real data through a host accessor and then +# raises before printing leaves the engine with nothing to test: no draft, no +# successful step, no stdout. It concluded there was nothing to extract while +# the host was holding everything the answer needed. Observed on a real run +# that retrieved 100 message GUIDs and then returned a bare budget error. + + +def _raising_code() -> str: + return '```python\nrows = query("SELECT 1")\nraise ValueError("boom")\n```' + + +def _run_with_probe(probe): + # Uniform responses: the loop's exact consumption (initial attempt, missing + # code repair, execution repair) is not what is under test here, and pinning + # a response count would make this fail for reasons unrelated to the probe. + responses = [MockResponse(text=_raising_code(), usage=_usage()) for _ in range(12)] + return run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(max_iterations=2), extractable_work_probe=probe), + ) + + +def test_engine_alone_finds_nothing_extractable_in_failed_steps() -> None: + """Baseline: the engine's own test is unchanged, so a trajectory of pure + errors with no draft and no stdout still yields no extraction.""" + result = _run_with_probe(None) + + assert result.extracted is False + assert result.error is not None + + +def test_host_probe_unlocks_extraction_the_engine_would_have_skipped() -> None: + """The host says its data layer retrieved something, so the one terminal + extract call runs and the run returns an answer instead of a bare error. + + This is the case seen on a real 314k-message corpus: generated code called + an accessor, recorded 100 message GUIDs, then raised before printing any of + them — leaving the engine no draft, no successful step, and no stdout.""" + result = _run_with_probe(lambda: True) + + assert result.extracted is True + assert result.error is None + + +def test_a_probe_saying_no_changes_nothing() -> None: + result = _run_with_probe(lambda: False) + + assert result.extracted is False + + +def test_a_raising_probe_cannot_fail_the_run_it_is_recovering() -> None: + def explode() -> bool: + raise RuntimeError("host ledger unavailable") + + with pytest.warns(RuntimeWarning, match="extractable-work probe failed"): + result = _run_with_probe(explode) + + # Treated as "no work" rather than propagating: a terminal recovery path + # must never be able to end the run it exists to rescue. + assert result.extracted is False + + +# --- What the extract pass is allowed to see ----------------------------- +# +# Unlocking the extract call was necessary but not sufficient: the prompt is +# built from draft + code + stdout, so a run whose data exists only in the host +# ledger handed the extract pass failing code and nothing else, and it returned +# InsufficientEvidence. The host can now contribute what its accessors actually +# returned. + + +def _captured_extract_prompt(**config_kwargs): + """Run to a terminal handoff and return the extract call's user prompt.""" + seen: list[str] = [] + + class CapturingClient(MockLLMClient): + def responses_create(self, messages, *args, **kwargs): # type: ignore[override] + seen.append(str(messages[-1].get("content", ""))) + return super().responses_create(messages, *args, **kwargs) + + responses = [MockResponse(text=_raising_code(), usage=_usage()) for _ in range(12)] + run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=CapturingClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(max_iterations=2), **config_kwargs), + ) + return seen[-1] if seen else "" + + +def test_host_observations_reach_the_extract_prompt() -> None: + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=lambda: ( + "[evidence-1] query(...) -> 84 rows\nSarah | dinner friday" + ), + ) + + assert "Data retrieved during the run" in prompt + assert "Sarah | dinner friday" in prompt + + +def test_observations_are_bounded_so_recovery_cannot_eat_its_own_budget() -> None: + """The extract call is deliberately uncached, so every character is paid at + full prefill and an unbounded host could spend the whole recovery call.""" + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=lambda: "x" * 50_000, + ) + + assert prompt.count("x") == _EXTRACT_HOST_CONTEXT_CHARS + + +def test_a_raising_context_provider_still_lets_extraction_run() -> None: + def explode() -> str: + raise RuntimeError("ledger unavailable") + + with pytest.warns(RuntimeWarning, match="extract-context provider failed"): + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=explode, + ) + + # Degraded, not fatal: the extract call still happened, just without them. + assert prompt + assert "Data retrieved during the run" not in prompt + + +def test_a_run_the_engine_could_already_extract_is_left_untouched() -> None: + """The narrow gate. A run whose draft or successful step the engine can see + gets the prompt it got before this hook existed — same answer, same spend, + same benchmark score. Only runs that produce nothing today change.""" + provider_calls: list[int] = [] + + responses = [ + MockResponse( + text="```python\nanswer['content'] = 'partial finding'\n```", + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis.", usage=_usage()), + ] + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig( + budget=Budget(max_iterations=1), + extractable_work_probe=lambda: True, + extract_context_provider=lambda: provider_calls.append(1) or "should not appear", + ), + ) + + assert result.extracted is True + # The engine saw the draft, so the host was never consulted at all. + assert provider_calls == []