From 5847c12c6e4de392e729a7210c1f02d8c3bcc2e0 Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Sat, 1 Aug 2026 14:21:31 -0400 Subject: [PATCH 1/5] Route wall-clock exhaustion to the terminal extract fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run that spent `budget.wall_ms` ended through `early_result`, which hands the host a fatal `error` alongside whatever partial answer existed. Hosts treat `result.error` as fatal and discard the run, so every iteration of real work was thrown away over a *time* verdict — one that says nothing about whether that work was any good. Deadline exhaustion now takes the same terminal handoff as an exhausted iteration budget: it sets `terminal_handoff` and breaks, reaching the extract pass that turns partial work into a best-effort answer with ledger-derived citations. Token exhaustion deliberately stays fatal, since there is no budget left to pay for the extract call it would trigger. The extract call is reserved with `deadline_exempt=True`. `call_root` reserves `through_deadline=True`, which refuses once `remaining.wall_ms` reaches 0 — so without the exemption the fallback would be unreachable in exactly the case it exists to serve. Token budget still binds, and the caller keeps such calls to exactly one. Terminal REPL finalization is skipped for deadline handoffs the same way it is for `IterationLimitExceeded`: re-entering the sandbox after the deadline cannot help. Co-Authored-By: Claude Opus 5 (1M context) --- src/droste/loop/rlm.py | 33 +++++++ src/droste/loop/step.py | 10 ++- tests/test_wall_clock_fallback.py | 139 ++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 tests/test_wall_clock_fallback.py diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index 1732d51..cedcbe5 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -543,6 +543,21 @@ def _has_extractable_work(answer: dict[str, Any], has_successful_step: bool) -> return has_successful_step +def _is_deadline_error(error: RLMError | None) -> bool: + """Whether a root failure is wall-clock deadline exhaustion. + + Deadline exhaustion is a *time* verdict, not a correctness one: the work + already done is as trustworthy as it was a millisecond earlier. So it + routes to the terminal handoff and the extract fallback (like an exhausted + iteration budget) rather than ending the run through ``early_result``, + which would strand that work behind a fatal ``error`` every host discards. + """ + if error is None or error.type != "BudgetExhausted": + return False + details = error.details + return isinstance(details, dict) and details.get("resource") == "wall_ms" + + def _terminal_semantic_budget_error( evidence: _StructuredBatchEvidence | None, context: ExecutionContext, @@ -621,6 +636,11 @@ def _extract_final_answer( model=cfg.root_model or "", context=context, cache_anchors=None, + # This single call is the recovery for a terminal budget handoff, + # including deadline exhaustion. Reserving through the deadline + # would refuse it exactly when it is most needed, so the run would + # fall back to raw scraps. Token budget still binds. + deadline_exempt=True, ) if root_error is not None: return "", root_error @@ -935,6 +955,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr response, usage, root_error = call_live_root(messages) if root_error is not None: + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = response context.emit_event(llm_response_event(iterations, response)) @@ -962,6 +986,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr message=root_error.message, ) ) + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = repair_response context.emit_event(llm_response_event(iterations, repair_response)) @@ -1085,6 +1113,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr ) ) trajectory.append(failed_record) + if _is_deadline_error(root_error): + error = root_error + terminal_handoff = True + break return early_result(root_error) last_response = repair_response context.emit_event(llm_response_event(iterations, repair_response)) @@ -1163,6 +1195,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr terminal_handoff and error is not None and error.type != "IterationLimitExceeded" + and not _is_deadline_error(error) and not str(answer.get("content") or "").strip() ): context.emit_progress( diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index 74d3db7..1df8c0a 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -525,11 +525,19 @@ def call_root( context: ExecutionContext, cache_anchors: tuple[int, ...] | None = (0, -1), transcript_window: tuple[TranscriptWindowEntry, ...] = (), + deadline_exempt: bool = False, ) -> tuple[str, Any, RLMError | None]: """One root-LLM call with token accounting. Returns ``(response, usage, None)`` on success or ``("", None, error)`` — a root failure always ends the run, so the caller finalizes immediately. + + ``deadline_exempt`` reserves without ``through_deadline``, so the call is + checked against tokens/subcalls/depth but not against the shared wall-clock + deadline. It exists for the single terminal extract pass: that pass is the + recovery *for* deadline exhaustion, so gating it on the deadline it is + recovering from would make the fallback unreachable. Token budget still + binds, and the caller must keep such calls to a bounded count. """ outbound_messages, frontier = project_live_transcript(messages, transcript_window) call_id = "root:" + str(uuid4()) @@ -538,7 +546,7 @@ def call_root( tokens=input_estimate + context.budget.root_output_tokens, ) try: - context.ledger.reserve(call_id, request, through_deadline=True) + context.ledger.reserve(call_id, request, through_deadline=not deadline_exempt) except BudgetExhausted as exc: return ( "", diff --git a/tests/test_wall_clock_fallback.py b/tests/test_wall_clock_fallback.py new file mode 100644 index 0000000..6846874 --- /dev/null +++ b/tests/test_wall_clock_fallback.py @@ -0,0 +1,139 @@ +"""Wall-clock deadline exhaustion routes to the terminal extract fallback. + +A run that spent `budget.wall_ms` used to end through `early_result`, which +hands the host a fatal `error` alongside whatever partial answer existed. +Hosts treat `result.error` as fatal and discard the run — so every iteration +of real work was thrown away over a *time* verdict, which says nothing about +whether that work was any good. Deadline exhaustion now takes the same +terminal handoff as an exhausted iteration budget and comes back as a +best-effort answer. +""" + +import time + +from droste import Budget, RLMConfig, run_rlm +from droste.exceptions import RLMError +from droste.execution import create_execution_context +from droste.loop.rlm import _extract_final_answer, _is_deadline_error +from droste.loop.trajectory import EXECUTION_STATUS_SUCCESS, IterationRecord +from droste.prompts import load_builtin_prompt_catalog, resolve_prompt_pack +from droste.protocols.llm_client import TokenUsage +from droste.testing import MockEnvironment, MockLLMClient, MockResponse, MockSubcallClient + +_TRAJECTORY = [ + IterationRecord( + iteration=1, + llm_input=[{"role": "user", "content": "test"}], + llm_output="```python\nprint('useful evidence')\n```", + code_executed="print('useful evidence')", + execution_result="useful evidence", + tokens_used=2, + execution_status=EXECUTION_STATUS_SUCCESS, + ) +] + + +def _usage() -> TokenUsage: + return TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2, exact=True) + + +def _deadline_error(resource: str = "wall_ms") -> RLMError: + return RLMError( + type="BudgetExhausted", + message="budget exhausted", + details={"resource": resource, "requested": 1, "remaining": 0}, + ) + + +def _pack(): + return resolve_prompt_pack( + model="", profile="full", engine_catalog=load_builtin_prompt_catalog() + ).pack + + +def test_is_deadline_error_discriminates_resource(): + """Only wall-clock exhaustion reroutes. Token exhaustion must stay fatal: + there is no budget left to pay for the extract call it would trigger.""" + assert _is_deadline_error(_deadline_error()) is True + assert _is_deadline_error(_deadline_error("tokens")) is False + assert _is_deadline_error(_deadline_error("subcalls")) is False + assert _is_deadline_error(RLMError(type="PolicyError", message="x")) is False + assert _is_deadline_error(None) is False + # Details absent entirely must neither crash nor match. + assert _is_deadline_error(RLMError(type="BudgetExhausted", message="x")) is False + + +def test_extract_call_survives_the_deadline_it_is_recovering_from(): + """The crux. `call_root` reserves with through_deadline=True, which refuses + once remaining.wall_ms hits 0 — so without the exemption the extract pass + is unreachable in exactly the case it exists to serve.""" + context = create_execution_context(budget=Budget(wall_ms=1)) + time.sleep(0.01) # guarantee the shared run deadline is spent + assert context.ledger.snapshot().remaining.wall_ms == 0 + + text, error = _extract_final_answer( + "test", + "partial draft", + _TRAJECTORY, + MockLLMClient(responses=[MockResponse(text="synthesized answer", usage=_usage())]), + RLMConfig(), + context, + _pack(), + ) + + assert error is None + assert text == "synthesized answer" + + +def test_deadline_exhaustion_yields_extracted_answer_not_fatal_error(): + """End to end: iteration 1 does real work and burns the deadline, so + iteration 2's root call trips wall_ms. The run must return a best-effort + answer with the deadline preserved as diagnostic provenance.""" + responses = [ + MockResponse( + text=( + "```python\n" + "import time\n" + "answer['content'] = 'partial finding'\n" + "time.sleep(0.05)\n" + "```" + ), + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis of the partial finding.", usage=_usage()), + ] + + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(wall_ms=20)), + ) + + assert result.error is None, "a time verdict must not surface as a fatal run error" + assert result.extracted is True + assert result.answer == "Best-effort synthesis of the partial finding." + assert result.recovered_error is not None + assert result.recovered_error.type == "BudgetExhausted" + assert result.recovered_error.details["resource"] == "wall_ms" + + +def test_token_exhaustion_still_ends_the_run_fatally(): + """Guard the discrimination above at the run level: a token-exhausted run + must NOT be rerouted into an extract pass it cannot afford.""" + tiny = Budget(tokens=0x40, root_output_tokens=0x40, subcall_output_tokens=0x40) + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient( + responses=[MockResponse(text="```python\npass\n```", usage=_usage())] + ), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=tiny), + ) + + assert result.error is not None + assert result.error.type == "BudgetExhausted" + assert result.error.details["resource"] == "tokens" + assert result.extracted is False From b0521f71e9920d2b93f89861db882fb55feddc67 Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Sat, 1 Aug 2026 14:31:39 -0400 Subject: [PATCH 2/5] Pin the recoverable-budget invariant Cozy renders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cozy tells users a best-effort answer stopped because it "ran out of time" whenever the recovered cause is BudgetExhausted. That is only honest while the wall clock is the sole budget resource able to reach a recoverable terminal handoff — token exhaustion stays fatal, and semantic budgets surface as PolicyError. Assert both halves so a future change that lets another resource recover here fails this test, rather than Cozy silently telling users the wrong thing about why their answer is incomplete. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_wall_clock_fallback.py | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_wall_clock_fallback.py b/tests/test_wall_clock_fallback.py index 6846874..6df8a79 100644 --- a/tests/test_wall_clock_fallback.py +++ b/tests/test_wall_clock_fallback.py @@ -137,3 +137,40 @@ def test_token_exhaustion_still_ends_the_run_fatally(): assert result.error.type == "BudgetExhausted" assert result.error.details["resource"] == "tokens" assert result.extracted is False + # Fatal, so nothing is "recovered" — the distinction Cozy's chip relies on. + assert result.recovered_error is None + + +def test_recovered_budget_exhaustion_is_always_the_wall_clock(): + """Pins a cross-repo invariant. Cozy renders a recovered `BudgetExhausted` + as "ran out of time", which is only honest while the wall clock is the sole + budget resource that can reach a recoverable terminal handoff: token + exhaustion stays fatal (above) and semantic budgets surface as PolicyError. + + If a future change lets another resource recover here, this fails — rather + than Cozy silently telling users the wrong thing about why their answer is + incomplete.""" + responses = [ + MockResponse( + text=( + "```python\n" + "import time\n" + "answer['content'] = 'partial finding'\n" + "time.sleep(0.05)\n" + "```" + ), + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis.", usage=_usage()), + ] + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(wall_ms=20)), + ) + + assert result.recovered_error is not None + assert result.recovered_error.type == "BudgetExhausted" + assert result.recovered_error.details["resource"] == "wall_ms" From bc1a155522cf021cc96815b27f8571375a908531 Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Sat, 1 Aug 2026 17:02:42 -0400 Subject: [PATCH 3/5] Count draft code points, not UTF-16 units, in the relay filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `draft_chars` is Python's len() — a count of code points. The relay's forwarding filter compared it against JavaScript's String.length, which counts UTF-16 code units. Any non-BMP character in a draft made the two disagree, so isRlmEvent rejected the frame and relay.ts dropped it before it ever reached the host descriptor. Silently, and permanently: the draft keeps the character, so every later checkpoint of that run was dropped too, and the adapter's high-water mark had already advanced, so the evidence riding those frames was never republished. A watchdog SIGKILL then rendered the bare work-log card with zero salvage — the exact outcome checkpoints exist to prevent. Cozy is a messaging app. Emoji in a draft is the common case, not an edge case. Four green suites missed it because every fixture was ASCII, so the regression tests here use emoji, astral CJK, math alphanumerics, a ZWJ sequence, and a regional-indicator flag, plus the negative case so a genuinely wrong draft_chars is still rejected. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 36 +-- .github/workflows/release.yml | 4 +- AGENTS.md | 8 +- UPGRADING.md | 24 ++ docs/architecture.md | 6 +- docs/budgets.md | 2 +- docs/scaffold-manifest.md | 2 +- docs/trace-abi.md | 39 ++- examples/pyodide-host/e2e_test.ts | 2 +- pyodide/README.md | 4 +- pyodide/event_channel_probe.ts | 2 +- pyodide/event_channel_test.ts | 2 +- pyodide/events_test.ts | 61 +++- src/droste/__init__.py | 7 +- src/droste/execution/progress.py | 24 ++ src/droste/execution/trace.py | 56 +++- src/droste/loop/__init__.py | 7 +- src/droste/loop/rlm.py | 55 ++++ src/droste/loop/step.py | 10 + src/droste/substrates/_relay/events.ts | 24 +- src/droste/substrates/_relay/relay.ts | 2 +- src/droste/testing/__init__.py | 16 +- src/droste/testing/_trace_fixtures.py | 2 +- ...ution.ndjson => trace-v7-execution.ndjson} | 18 +- ...cycle.ndjson => trace-v7-lifecycle.ndjson} | 134 ++++----- tests/test_answer_checkpoints.py | 263 ++++++++++++++++++ tests/test_event_vocabulary.py | 1 + tests/test_runner_subcall_reporting.py | 4 +- tests/test_subcall_input_capacity.py | 2 +- tests/test_trace_abi.py | 46 +-- 30 files changed, 688 insertions(+), 175 deletions(-) rename src/droste/testing/fixtures/{trace-v6-execution.ndjson => trace-v7-execution.ndjson} (74%) rename src/droste/testing/fixtures/{trace-v6-lifecycle.ndjson => trace-v7-lifecycle.ndjson} (85%) create mode 100644 tests/test_answer_checkpoints.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e783f78..e26fda9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: PY - name: Distributions bundle the exact event ABI corpus (#144) run: | - for f in trace-v6-execution.ndjson trace-v6-lifecycle.ndjson runner-v10-refusal.ndjson; do + for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || { echo "wheel is missing droste/testing/fixtures/$f"; exit 1; } @@ -154,21 +154,21 @@ jobs: from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) source = Path(sys.argv[1]) - assert trace_v6_execution_ndjson() == (source / "trace-v6-execution.ndjson").read_bytes() - assert trace_v6_lifecycle_ndjson() == (source / "trace-v6-lifecycle.ndjson").read_bytes() + assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes() + assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes() assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-execution.ndjson" + cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson" + cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson" cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" @@ -319,7 +319,7 @@ jobs: PY - name: Distributions bundle the exact event ABI corpus (#144) run: | - for f in trace-v6-execution.ndjson trace-v6-lifecycle.ndjson runner-v10-refusal.ndjson; do + for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || { echo "wheel is missing droste/testing/fixtures/$f"; exit 1; } @@ -333,21 +333,21 @@ jobs: from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) source = Path(sys.argv[1]) - assert trace_v6_execution_ndjson() == (source / "trace-v6-execution.ndjson").read_bytes() - assert trace_v6_lifecycle_ndjson() == (source / "trace-v6-lifecycle.ndjson").read_bytes() + assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes() + assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes() assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v6-execution.ndjson" + cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson" + cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson" cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c76b23f..ee1c0ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,8 +80,8 @@ jobs: staging="droste-relay-$GITHUB_REF_NAME" mkdir -p "$staging/conformance" relay-dist cp src/droste/substrates/_relay/*.ts pyodide/README.md "$staging/" - cp src/droste/testing/fixtures/trace-v6-execution.ndjson \ - src/droste/testing/fixtures/trace-v6-lifecycle.ndjson \ + cp src/droste/testing/fixtures/trace-v7-execution.ndjson \ + src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \ src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$staging/conformance/" printf '%s %s\n' "$GITHUB_REF_NAME" "$GITHUB_SHA" > "$staging/DROSTE_VERSION" diff --git a/AGENTS.md b/AGENTS.md index 0714b90..49a5ad7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ evidence with that status rather than leaving the model to interpret prefixes. ## Trace ABI -- Every structured event is a strict Trace ABI v6 value. Stamp it exactly once +- Every structured event is a strict Trace ABI v7 value. Stamp it exactly once through `ExecutionContext`; do not emit raw or partially enveloped event dictionaries at host boundaries. - Treat every envelope/body, classification, and ordering change as an ABI @@ -189,7 +189,7 @@ evidence with that status rather than leaving the model to interpret prefixes. - A strict published event vocabulary/body change requires a Trace ABI bump and, when embedded in runner output, an atomic runner-protocol bump. Do not expand an old strict version in place or add a compatibility decoder in the engine. -- Trace ABI v6 usage is `resolved` only when both root and subcall scopes have +- Trace ABI v7 usage is `resolved` only when both root and subcall scopes have complete provider usage. Missing or malformed usage preserves any known counts, marks that scope `complete=false`, and makes terminal usage `kind="partial"`; never report conservative reservations as provider usage. @@ -239,7 +239,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial evidence rather than invalidating an otherwise valid envelope. - `reasoning_tokens` is a non-negative breakdown inside `completion_tokens`. Preserve it through internal usage copies, folds, and root/subcall - `ExecutionStats`, but do not add it to totals or Trace ABI v6 events. + `ExecutionStats`, but do not add it to totals or Trace ABI v7 events. Observation basis and reasoning usage are internal callback/accounting facts; do not add them to the public usage projections. @@ -272,7 +272,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial and settle conservatively rather than falling back to the start estimate. - `ExecutionStats` folds those same cache classes separately for root and - subcall scopes. Trace ABI v6 exposes them in every durable usage breakdown; + subcall scopes. Trace ABI v7 exposes them in every durable usage breakdown; complete scopes require the disjoint cache classes to fit inside inclusive input tokens, while partial scopes preserve independently validated counts. ModelRelay names the classes `cache_read_input_tokens` and diff --git a/UPGRADING.md b/UPGRADING.md index fcf7dbd..32bd56d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -13,6 +13,30 @@ consumers, and Pyodide-substrate integrations staging the Deno relay. ## Unreleased (post-0.21.1) +### Trace ABI v7 publishes answer-state checkpoints + +The new configurable `checkpoint` event carries the draft the engine currently +holds — `checkpoint_seq`, `draft`, `draft_chars`, `ready`, and an opaque +`payload` — after every executed step whose draft moved. A host that loses the +process (a watchdog kill, a crashed substrate) can render the last checkpoint +instead of nothing. It is retention-gated like every other content-bearing +event, so it reaches a terminal record only when named in +`TraceRetentionPolicy.retain`. + +`RLMConfig.checkpoint_payload_provider` is the optional callable that fills +`payload`. It returns any JSON object or `None`; the engine never inspects the +value and never schema-checks it. A provider that raises is reported through a +`RuntimeWarning` and the checkpoint carries `payload: null` — a checkpoint can +never fail a run. + +Hosts must accept Trace ABI 7 and update to the `trace_v7_*` conformance +fixtures (`droste.testing.trace_v7_lifecycle_ndjson` / +`trace_v7_execution_ndjson`, backing `trace-v7-lifecycle.ndjson` / +`trace-v7-execution.ndjson`). Scaffold manifests report `abis.trace: 7`, so +every manifest id changes; pinned ids must be re-derived. Strict v6 readers +reject the new event and the new version, so this is an atomic consumer +migration. + ## 0.21.1 (from 0.21.0) ### Streamed Responses preserve their terminal stop reason diff --git a/docs/architecture.md b/docs/architecture.md index d601ee3..8020424 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -293,14 +293,14 @@ exceptions use the same closed five-field shape as other preflight responses; run exceptions use the ordinary run response shape. Completed responses also carry the policy-resolved -[Trace ABI v6](trace-abi.md) `run_record`. Live events and terminal records use +[Trace ABI v7](trace-abi.md) `run_record`. Live events and terminal records use the same strict envelope and projection. Persistence remains a host I/O decision; the engine never opens a trace store. The Deno/Pyodide relay keeps its three process output concerns physically separate. fd1 carries exactly one unary response JSON line. A required `DROSTE_RELAY_EVENT_FD` names one inherited writable descriptor (fd3 by -convention) that carries canonical Trace ABI v6 NDJSON only. An external +convention) that carries canonical Trace ABI v7 NDJSON only. An external launcher also includes that number in `DENO_EXTRA_STDIO_FDS`, which Deno consumes at startup to register inherited descriptors above fd2. Passing an OS-level descriptor without this Deno marker leaves it unavailable to relay @@ -376,7 +376,7 @@ are versioned, each by a single integer: Protocol v9 embeds Trace ABI v5 and its transient cumulative `usage_progress` event at settled root/subcall boundaries. The bump prevents a v8 host from silently dropping the live accounting signal. - Protocol v10 embeds Trace ABI v6 and scaffold manifest v3, adding the + Protocol v10 embeds Trace ABI v7 and scaffold manifest v3, adding the required seventh `max_iterations` authorization field. The exact seven-field budget prevents a v9 host from silently submitting an unbounded loop contract. diff --git a/docs/budgets.md b/docs/budgets.md index 0b51df4..757214c 100644 --- a/docs/budgets.md +++ b/docs/budgets.md @@ -70,7 +70,7 @@ reservations fails loudly. ## Trace facts -Every mutation is a durable Trace ABI v6 `budget` event from +Every mutation is a durable Trace ABI v7 `budget` event from `source="budget_ledger"`: `reserve`, `commit`, `refund`, or `exhaust`. Mutation events carry `resource`, non-negative `amount`, and `call_id`. The terminal snapshot records the configured, consumed, and remaining vectors. diff --git a/docs/scaffold-manifest.md b/docs/scaffold-manifest.md index 116c76d..ed7d576 100644 --- a/docs/scaffold-manifest.md +++ b/docs/scaffold-manifest.md @@ -24,7 +24,7 @@ inference facts. "schema_version": 3, "engine": {"version": "0.10.6", "source_revision": null}, "abis": { - "kernel": 1, "capability": 1, "trace": 6, + "kernel": 1, "capability": 1, "trace": 7, "prompt_pack": 2, "provider": 4, "runner": 10 }, "prompt_pack": { diff --git a/docs/trace-abi.md b/docs/trace-abi.md index 7dc280a..9d2a5bd 100644 --- a/docs/trace-abi.md +++ b/docs/trace-abi.md @@ -1,4 +1,4 @@ -# Trace ABI v6 +# Trace ABI v7 Droste exposes one append-only event stream and one policy-resolved terminal `RunRecord`. The engine creates values; it does not choose a database or write @@ -15,7 +15,7 @@ does not change merely because a released fixture is added. ## Event envelope -Every event is a strict v6 value with these fields: +Every event is a strict v7 value with these fields: ```json { @@ -23,7 +23,7 @@ Every event is a strict v6 value with these fields: "seq": 1, "timestamp": "2026-07-14T05:00:00Z", "type": "progress", - "version": 6, + "version": 7, "persistence_class": "transient", "parent_run_id": "optional-parent", "depth": 0, @@ -48,7 +48,7 @@ The persistence class is exhaustive and fixed by event type: | Class | Event types | Rule | | --- | --- | --- | | `durable` | `usage`, `budget`, `policy`, `capability`, `done` | Always in the terminal record | -| `configurable` | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay` | Included only when named by `TraceRetentionPolicy.retain` | +| `configurable` | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay`, `checkpoint` | Included only when named by `TraceRetentionPolicy.retain` | | `transient` | `startup`, `progress`, `reasoning_delta`, `usage_progress` | Live delivery only; never in the terminal record | Retention governs the terminal record, not the live channel. `result` is @@ -56,7 +56,7 @@ always delivered once before `done`, even when it is not retained. `replay` is different: it is emitted only when the host explicitly selects replay retention. -## Exhaustive v6 bodies +## Exhaustive v7 bodies Every event body has a fixed top-level schema. Optional fields are marked `?`. Objects named below are JSON objects; all other types are primitive. @@ -75,6 +75,7 @@ Objects named below are JSON objects; all other types are primitive. | `repair` | `phase: "start"|"completion"|"failure"`, `kind: "missing_code"|"execution_error"|"terminal"`, `iteration`, `error?` only on failure | | `extract` | `phase: "start"|"completion"|"failure"`, `iteration`, `extract_error?` only on failure | | `result`, `replay` | `result: object` | +| `checkpoint` | `iteration: integer`, `checkpoint_seq: integer`, `draft: string`, `draft_chars: integer`, `ready: boolean`, `payload: object|null` | | `usage_progress` | `boundary: "root"|"subcall"`, `kind`, cumulative `root`, `subcall`, `unattributed`, and `total_tokens` | | `usage` | `kind: "resolved"|"partial"`, `root: object`, `subcall: object`, `unattributed: object`, `total_tokens: integer`, `wall_time_ms: integer` | | `budget` | `kind: "snapshot"|"mutation"`, `source: string`, plus the kind-specific fields below | @@ -118,7 +119,7 @@ partial observation preserves its reported counters and marks the affected scope incomplete. The last progress snapshot reconciles with terminal `usage` unless a later boundary has no numeric usage observation. -The budget body remains a discriminated event in Trace ABI v6. The terminal snapshot uses +The budget body remains a discriminated event in Trace ABI v7. The terminal snapshot uses `kind="snapshot"`, `source="budget_ledger"`, and `configured`, `consumed`, and `remaining` objects. The configured object includes the structural `max_iterations` ceiling; terminal `iterations` records how many iterations @@ -193,6 +194,24 @@ start and exactly one completion or failure once entered. Extract fallback does the same; a failure carries the typed `extract_error`. The canonical `result` still carries the unary-equivalent answer and `done` remains content-free. +## Answer-state checkpoints + +`checkpoint` publishes the answer state the engine holds right now, so a host +that loses the process still has the last draft it saw. The engine emits one +after every executed step whose draft moved, or whose host had something of its +own to add. `checkpoint_seq` starts at one and strictly increases within a +`run_id`; `draft_chars` always equals `len(draft)`. + +`payload` is opaque. Hosts fill it through `RLMConfig.checkpoint_payload_provider`, +a callable returning any JSON object or `None`. The engine carries the value +across without inspecting it and never schema-checks its contents, exactly as +the relay carries adapter `meta`. A provider that raises is reported and the +checkpoint carries `payload: null`: a checkpoint can never fail a run. + +The event is `configurable`, not `durable`. It carries draft content, so it +reaches a terminal record only when a host names it in +`TraceRetentionPolicy.retain`. + ## Terminal reconciliation Finalization emits resolved `usage`, `budget`, and `policy`; always delivers the @@ -211,12 +230,12 @@ and sdist. Python consumers load them through package resources: ```python from droste.testing import ( runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) -execution_lines = trace_v6_execution_ndjson().splitlines() -event_lines = trace_v6_lifecycle_ndjson().splitlines() +execution_lines = trace_v7_execution_ndjson().splitlines() +event_lines = trace_v7_lifecycle_ndjson().splitlines() pre_admission_refusal = runner_v10_refusal_ndjson() ``` diff --git a/examples/pyodide-host/e2e_test.ts b/examples/pyodide-host/e2e_test.ts index 8910786..c5507d6 100644 --- a/examples/pyodide-host/e2e_test.ts +++ b/examples/pyodide-host/e2e_test.ts @@ -33,7 +33,7 @@ const RUNNER_REFUSAL_FIXTURE = new URL( import.meta.url, ); const TRACE_LIFECYCLE_FIXTURE = new URL( - "../../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); const TEST_BUDGET = { diff --git a/pyodide/README.md b/pyodide/README.md index fae960c..fea959b 100644 --- a/pyodide/README.md +++ b/pyodide/README.md @@ -243,7 +243,7 @@ The process has three independent output lanes: | Descriptor | Contract | |------------|----------| | fd1 | Exactly one unary response JSON line. Adapter-owned responses use its HostResponse schema; pre-adapter event-channel failures use the relay-level error above. | -| configured event descriptor (fd3 by convention) | Canonical Trace ABI v6 NDJSON only. | +| configured event descriptor (fd3 by convention) | Canonical Trace ABI v7 NDJSON only. | | fd2 | Diagnostics only; never parse or promote these bytes as events. | Drain fd2 and the event descriptor concurrently. A hard cancellation or @@ -466,7 +466,7 @@ Roughly three tiers of coverage: - **Extract-fallback failure rate is unknown.** When `max_iterations` is exhausted without `answer["ready"]`, one more LLM call tries to synthesize a best-effort answer; a failure there now surfaces as a structured - `extract_error` (result field + the Trace ABI v6 `extract` failure event) instead of + `extract_error` (result field + the Trace ABI v7 `extract` failure event) instead of silently falling back to raw loop output, but there's no data yet on how often that call actually fails or why. No retry has been added — that's a decision for once real failure data exists, not before. diff --git a/pyodide/event_channel_probe.ts b/pyodide/event_channel_probe.ts index 93e6d3a..eb9ab23 100644 --- a/pyodide/event_channel_probe.ts +++ b/pyodide/event_channel_probe.ts @@ -4,7 +4,7 @@ const mode = Deno.args[0]; const channel = eventChannelFromEnvironment(); const fixture = await Deno.readTextFile( new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ), ); diff --git a/pyodide/event_channel_test.ts b/pyodide/event_channel_test.ts index 508689a..b8a7d1c 100644 --- a/pyodide/event_channel_test.ts +++ b/pyodide/event_channel_test.ts @@ -8,7 +8,7 @@ import { import { isRlmEvent } from "../src/droste/substrates/_relay/events.ts"; const TRACE_LIFECYCLE_FIXTURE = new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); diff --git a/pyodide/events_test.ts b/pyodide/events_test.ts index 7728b8c..80867fb 100644 --- a/pyodide/events_test.ts +++ b/pyodide/events_test.ts @@ -48,6 +48,14 @@ const BODIES: Record> = { extract: { phase: "start", iteration: 1 }, result: { result: {} }, replay: { result: {} }, + checkpoint: { + iteration: 1, + checkpoint_seq: 1, + draft: "draft", + draft_chars: 5, + ready: false, + payload: null, + }, usage: { kind: "resolved", root: { @@ -161,7 +169,7 @@ function wire( run_id: "run-1", seq: 1, timestamp: "2026-07-14T00:00:00Z", - version: 6, + version: 7, persistence_class: persistence ?? PERSISTENCE_BY_TYPE[type], depth: 0, ...body, @@ -180,6 +188,52 @@ Deno.test("carries the real payload for a code event (live code streaming)", () ); }); +// `draft_chars` is Python's len() — code points. Comparing it against +// String.length (UTF-16 code units) drops every draft containing a non-BMP +// character, and keeps dropping them, because the draft keeps the character. +// A messaging corpus is full of emoji, so this is the common case, not an edge. +Deno.test("forwards a checkpoint whose draft is not ASCII", () => { + for ( + const draft of [ + "Sarah said 🎉", + "𝐛𝐨𝐥𝐝 math alphanumerics", + "𠮷野家", // astral CJK + "family: 👨‍👩‍👧‍👦", + "flag: 🇯🇵", + ] + ) { + const body = { + iteration: 1, + checkpoint_seq: 1, + draft, + // What droste actually stamps: len(draft) in Python. + draft_chars: [...draft].length, + ready: false, + payload: null, + }; + assert(isRlmEvent(wire("checkpoint", body)), `should forward: ${draft}`); + } +}); + +Deno.test("still rejects a checkpoint whose draft_chars is simply wrong", () => { + const draft = "Sarah said 🎉"; + for (const draft_chars of [[...draft].length + 1, draft.length + 5, 0]) { + assert( + !isRlmEvent( + wire("checkpoint", { + iteration: 1, + checkpoint_seq: 1, + draft, + draft_chars, + ready: false, + payload: null, + }), + ), + `should drop draft_chars=${draft_chars}`, + ); + } +}); + Deno.test("drops non-events: loader chatter, stray prints, empty lines", () => { for ( const noise of [ @@ -329,7 +383,7 @@ Deno.test("successful output beginning ERROR remains an output event", () => { Deno.test("Python and relay accept the same execution golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v6-execution.ndjson", + "../src/droste/testing/fixtures/trace-v7-execution.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); @@ -373,7 +427,7 @@ Deno.test("Python and relay accept the same execution golden NDJSON", async () = Deno.test("Python and relay accept the same lifecycle golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v6-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); @@ -551,6 +605,7 @@ Deno.test("vocabulary matches the engine's emitters", () => { [ "budget", "capability", + "checkpoint", "code", "done", "execution_error", diff --git a/src/droste/__init__.py b/src/droste/__init__.py index fe31564..cdcecca 100644 --- a/src/droste/__init__.py +++ b/src/droste/__init__.py @@ -105,7 +105,11 @@ preflight_rlm, run_rlm, ) -from .loop.step import ReadyMetadataValidator, ReadyMetadataValidatorError +from .loop.step import ( + CheckpointPayloadProvider, + ReadyMetadataValidator, + ReadyMetadataValidatorError, +) from .policy import PolicyHints from .prompts.builder import SystemPromptBuilder from .prompts.pack import ( @@ -163,6 +167,7 @@ "RLMPreflight", "RLMResult", "RLM_PREFLIGHT_SCHEMA_VERSION", + "CheckpointPayloadProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", "extract_code_block", diff --git a/src/droste/execution/progress.py b/src/droste/execution/progress.py index e457491..63a571d 100644 --- a/src/droste/execution/progress.py +++ b/src/droste/execution/progress.py @@ -45,6 +45,7 @@ "extract", # discriminated terminal extraction lifecycle facts "result", # canonical unary-equivalent final result (without trajectory) "replay", # configurable replay input/output details + "checkpoint", # {checkpoint_seq, draft, draft_chars, ready, payload} answer state "usage_progress", # transient cumulative usage at a settled model boundary "usage", # durable resolved token/call accounting "budget", # durable configured/consumed budget facts @@ -147,6 +148,29 @@ def extract_event( return value +def checkpoint_event( + iteration: int, + checkpoint_seq: int, + draft: str, + *, + ready: bool, + payload: Any = None, +) -> dict[str, Any]: + """Build one answer-state checkpoint. + + ``payload`` is an opaque host value: the engine carries it across without + ever inspecting it, exactly as the relay carries adapter ``meta``.""" + return { + "type": "checkpoint", + "iteration": iteration, + "checkpoint_seq": checkpoint_seq, + "draft": draft, + "draft_chars": len(draft), + "ready": ready, + "payload": payload, + } + + # --- sinks ------------------------------------------------------------------- diff --git a/src/droste/execution/trace.py b/src/droste/execution/trace.py index e4781c4..9cea952 100644 --- a/src/droste/execution/trace.py +++ b/src/droste/execution/trace.py @@ -17,7 +17,7 @@ from typing import Any, Callable, Mapping from uuid import uuid4 -TRACE_ABI_VERSION = 6 +TRACE_ABI_VERSION = 7 class PersistenceClass(str, Enum): @@ -41,6 +41,7 @@ class PersistenceClass(str, Enum): "repair", "result", "replay", + "checkpoint", } ) TRANSIENT_EVENT_TYPES = frozenset({"startup", "progress", "reasoning_delta", "usage_progress"}) @@ -57,7 +58,7 @@ class PersistenceClass(str, Enum): EventFieldType = type | tuple[type, ...] EventBodySchema = tuple[Mapping[str, EventFieldType], Mapping[str, EventFieldType]] -# One exhaustive v6 table. The first mapping is required fields; the second is +# One exhaustive v7 table. The first mapping is required fields; the second is # optional fields. Nested broker/result values keep their own schema authority. EVENT_BODY_SCHEMAS: Mapping[str, EventBodySchema] = MappingProxyType( { @@ -113,6 +114,19 @@ class PersistenceClass(str, Enum): ), "result": ({"result": Mapping}, {}), "replay": ({"result": Mapping}, {}), + "checkpoint": ( + { + "iteration": int, + "checkpoint_seq": int, + "draft": str, + "draft_chars": int, + "ready": bool, + # Opaque host value. The envelope pins the container only; the + # contents are never inspected and never schema-checked. + "payload": (Mapping, _NONE_TYPE), + }, + {}, + ), "usage_progress": ( { "boundary": str, @@ -184,11 +198,11 @@ def _matches_field_type(value: Any, expected: EventFieldType) -> bool: def validate_event_body(event_type: str, body: Mapping[str, Any]) -> None: - """Validate one body against the exhaustive Trace ABI v6 table.""" + """Validate one body against the exhaustive Trace ABI v7 table.""" try: required, optional = EVENT_BODY_SCHEMAS[event_type] except KeyError as exc: - raise ValueError(f"event type {event_type!r} has no v5 body schema") from exc + raise ValueError(f"event type {event_type!r} has no v7 body schema") from exc missing = required.keys() - body.keys() if missing: raise ValueError(f"event {event_type!r} missing body fields: " + ", ".join(sorted(missing))) @@ -231,9 +245,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if event_type == "subcall": phase = body["phase"] if phase not in {"start", "progress", "completion", "failure"}: - raise ValueError("subcall phase is not recognized by Trace ABI v6") + raise ValueError("subcall phase is not recognized by Trace ABI v7") if body["operation"] not in {"llm_query", "llm_batch", "llm_batch_with_errors"}: - raise ValueError("subcall operation is not recognized by Trace ABI v6") + raise ValueError("subcall operation is not recognized by Trace ABI v7") if not body["call_id"]: raise ValueError("subcall call_id must not be empty") if body["iteration"] < 1: @@ -292,9 +306,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("subcall batch_count must be non-negative") elif event_type == "repair": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("repair phase is not recognized by Trace ABI v6") + raise ValueError("repair phase is not recognized by Trace ABI v7") if body["kind"] not in {"missing_code", "execution_error", "terminal"}: - raise ValueError("repair kind is not recognized by Trace ABI v6") + raise ValueError("repair kind is not recognized by Trace ABI v7") if body["iteration"] < 1: raise ValueError("repair iteration must be positive") error = body.get("error") @@ -310,7 +324,7 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("repair start/completion cannot carry error") elif event_type == "extract": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("extract phase is not recognized by Trace ABI v6") + raise ValueError("extract phase is not recognized by Trace ABI v7") if body["iteration"] < 1: raise ValueError("extract iteration must be positive") extract_error = body.get("extract_error") @@ -324,6 +338,13 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: ) elif extract_error is not None: raise ValueError("extract start/completion cannot carry extract_error") + elif event_type == "checkpoint": + if body["iteration"] < 1: + raise ValueError("checkpoint iteration must be positive") + if body["checkpoint_seq"] < 1: + raise ValueError("checkpoint checkpoint_seq must be positive") + if body["draft_chars"] != len(body["draft"]): + raise ValueError("checkpoint draft_chars must equal the draft length") elif event_type in {"usage", "usage_progress"}: if event_type == "usage_progress" and body["boundary"] not in {"root", "subcall"}: raise ValueError("usage_progress boundary must be 'root' or 'subcall'") @@ -408,7 +429,7 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if not required <= body.keys(): raise ValueError("budget mutation requires action, resource, and amount") if body["action"] not in {"reserve", "commit", "refund", "exhaust"}: - raise ValueError("budget mutation action is not recognized by Trace ABI v6") + raise ValueError("budget mutation action is not recognized by Trace ABI v7") if body["amount"] < 0: raise ValueError("budget mutation amount must be non-negative") else: @@ -465,10 +486,10 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: ScaffoldManifest.from_dict(scaffold_manifest) elif event_type == "policy": if body["outcome"] not in {"passed", "violated", "not_evaluated", "not_enforced"}: - raise ValueError("policy outcome is not recognized by Trace ABI v6") + raise ValueError("policy outcome is not recognized by Trace ABI v7") elif event_type == "done": if body["status"] not in {"success", "error", "cancelled"}: - raise ValueError("done status is not recognized by Trace ABI v6") + raise ValueError("done status is not recognized by Trace ABI v7") if body["iterations"] < 0: raise ValueError("done iterations must be non-negative") stdout_chars = body.get("stdout_chars") @@ -619,7 +640,7 @@ def as_dict(self) -> dict[str, Any]: def parse_event(value: RunEvent | Mapping[str, Any]) -> RunEvent: - """The one strict parser for Trace ABI v6 event values.""" + """The one strict parser for Trace ABI v7 event values.""" if isinstance(value, RunEvent): return value required = { @@ -860,6 +881,7 @@ class TraceRecorder: _events: list[RunEvent] = field(default_factory=list, init=False, repr=False) _terminal_record: RunRecord | None = field(default=None, init=False, repr=False) _started_monotonic: float | None = field(default=None, init=False, repr=False) + _last_checkpoint_seq: int = field(default=0, init=False, repr=False) _lock: Lock = field(default_factory=Lock, init=False, repr=False) def __post_init__(self) -> None: @@ -921,6 +943,14 @@ def _append_locked(self, event: Mapping[str, Any]) -> RunEvent: persistence_class=persistence_class_for(event_type), body=body, ) + if event_type == "checkpoint": + # ``checkpoint_seq`` is the emitter's answer-state ordinal, so the + # recorder — the one authority for a run_id — pins it strictly + # increasing the way it pins ``seq``. + checkpoint_seq = value.body["checkpoint_seq"] + if checkpoint_seq <= self._last_checkpoint_seq: + raise ValueError("checkpoint_seq must strictly increase within a run") + self._last_checkpoint_seq = checkpoint_seq self._events.append(value) return value diff --git a/src/droste/loop/__init__.py b/src/droste/loop/__init__.py index cec1231..853ede2 100644 --- a/src/droste/loop/__init__.py +++ b/src/droste/loop/__init__.py @@ -1,6 +1,10 @@ from .code_extractor import extract_code_block from .rlm import RLMConfig, RLMPreflight, RLMResult, preflight_rlm, run_rlm -from .step import ReadyMetadataValidator, ReadyMetadataValidatorError +from .step import ( + CheckpointPayloadProvider, + ReadyMetadataValidator, + ReadyMetadataValidatorError, +) from .trajectory import IterationRecord __all__ = [ @@ -11,6 +15,7 @@ "RLMResult", "extract_code_block", "IterationRecord", + "CheckpointPayloadProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", ] diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index cedcbe5..56e6af3 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -18,6 +18,7 @@ ) from ..execution.progress import ( EventCallback, + checkpoint_event, extract_event, iteration_start_event, llm_response_event, @@ -139,6 +140,30 @@ def _warn_environment_cleanup(cleanup_error: BaseException) -> None: ) +def _warn_checkpoint(reason: str, error: BaseException) -> None: + """Report a degraded checkpoint. A checkpoint never fails a run.""" + + detail = " ".join(str(error).split())[:1_000] + warnings.warn( + f"RLM checkpoint {reason}: {type(error).__name__}: {detail}", + RuntimeWarning, + stacklevel=3, + ) + + +def _checkpoint_payload(cfg: "RLMConfig") -> Any: + """Resolve the host's opaque checkpoint payload, never inspecting it.""" + + provider = cfg.checkpoint_payload_provider + if provider is None: + return None + try: + return provider() + except Exception as provider_error: + _warn_checkpoint("payload provider failed", provider_error) + return None + + ProgressCallback = Any @@ -819,6 +844,8 @@ def run_rlm( transcript_window: list[TranscriptWindowEntry] = [] previous_draft_snapshot = str(answer.get("content", "")) code = "" + checkpoint_seq = 0 + checkpoint_draft = "" def step_kwargs() -> dict[str, Any]: return dict( @@ -834,6 +861,32 @@ def step_kwargs() -> dict[str, Any]: ready_metadata_validator=ready_metadata_validator, ) + def emit_checkpoint() -> None: + """Publish answer state so a killed run stays renderable by a host. + + A checkpoint is worth emitting when the draft moved or the host has + something of its own to add; neither the host's provider nor the + emission itself may fail the run.""" + nonlocal checkpoint_seq, checkpoint_draft + draft = str(answer.get("content") or "") + payload = _checkpoint_payload(cfg) + if draft == checkpoint_draft and payload is None: + return + checkpoint_seq += 1 + checkpoint_draft = draft + try: + context.emit_event( + checkpoint_event( + iterations, + checkpoint_seq, + draft, + ready=bool(answer.get("ready")), + payload=payload, + ) + ) + except Exception as checkpoint_error: + _warn_checkpoint("dropped", checkpoint_error) + def early_result(run_error: RLMError | None) -> RLMResult: return finalize( answer_text=_best_answer(answer, last_output, last_response, last_execution_status), @@ -1043,6 +1096,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr last_execution_status = outcome.execution_status error = outcome.error answer_metadata = outcome.answer_metadata + emit_checkpoint() if outcome.error is None: has_successful_step = True trajectory.append( @@ -1136,6 +1190,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr last_execution_status = outcome.execution_status error = outcome.error answer_metadata = outcome.answer_metadata + emit_checkpoint() if outcome.error is None: code = repaired_code has_successful_step = True diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index 1df8c0a..e457879 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -79,6 +79,11 @@ Sequence[str], ] +# Resolves the opaque value carried by each answer-state checkpoint. The engine +# never inspects what this returns and never schema-checks it: hosts use it to +# ferry their own answer-critical state (whatever that is) alongside the draft. +CheckpointPayloadProvider: TypeAlias = Callable[[], Any] + class ReadyMetadataValidatorError(RuntimeError): """Trusted host validator failed instead of returning repairable violations.""" @@ -108,6 +113,11 @@ class RLMConfig: on_run_record: RunRecordCallback | None = None rollout: RolloutConfiguration = field(default_factory=RolloutConfiguration) checkpoint_requirements: ScaffoldRequirements | None = None + # Called immediately before each answer-state checkpoint. Returns a + # JSON-serializable value, or None for "nothing to add". A provider that + # raises is reported and the checkpoint carries a null payload: a + # checkpoint can never fail a run. + checkpoint_payload_provider: CheckpointPayloadProvider | None = None @dataclass diff --git a/src/droste/substrates/_relay/events.ts b/src/droste/substrates/_relay/events.ts index fc0afef..c4375a9 100644 --- a/src/droste/substrates/_relay/events.ts +++ b/src/droste/substrates/_relay/events.ts @@ -20,6 +20,7 @@ export const RLM_EVENT_TYPES = new Set([ "extract", // discriminated extract start/completion/failure "result", // canonical unary-equivalent final result "replay", // configurable replay details + "checkpoint", // {checkpoint_seq, draft, draft_chars, ready, payload} answer state "usage_progress", // cumulative usage at a settled root or subcall boundary "usage", // durable resolved-or-partial provider accounting "budget", // durable budget facts @@ -43,6 +44,7 @@ export const PERSISTENCE_BY_TYPE: Readonly> = { extract: "configurable", result: "configurable", replay: "configurable", + checkpoint: "configurable", usage: "durable", budget: "durable", policy: "durable", @@ -311,6 +313,26 @@ function validBody(type: string, body: Record): boolean { case "result": case "replay": return exactBody(body, ["result"]) && isObject(body.result); + case "checkpoint": + // `payload` is opaque: the container is pinned, the contents never are. + return exactBody(body, [ + "iteration", + "checkpoint_seq", + "draft", + "draft_chars", + "ready", + "payload", + ]) && integerField("iteration") && Number(body.iteration) >= 1 && + integerField("checkpoint_seq") && Number(body.checkpoint_seq) >= 1 && + stringField("draft") && integerField("draft_chars") && + // `draft_chars` is Python's len(): a COUNT OF CODE POINTS. JavaScript's + // String.length counts UTF-16 code units, so any non-BMP character + // (emoji, astral CJK) makes the two disagree and would drop the frame + // here — silently, and for every later checkpoint too, since the draft + // keeps the character. Spreading the string iterates code points. + Number(body.draft_chars) === [...String(body.draft)].length && + typeof body.ready === "boolean" && + (body.payload === null || isObject(body.payload)); case "usage_progress": return validUsageBody(body, true); case "usage": @@ -409,7 +431,7 @@ export function isRlmEvent(line: string): boolean { Number.isInteger(o.seq) && o.seq > 0 && typeof o.timestamp === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(o.timestamp) && - o.version === 6 && + o.version === 7 && o.persistence_class === PERSISTENCE_BY_TYPE[o.type] && Number.isInteger(o.depth) && o.depth >= 0 && (o.depth === 0 diff --git a/src/droste/substrates/_relay/relay.ts b/src/droste/substrates/_relay/relay.ts index b85f7f6..0ed5af3 100644 --- a/src/droste/substrates/_relay/relay.ts +++ b/src/droste/substrates/_relay/relay.ts @@ -455,7 +455,7 @@ function writeRelayEvent(obj: Record): void { depth: 1, seq: ++relaySeq, timestamp: new Date().toISOString(), - version: 6, + version: 7, persistence_class: "transient", }; eventChannel.writeFrame(JSON.stringify(event)); diff --git a/src/droste/testing/__init__.py b/src/droste/testing/__init__.py index 6bb5c0b..e6b468a 100644 --- a/src/droste/testing/__init__.py +++ b/src/droste/testing/__init__.py @@ -16,16 +16,16 @@ from .subcall_client import MockSubcallClient -def trace_v6_lifecycle_ndjson() -> bytes: - """Return the shared Trace ABI v6 lifecycle conformance corpus.""" +def trace_v7_lifecycle_ndjson() -> bytes: + """Return the shared Trace ABI v7 lifecycle conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v6-lifecycle.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v7-lifecycle.ndjson").read_bytes() -def trace_v6_execution_ndjson() -> bytes: - """Return the shared Trace ABI v6 response/code/output/error conformance corpus.""" +def trace_v7_execution_ndjson() -> bytes: + """Return the shared Trace ABI v7 response/code/output/error conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v6-execution.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v7-execution.ndjson").read_bytes() def runner_v10_refusal_ndjson() -> bytes: @@ -50,6 +50,6 @@ def runner_v10_refusal_ndjson() -> bytes: "require_ordered_terminal_events", "require_unknown_completion", "run_while_blocked", - "trace_v6_execution_ndjson", - "trace_v6_lifecycle_ndjson", + "trace_v7_execution_ndjson", + "trace_v7_lifecycle_ndjson", ] diff --git a/src/droste/testing/_trace_fixtures.py b/src/droste/testing/_trace_fixtures.py index c7e444c..409ebc2 100644 --- a/src/droste/testing/_trace_fixtures.py +++ b/src/droste/testing/_trace_fixtures.py @@ -22,7 +22,7 @@ def _ndjson(events: tuple[RunEvent, ...]) -> bytes: ) -def build_trace_v6_execution_ndjson() -> bytes: +def build_trace_v7_execution_ndjson() -> bytes: """Build the deterministic code/output/error projection corpus.""" started_at = datetime(2026, 7, 16, tzinfo=timezone.utc) diff --git a/src/droste/testing/fixtures/trace-v6-execution.ndjson b/src/droste/testing/fixtures/trace-v7-execution.ndjson similarity index 74% rename from src/droste/testing/fixtures/trace-v6-execution.ndjson rename to src/droste/testing/fixtures/trace-v7-execution.ndjson index 2794b1d..6ebe74b 100644 --- a/src/droste/testing/fixtures/trace-v6-execution.ndjson +++ b/src/droste/testing/fixtures/trace-v7-execution.ndjson @@ -1,9 +1,9 @@ -{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":6,"persistence_class":"configurable","depth":0} -{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0} -{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":6,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":7,"persistence_class":"configurable","depth":0} +{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0} +{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":7,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} diff --git a/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson b/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson similarity index 85% rename from src/droste/testing/fixtures/trace-v6-lifecycle.ndjson rename to src/droste/testing/fixtures/trace-v7-lifecycle.ndjson index 298ab44..834e9ea 100644 --- a/src/droste/testing/fixtures/trace-v6-lifecycle.ndjson +++ b/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson @@ -1,67 +1,67 @@ -{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":6,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6","scaffold_manifest_version":3} -{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} -{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} -{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} -{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} -{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} -{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} -{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} -{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} -{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} -{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6"},"stdout_chars":13}} -{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:0c03ca97f23fd4fc14ef8373196386a17eafb2c3d70b9adcd85a22b67d460cc6","scaffold_manifest_version":3,"stdout_chars":13} -{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} -{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} -{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7"},"stdout_chars":0}} -{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} -{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} -{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} -{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} -{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a0931838ab34bac30b7f49133a004b0a39c55f190efd91a136b5f084e258d111"},"stdout_chars":4097}} -{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a0931838ab34bac30b7f49133a004b0a39c55f190efd91a136b5f084e258d111","scaffold_manifest_version":3,"stdout_chars":4097} -{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} -{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} -{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7"},"stdout_chars":0}} -{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:601040ba53cd0d35d097fd017d5673e6453e06289f7931d01387a85a4aa65ae7","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} -{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":6,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} -{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":6,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":6,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":6,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} -{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":6,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} -{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":6,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":6,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":6},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:113605ba678f547eab4a9dc81a5e63def14916bb51c50cbfc99ef0397590bf5d"},"stdout_chars":0}} -{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":6,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:113605ba678f547eab4a9dc81a5e63def14916bb51c50cbfc99ef0397590bf5d","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":7,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3} +{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} +{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} +{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} +{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} +{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} +{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} +{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} +{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} +{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} +{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9"},"stdout_chars":13}} +{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3,"stdout_chars":13} +{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} +{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} +{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} +{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} +{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} +{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} +{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775"},"stdout_chars":4097}} +{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775","scaffold_manifest_version":3,"stdout_chars":4097} +{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} +{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} +{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} +{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":7,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} +{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":7,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":7,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":7,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} +{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":7,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} +{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":7,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":7,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d"},"stdout_chars":0}} +{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":7,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d","scaffold_manifest_version":3,"stdout_chars":0} diff --git a/tests/test_answer_checkpoints.py b/tests/test_answer_checkpoints.py new file mode 100644 index 0000000..e1b7e0d --- /dev/null +++ b/tests/test_answer_checkpoints.py @@ -0,0 +1,263 @@ +"""Answer-state checkpoints (Trace ABI v7). + +All answer-critical state used to live only in the loop's memory until the +terminal result. A host that lost the process — a watchdog kill, a crashed +substrate — lost every draft with it. The `checkpoint` event publishes that +state as it moves, so a kill becomes a render rather than a recovery. + +The engine stays agnostic about what a host considers answer-critical: +`payload` is opaque, carried across without inspection or schema checks, and +nothing about a checkpoint may ever fail a run. +""" + +from __future__ import annotations + +import pytest + +from droste import RLMConfig, RunEvent, TraceRetentionPolicy, run_rlm +from droste.execution.progress import EVENT_TYPES, checkpoint_event +from droste.execution.trace import ( + TRACE_ABI_VERSION, + PersistenceClass, + TraceRecorder, + persistence_class_for, + select_retained_events, +) +from droste.protocols.llm_client import TokenUsage +from droste.testing import MockEnvironment, MockLLMClient, MockResponse, MockSubcallClient + + +def _reply(code: str) -> MockResponse: + return MockResponse( + text=f"```python\n{code}\n```", + usage=TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2, exact=True), + ) + + +def _draft_reply(content: str) -> MockResponse: + return _reply(f"answer['content'] = {content!r}") + + +def _ready_reply(content: str) -> MockResponse: + return _reply(f"answer['content'] = {content!r}\nanswer['ready'] = True") + + +def _run(responses: list[MockResponse], **config_kwargs: object) -> tuple[object, list[dict]]: + events: list[dict] = [] + result = run_rlm( + question="q", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(**config_kwargs), # type: ignore[arg-type] + on_event=events.append, + ) + return result, [event for event in events if event["type"] == "checkpoint"] + + +def _valid_body(**overrides: object) -> dict[str, object]: + body: dict[str, object] = { + "type": "checkpoint", + "iteration": 1, + "checkpoint_seq": 1, + "draft": "partial answer", + "draft_chars": len("partial answer"), + "ready": False, + "payload": None, + } + body.update(overrides) + return body + + +# --- Layer 1: the wire contract ---------------------------------------------- + + +def test_trace_abi_is_version_seven_and_knows_checkpoint() -> None: + assert TRACE_ABI_VERSION == 7 + assert "checkpoint" in EVENT_TYPES + + +def test_checkpoint_is_retention_gated_content_not_a_durable_fact() -> None: + """A checkpoint carries the draft itself, so it must be selectable by the + same retention machinery as every other content-bearing event — never + durable, which would persist message-derived content unconditionally.""" + assert persistence_class_for("checkpoint") is PersistenceClass.CONFIGURABLE + + recorder = TraceRecorder(run_id="retention") + event = recorder.append(_valid_body()) + assert select_retained_events((event,), TraceRetentionPolicy()) == () + retained = select_retained_events( + (event,), TraceRetentionPolicy(retain=frozenset({"checkpoint"})) + ) + assert retained == (event,) + + +def test_checkpoint_envelope_is_strict() -> None: + recorder = TraceRecorder(run_id="strict-checkpoint") + + with pytest.raises(ValueError, match="draft_chars must equal the draft length"): + recorder.append(_valid_body(draft_chars=3)) + with pytest.raises(ValueError, match="checkpoint_seq must be positive"): + recorder.append(_valid_body(checkpoint_seq=0)) + with pytest.raises(ValueError, match="checkpoint iteration must be positive"): + recorder.append(_valid_body(iteration=0)) + with pytest.raises(TypeError, match="ready.*invalid type"): + recorder.append(_valid_body(ready="no")) + with pytest.raises(ValueError, match="missing body fields: draft"): + body = _valid_body() + del body["draft"] + recorder.append(body) + with pytest.raises(ValueError, match="unknown body fields: evidence"): + recorder.append(_valid_body(evidence=[])) + + +def test_checkpoint_seq_strictly_increases_within_a_run() -> None: + recorder = TraceRecorder(run_id="ordered-checkpoints") + recorder.append(_valid_body(checkpoint_seq=1)) + recorder.append(_valid_body(checkpoint_seq=4)) + + with pytest.raises(ValueError, match="strictly increase"): + recorder.append(_valid_body(checkpoint_seq=4)) + with pytest.raises(ValueError, match="strictly increase"): + recorder.append(_valid_body(checkpoint_seq=2)) + + # A different run owns its own ordinals. + other = TraceRecorder(run_id="other-run") + assert other.append(_valid_body(checkpoint_seq=1)).body["checkpoint_seq"] == 1 + + +def test_checkpoint_payload_is_carried_but_never_schema_checked() -> None: + """The envelope pins the container; the contents are the host's business. + Anything narrower would make droste learn what a host puts in there.""" + payload = { + "anything": [{"nested": 1}, None, "text"], + "the_engine": {"does": {"not": {"care": True}}}, + } + recorder = TraceRecorder(run_id="opaque-payload") + event = recorder.append(_valid_body(payload=payload)) + assert event.as_dict()["payload"] == payload + + assert recorder.append(_valid_body(checkpoint_seq=2, payload=None)).body["payload"] is None + with pytest.raises(TypeError, match="payload.*invalid type"): + recorder.append(_valid_body(checkpoint_seq=3, payload=["not", "an", "object"])) + + +def test_checkpoint_event_builder_derives_draft_chars() -> None: + assert checkpoint_event(2, 3, "abcd", ready=True, payload={"k": 1}) == { + "type": "checkpoint", + "iteration": 2, + "checkpoint_seq": 3, + "draft": "abcd", + "draft_chars": 4, + "ready": True, + "payload": {"k": 1}, + } + + +# --- Layer 2: the loop emission point and the payload hook -------------------- + + +def test_checkpoint_follows_each_executed_step_whose_draft_moved() -> None: + result, checkpoints = _run([_draft_reply("first draft"), _ready_reply("final answer")]) + + assert result.ready + assert [event["checkpoint_seq"] for event in checkpoints] == [1, 2] + assert [event["draft"] for event in checkpoints] == ["first draft", "final answer"] + assert [event["draft_chars"] for event in checkpoints] == [11, 12] + assert [event["ready"] for event in checkpoints] == [False, True] + assert [event["iteration"] for event in checkpoints] == [1, 2] + assert all(event["payload"] is None for event in checkpoints) + assert all(event["version"] == 7 for event in checkpoints) + assert all(event["persistence_class"] == "configurable" for event in checkpoints) + + +def test_repaired_code_checkpoints_its_own_draft() -> None: + """The repaired attempt is what actually ran; its draft must be published + too, or a kill right after a repair renders the pre-repair answer.""" + _, checkpoints = _run( + [ + _reply("answer['content'] = 'salvageable'\nraise ValueError('boom')"), + _ready_reply("repaired answer"), + ] + ) + + assert [event["draft"] for event in checkpoints] == ["salvageable", "repaired answer"] + assert [event["checkpoint_seq"] for event in checkpoints] == [1, 2] + assert [event["ready"] for event in checkpoints] == [False, True] + + +def test_unmoved_draft_without_a_payload_emits_nothing() -> None: + _, checkpoints = _run([_reply("print('no draft')"), _ready_reply("done")]) + + assert [event["draft"] for event in checkpoints] == ["done"] + + +def test_payload_alone_is_reason_enough_to_checkpoint() -> None: + """A host whose own state moved needs a checkpoint even when the draft + stood still — the draft is not the only answer-critical state.""" + payloads = iter([{"host": 1}, {"host": 2}]) + _, checkpoints = _run( + [_reply("print('no draft')"), _ready_reply("done")], + checkpoint_payload_provider=lambda: next(payloads), + ) + + assert [event["draft"] for event in checkpoints] == ["", "done"] + assert [event["payload"] for event in checkpoints] == [{"host": 1}, {"host": 2}] + + +def test_payload_provider_value_reaches_the_wire_unexamined() -> None: + payload = {"opaque": [{"to": "droste"}, 7, None], "totals": {"a": 1}} + _, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=lambda: payload, + ) + + assert [event["payload"] for event in checkpoints] == [payload] + + +def test_raising_payload_provider_reports_and_still_checkpoints() -> None: + def provider() -> dict[str, object]: + raise RuntimeError("host ledger exploded") + + with pytest.warns(RuntimeWarning, match="checkpoint payload provider failed"): + result, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=provider, + ) + + assert result.ready + assert result.answer == "answer" + assert [event["payload"] for event in checkpoints] == [None] + assert [event["draft"] for event in checkpoints] == ["answer"] + + +def test_unrepresentable_payload_drops_the_checkpoint_not_the_run() -> None: + """A payload droste cannot put on the wire is still the host's mistake to + fix, never a reason to lose a completed run.""" + with pytest.warns(RuntimeWarning, match="checkpoint dropped"): + result, checkpoints = _run( + [_ready_reply("answer")], + checkpoint_payload_provider=lambda: object(), + ) + + assert result.ready + assert result.answer == "answer" + assert result.error is None + assert checkpoints == [] + + +def test_checkpoints_reach_a_run_record_only_when_retained() -> None: + result, checkpoints = _run([_ready_reply("answer")]) + assert checkpoints + assert all(event.type != "checkpoint" for event in result.run_record.events) + + retained, _ = _run( + [_ready_reply("answer")], + trace_retention=TraceRetentionPolicy( + retain=frozenset({"checkpoint"}), policy_id="checkpoint-retained" + ), + ) + checkpoint_events: list[RunEvent] = [ + event for event in retained.run_record.events if event.type == "checkpoint" + ] + assert [event.body["draft"] for event in checkpoint_events] == ["answer"] diff --git a/tests/test_event_vocabulary.py b/tests/test_event_vocabulary.py index a63531a..f8c3c1e 100644 --- a/tests/test_event_vocabulary.py +++ b/tests/test_event_vocabulary.py @@ -118,6 +118,7 @@ def test_run_rlm_event_stream_through_attached_sink() -> None: "progress", "code", "output", + "checkpoint", "usage", "budget", "policy", diff --git a/tests/test_runner_subcall_reporting.py b/tests/test_runner_subcall_reporting.py index 0a78df5..da35b27 100644 --- a/tests/test_runner_subcall_reporting.py +++ b/tests/test_runner_subcall_reporting.py @@ -253,7 +253,7 @@ def __init__(self, **kwargs: Any) -> None: "seed": 17, } assert manifest["abis"]["runner"] == 10 - assert manifest["abis"]["trace"] == 6 + assert manifest["abis"]["trace"] == 7 assert manifest["engine"]["source_revision"] == "commit-a" assert manifest["id"].startswith("sha256:") assert "trajectory" not in response @@ -295,7 +295,7 @@ def __init__(self, **kwargs: Any) -> None: ] assert len({event["call_id"] for event in live_subcalls}) == 1 assert live_subcalls[1]["checkpoint"] == {"tokens": 5, "subcalls": 1} - assert all(event["iteration"] == 1 and event["version"] == 6 for event in live_subcalls) + assert all(event["iteration"] == 1 and event["version"] == 7 for event in live_subcalls) assert capability["outcome"]["capability_id"]["operation"] == "llm_query" assert "params" not in capability["outcome"] assert "result" not in capability["outcome"] diff --git a/tests/test_subcall_input_capacity.py b/tests/test_subcall_input_capacity.py index fbc3cc9..fafea8c 100644 --- a/tests/test_subcall_input_capacity.py +++ b/tests/test_subcall_input_capacity.py @@ -349,7 +349,7 @@ def test_runner_preflight_records_declared_capacity() -> None: } -def test_runner_v5_is_refused_before_trace_v6_can_be_ignored() -> None: +def test_runner_v5_is_refused_before_trace_v7_can_be_ignored() -> None: response = run_worker( { "protocol_version": 5, diff --git a/tests/test_trace_abi.py b/tests/test_trace_abi.py index ec84f56..4e85920 100644 --- a/tests/test_trace_abi.py +++ b/tests/test_trace_abi.py @@ -33,10 +33,10 @@ MockResponse, MockSubcallClient, runner_v10_refusal_ndjson, - trace_v6_execution_ndjson, - trace_v6_lifecycle_ndjson, + trace_v7_execution_ndjson, + trace_v7_lifecycle_ndjson, ) -from droste.testing._trace_fixtures import build_trace_v6_execution_ndjson +from droste.testing._trace_fixtures import build_trace_v7_execution_ndjson from droste_runner.run import run as run_worker @@ -233,7 +233,7 @@ def test_usage_progress_emits_cumulative_role_boundaries_without_estimating() -> assert progress[1]["root"]["total_tokens"] == 10 assert progress[1]["subcall"]["total_tokens"] == 7 assert progress[1]["persistence_class"] == "transient" - assert progress[1]["version"] == 6 + assert progress[1]["version"] == 7 def test_concurrent_subcall_usage_progress_is_serialized_and_monotonic() -> None: @@ -582,7 +582,7 @@ def test_run_record_allows_repeated_durable_budget_mutations() -> None: ] -def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: +def test_parser_requires_v7_envelope_and_rejects_false_classification() -> None: with pytest.raises(ValueError, match="missing envelope fields"): parse_event({"type": "code", "iteration": 1, "code": "print(1)"}) @@ -592,7 +592,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "configurable", "depth": 0, "iteration": 1, @@ -607,7 +607,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 2, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "transient", "depth": 0, "engine_version": "0.10.6", @@ -624,7 +624,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 6, + "version": 7, "persistence_class": "durable", "depth": 0, "code": "print(1)", @@ -638,7 +638,7 @@ def test_parser_requires_v6_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T01:00:00+01:00", - "version": 6, + "version": 7, "persistence_class": "transient", "depth": 0, "status": "working", @@ -673,7 +673,7 @@ def test_event_bodies_reject_missing_unknown_and_wrong_primitive_fields() -> Non recorder.finish(terminal) -def test_trace_v6_budget_snapshot_requires_max_iterations() -> None: +def test_trace_v7_budget_snapshot_requires_max_iterations() -> None: recorder = TraceRecorder(run_id="strict-budget") budget = json.loads(json.dumps(_terminal()["budget"])) del budget["configured"]["max_iterations"] @@ -844,11 +844,11 @@ def test_trace_abi_v3_is_rejected_instead_of_dropping_cache_usage() -> None: ) -def _trace_v6_golden_runs() -> dict[str, list[RunEvent]]: +def _trace_v7_golden_runs() -> dict[str, list[RunEvent]]: runs: dict[str, list[RunEvent]] = {} previous_run_id: str | None = None closed: set[str] = set() - for line in trace_v6_lifecycle_ndjson().decode("utf-8").splitlines(): + for line in trace_v7_lifecycle_ndjson().decode("utf-8").splitlines(): event = parse_event(json.loads(line)) if event.run_id != previous_run_id: if event.run_id in closed: @@ -860,9 +860,9 @@ def _trace_v6_golden_runs() -> dict[str, list[RunEvent]]: return runs -def test_trace_v6_execution_fixture_is_canonical_and_exact() -> None: - fixture = trace_v6_execution_ndjson() - assert fixture == build_trace_v6_execution_ndjson() +def test_trace_v7_execution_fixture_is_canonical_and_exact() -> None: + fixture = trace_v7_execution_ndjson() + assert fixture == build_trace_v7_execution_ndjson() events = [parse_event(json.loads(line)) for line in fixture.decode("utf-8").splitlines()] root = events[:6] @@ -894,8 +894,8 @@ def _error_type(value: object) -> object: return value.get("type") if isinstance(value, Mapping) else None -def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: - runs = _trace_v6_golden_runs() +def test_trace_v7_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: + runs = _trace_v7_golden_runs() assert set(runs) == { "golden-success", "golden-recovered", @@ -928,7 +928,7 @@ def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> No "prompt_pack": 1, "provider": 4, "runner": 10, - "trace": 6, + "trace": 7, } assert dict(scaffold_manifest.body["budget"]) == dict(done["budget"]["configured"]) assert ( @@ -991,8 +991,8 @@ def test_trace_v6_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> No assert terminal["usage"]["subcall"]["cache_creation_tokens"] == 1 -def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: - events = [event for run in _trace_v6_golden_runs().values() for event in run] +def test_trace_v7_golden_corpus_covers_each_discriminated_lifecycle() -> None: + events = [event for run in _trace_v7_golden_runs().values() for event in run] subcalls = [event for event in events if event.type == "subcall"] repairs = [event for event in events if event.type == "repair"] extracts = [event for event in events if event.type == "extract"] @@ -1019,7 +1019,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert {event.body["boundary"] for event in usage_progress} == {"root", "subcall"} assert all(event.persistence_class is PersistenceClass.TRANSIENT for event in usage_progress) - output_limit = _trace_v6_golden_runs()["golden-output-limit"] + output_limit = _trace_v7_golden_runs()["golden-output-limit"] assert not any(event.type == "output" for event in output_limit) assert any( event.type == "execution_error" and event.body["error_type"] == "SandboxError" @@ -1043,7 +1043,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: event.body["message"] for event in output_limit if event.type == "execution_error" ) assert f"exceeded {output_manifest['sandbox']['output_chars']} characters" in output_error - cancelled = _trace_v6_golden_runs()["golden-cancelled"] + cancelled = _trace_v7_golden_runs()["golden-cancelled"] assert cancelled[-1].body["status"] == "error" assert any( event.type == "execution_error" and event.body["error_type"] == "CapabilityCallError" @@ -1057,7 +1057,7 @@ def test_trace_v6_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert cancelled[-1].body["usage"]["root"]["complete"] is False assert cancelled[-1].body["usage"]["subcall"]["complete"] is False - extract_failed = _trace_v6_golden_runs()["golden-extract-failed"] + extract_failed = _trace_v7_golden_runs()["golden-extract-failed"] result = next(event.body["result"] for event in extract_failed if event.type == "result") assert result["answer"] == f"Error: {result['error']['message']}" assert result["error"]["details"]["withheld_content"] == "retained evidence" From e71beb82d87afa8aab756d31d4201d08a8e5e8a5 Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Mon, 3 Aug 2026 14:33:48 -0400 Subject: [PATCH 4/5] Let the host say when there is work worth extracting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine decides whether a terminal extract call is worth making from what it owns: a retained draft, or a step that ran to completion. Neither exists when generated code calls a host accessor and then raises before printing — no draft, no successful step, no stdout — so the engine concludes there is nothing to extract while the host is holding every row the answer needed. Observed on a real 314k-message corpus: a run retrieved 84 message GUIDs across four ledger records and returned a bare BudgetExhausted, having never attempted extraction at all. The engine cannot fix this alone without learning what a host data layer is, which is exactly the boundary this design exists to keep. So it asks instead. `RLMConfig.extractable_work_probe` is a host-supplied predicate, consulted only alongside the engine's own test and never replacing it. A probe that raises is treated as "no work" — a terminal recovery path must not be able to fail the run it is recovering. Known follow-up, deliberately not in this commit: unlocking the extract call is necessary but not sufficient. `_trajectory_summary` builds the extract prompt from draft + per-iteration code + stdout, so a run whose data exists only in the host ledger hands the extract pass failing code and nothing else. Measured: extraction now runs and returns InsufficientEvidence rather than being skipped. Closing that needs a second, symmetric hook letting the host contribute opaque context to the extract prompt. Co-Authored-By: Claude Opus 5 (1M context) --- src/droste/__init__.py | 2 + src/droste/loop/__init__.py | 2 + src/droste/loop/rlm.py | 33 ++++++++++++++- src/droste/loop/step.py | 9 ++++ tests/test_wall_clock_fallback.py | 69 +++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/droste/__init__.py b/src/droste/__init__.py index cdcecca..83141a1 100644 --- a/src/droste/__init__.py +++ b/src/droste/__init__.py @@ -107,6 +107,7 @@ ) from .loop.step import ( CheckpointPayloadProvider, + ExtractableWorkProbe, ReadyMetadataValidator, ReadyMetadataValidatorError, ) @@ -168,6 +169,7 @@ "RLMResult", "RLM_PREFLIGHT_SCHEMA_VERSION", "CheckpointPayloadProvider", + "ExtractableWorkProbe", "ReadyMetadataValidator", "ReadyMetadataValidatorError", "extract_code_block", diff --git a/src/droste/loop/__init__.py b/src/droste/loop/__init__.py index 853ede2..50cf0ea 100644 --- a/src/droste/loop/__init__.py +++ b/src/droste/loop/__init__.py @@ -2,6 +2,7 @@ from .rlm import RLMConfig, RLMPreflight, RLMResult, preflight_rlm, run_rlm from .step import ( CheckpointPayloadProvider, + ExtractableWorkProbe, ReadyMetadataValidator, ReadyMetadataValidatorError, ) @@ -16,6 +17,7 @@ "extract_code_block", "IterationRecord", "CheckpointPayloadProvider", + "ExtractableWorkProbe", "ReadyMetadataValidator", "ReadyMetadataValidatorError", ] diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index 56e6af3..2e73260 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -568,6 +568,34 @@ def _has_extractable_work(answer: dict[str, Any], has_successful_step: bool) -> return has_successful_step +def _host_reports_extractable_work(cfg: "RLMConfig") -> bool: + """Whether the host says its own state holds work worth extracting. + + The engine's test above sees only what the engine owns. Generated code that + retrieves real data through a host accessor and then raises before printing + leaves no draft, no successful step, and no stdout — so the engine concludes + there is nothing to extract while the host is holding everything the answer + needed. Asking rather than inferring keeps the engine ignorant of what the + host's data layer is, which is the whole point of the boundary. + + A probe that raises is treated as "no": a terminal recovery path must not be + able to fail the run it is recovering. + """ + + probe = cfg.extractable_work_probe + if probe is None: + return False + try: + return bool(probe()) + except Exception as exc: # noqa: BLE001 - never let a probe end the run + warnings.warn( + f"RLM extractable-work probe failed, treating as no work: {exc}", + RuntimeWarning, + stacklevel=2, + ) + return False + + def _is_deadline_error(error: RLMError | None) -> bool: """Whether a root failure is wall-clock deadline exhaustion. @@ -1357,7 +1385,10 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr not answer.get("ready") and terminal_handoff and trajectory - and _has_extractable_work(answer, has_successful_step) + and ( + _has_extractable_work(answer, has_successful_step) + or _host_reports_extractable_work(cfg) + ) ): context.emit_progress("Loop ended unconfirmed: extracting best final answer...") context.emit_event(extract_event(iterations, "start")) diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index e457879..10e883f 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -84,6 +84,14 @@ # ferry their own answer-critical state (whatever that is) alongside the draft. CheckpointPayloadProvider: TypeAlias = Callable[[], Any] +# Answers "is there work here worth one terminal extract call?" for state the +# engine cannot see. The engine's own test — a retained draft, or a step that +# executed successfully — misses the case where generated code retrieved real +# data through a host accessor and then raised before printing any of it. The +# engine has no signal there (no draft, no successful step, no stdout), but the +# host does, and stays the only thing that knows what its own data layer did. +ExtractableWorkProbe: TypeAlias = Callable[[], bool] + class ReadyMetadataValidatorError(RuntimeError): """Trusted host validator failed instead of returning repairable violations.""" @@ -118,6 +126,7 @@ class RLMConfig: # raises is reported and the checkpoint carries a null payload: a # checkpoint can never fail a run. checkpoint_payload_provider: CheckpointPayloadProvider | None = None + extractable_work_probe: ExtractableWorkProbe | None = None @dataclass diff --git a/tests/test_wall_clock_fallback.py b/tests/test_wall_clock_fallback.py index 6df8a79..e6a20f2 100644 --- a/tests/test_wall_clock_fallback.py +++ b/tests/test_wall_clock_fallback.py @@ -11,6 +11,8 @@ import time +import pytest + from droste import Budget, RLMConfig, run_rlm from droste.exceptions import RLMError from droste.execution import create_execution_context @@ -174,3 +176,70 @@ def test_recovered_budget_exhaustion_is_always_the_wall_clock(): assert result.recovered_error is not None assert result.recovered_error.type == "BudgetExhausted" assert result.recovered_error.details["resource"] == "wall_ms" + + +# --- The host knows about work the engine cannot see --------------------- +# +# Generated code that retrieves real data through a host accessor and then +# raises before printing leaves the engine with nothing to test: no draft, no +# successful step, no stdout. It concluded there was nothing to extract while +# the host was holding everything the answer needed. Observed on a real run +# that retrieved 100 message GUIDs and then returned a bare budget error. + + +def _raising_code() -> str: + return '```python\nrows = query("SELECT 1")\nraise ValueError("boom")\n```' + + +def _run_with_probe(probe): + # Uniform responses: the loop's exact consumption (initial attempt, missing + # code repair, execution repair) is not what is under test here, and pinning + # a response count would make this fail for reasons unrelated to the probe. + responses = [MockResponse(text=_raising_code(), usage=_usage()) for _ in range(12)] + return run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(max_iterations=2), extractable_work_probe=probe), + ) + + +def test_engine_alone_finds_nothing_extractable_in_failed_steps() -> None: + """Baseline: the engine's own test is unchanged, so a trajectory of pure + errors with no draft and no stdout still yields no extraction.""" + result = _run_with_probe(None) + + assert result.extracted is False + assert result.error is not None + + +def test_host_probe_unlocks_extraction_the_engine_would_have_skipped() -> None: + """The host says its data layer retrieved something, so the one terminal + extract call runs and the run returns an answer instead of a bare error. + + This is the case seen on a real 314k-message corpus: generated code called + an accessor, recorded 100 message GUIDs, then raised before printing any of + them — leaving the engine no draft, no successful step, and no stdout.""" + result = _run_with_probe(lambda: True) + + assert result.extracted is True + assert result.error is None + + +def test_a_probe_saying_no_changes_nothing() -> None: + result = _run_with_probe(lambda: False) + + assert result.extracted is False + + +def test_a_raising_probe_cannot_fail_the_run_it_is_recovering() -> None: + def explode() -> bool: + raise RuntimeError("host ledger unavailable") + + with pytest.warns(RuntimeWarning, match="extractable-work probe failed"): + result = _run_with_probe(explode) + + # Treated as "no work" rather than propagating: a terminal recovery path + # must never be able to end the run it exists to rescue. + assert result.extracted is False From 0924f0579791f1da6679972d14c047f191682f9d Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Mon, 3 Aug 2026 15:01:09 -0400 Subject: [PATCH 5/5] Let the host show the extract pass what it retrieved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlocking the terminal extract call was necessary but not sufficient. The extract prompt is built from the draft plus per-iteration code and stdout — everything the engine can see — but a REPL only surfaces what generated code chose to print. Code that fetched rows and then raised prints nothing, so the extract pass was handed failing code and nothing else and answered InsufficientEvidence while the rows sat in the host's ledger. Measured on a real 314k-message corpus: four ledger records and 84 message GUIDs retrieved, extraction attempted, extraction declined. `RLMConfig.extract_context_provider` returns pre-rendered text the engine bounds and concatenates without parsing — the same opacity contract as the checkpoint payload. Bounded at 8k chars because the extract call is deliberately uncached (`cache_anchors=None`), so every character is paid at full prefill and an unbounded host could spend the recovery call's whole input budget describing what it found. A provider that raises contributes nothing rather than ending the run it is recovering. Deliberately narrow: observations are supplied only when the HOST is what made extraction possible. When the engine can already see the work — a retained draft, or a step that ran to completion — the prompt stays byte-identical to what it was before this hook existed. Runs that extract successfully today keep their answers, their spend, and their benchmark scores; the behavior change is confined to runs that produce nothing. This is ReAct's shape rather than a new invention: dspy.ReAct puts every tool's return value straight into the trajectory, so extraction always sees the data. dspy.RLM, being REPL-based, has exactly the gap fixed here — its `variables_info` is built once from the inputs and never reflects what execution retrieved. Co-Authored-By: Claude Opus 5 (1M context) --- src/droste/__init__.py | 2 + src/droste/loop/__init__.py | 2 + src/droste/loop/rlm.py | 55 ++++++++++++++-- src/droste/loop/step.py | 10 +++ tests/test_wall_clock_fallback.py | 103 +++++++++++++++++++++++++++++- 5 files changed, 167 insertions(+), 5 deletions(-) diff --git a/src/droste/__init__.py b/src/droste/__init__.py index 83141a1..0bf68aa 100644 --- a/src/droste/__init__.py +++ b/src/droste/__init__.py @@ -108,6 +108,7 @@ from .loop.step import ( CheckpointPayloadProvider, ExtractableWorkProbe, + ExtractContextProvider, ReadyMetadataValidator, ReadyMetadataValidatorError, ) @@ -170,6 +171,7 @@ "RLM_PREFLIGHT_SCHEMA_VERSION", "CheckpointPayloadProvider", "ExtractableWorkProbe", + "ExtractContextProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", "extract_code_block", diff --git a/src/droste/loop/__init__.py b/src/droste/loop/__init__.py index 50cf0ea..6f6c9ed 100644 --- a/src/droste/loop/__init__.py +++ b/src/droste/loop/__init__.py @@ -3,6 +3,7 @@ from .step import ( CheckpointPayloadProvider, ExtractableWorkProbe, + ExtractContextProvider, ReadyMetadataValidator, ReadyMetadataValidatorError, ) @@ -18,6 +19,7 @@ "IterationRecord", "CheckpointPayloadProvider", "ExtractableWorkProbe", + "ExtractContextProvider", "ReadyMetadataValidator", "ReadyMetadataValidatorError", ] diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index 2e73260..45df043 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -106,6 +106,10 @@ _EXTRACT_CODE_CHARS = 1000 _EXTRACT_OUTPUT_CHARS = 1500 _EXTRACT_SUMMARY_CHARS = 60000 +# Host observations ride an uncached extract call, so this is deliberately far +# tighter than the trajectory budget above: enough rows to answer from, not the +# whole ledger. +_EXTRACT_HOST_CONTEXT_CHARS = 8000 # Sentinel used in extract summaries for iterations that printed nothing; the # conversational nudge shown to the in-loop model must not read as real output. @@ -568,6 +572,36 @@ def _has_extractable_work(answer: dict[str, Any], has_successful_step: bool) -> return has_successful_step +def _host_extract_context(cfg: "RLMConfig") -> str: + """Host-rendered observations to append to the terminal extract prompt. + + Bounded here rather than trusting the host: this rides on an extract call + that is deliberately uncached (`cache_anchors=None`), so every character is + paid at full prefill, and an unbounded host could spend the recovery call's + whole input budget describing what it found. + + A provider that raises contributes nothing, for the same reason the probe + does: a terminal recovery path must not be able to fail the run it is + recovering. + """ + + provider = cfg.extract_context_provider + if provider is None: + return "" + try: + rendered = provider() + except Exception as exc: # noqa: BLE001 - never let a provider end the run + warnings.warn( + f"RLM extract-context provider failed, continuing without it: {exc}", + RuntimeWarning, + stacklevel=2, + ) + return "" + if not isinstance(rendered, str) or not rendered.strip(): + return "" + return rendered[:_EXTRACT_HOST_CONTEXT_CHARS] + + def _host_reports_extractable_work(cfg: "RLMConfig") -> bool: """Whether the host says its own state holds work worth extracting. @@ -648,6 +682,7 @@ def _extract_final_answer( cfg: "RLMConfig", context: ExecutionContext, prompt_pack: PromptPack, + host_context: str = "", ) -> tuple[str, RLMError | None]: """One extract pass: when a terminal budget handoff occurs without answer['ready'], make one @@ -661,8 +696,13 @@ def _extract_final_answer( this" apart from "extraction failed, this is raw debug output," instead of both cases looking identical.""" try: + # Host observations sit between the trajectory and the sentinel, and are + # labelled as retrieved data rather than as something the code printed — + # they exist precisely because the code did not print them. + observations = f"\n\nData retrieved during the run:\n{host_context}" if host_context else "" history = ( _trajectory_summary(draft, trajectory) + + observations + f"\n\nUnable sentinel: {prompt_pack.unable_sentinel}" ) slots = PromptSlots( @@ -1381,14 +1421,20 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr was_extracted = False extract_error: RLMError | None = None recovered_error: RLMError | None = None + # Host observations are supplied ONLY when the host is what made this + # extraction possible. When the engine can already see the work — a + # retained draft, or a step that ran to completion — the prompt stays + # byte-identical to what it was before this hook existed, so runs that + # already extract successfully keep their answers, their spend, and + # their benchmark scores. The behavior change is confined to runs that + # produce nothing today. + engine_sees_work = _has_extractable_work(answer, has_successful_step) + host_sees_work = not engine_sees_work and _host_reports_extractable_work(cfg) if ( not answer.get("ready") and terminal_handoff and trajectory - and ( - _has_extractable_work(answer, has_successful_step) - or _host_reports_extractable_work(cfg) - ) + and (engine_sees_work or host_sees_work) ): context.emit_progress("Loop ended unconfirmed: extracting best final answer...") context.emit_event(extract_event(iterations, "start")) @@ -1401,6 +1447,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr cfg, context, resolved_prompt_pack.pack, + host_context=_host_extract_context(cfg) if host_sees_work else "", ) if extracted: context.emit_event(extract_event(iterations, "completion")) diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index 10e883f..e24d067 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -92,6 +92,15 @@ # host does, and stays the only thing that knows what its own data layer did. ExtractableWorkProbe: TypeAlias = Callable[[], bool] +# Renders host-held observations for the terminal extract prompt. The engine +# builds that prompt from the draft and per-iteration code/stdout, which is +# everything it can see — but a REPL only surfaces what generated code chose to +# print, so a run that fetched rows and then raised leaves the extract pass +# looking at failing code and nothing else. This lets the host contribute what +# its accessors actually returned. Pre-rendered text: the engine bounds it and +# concatenates it, and never parses or inspects it. +ExtractContextProvider: TypeAlias = Callable[[], str] + class ReadyMetadataValidatorError(RuntimeError): """Trusted host validator failed instead of returning repairable violations.""" @@ -127,6 +136,7 @@ class RLMConfig: # checkpoint can never fail a run. checkpoint_payload_provider: CheckpointPayloadProvider | None = None extractable_work_probe: ExtractableWorkProbe | None = None + extract_context_provider: ExtractContextProvider | None = None @dataclass diff --git a/tests/test_wall_clock_fallback.py b/tests/test_wall_clock_fallback.py index e6a20f2..bfa7c47 100644 --- a/tests/test_wall_clock_fallback.py +++ b/tests/test_wall_clock_fallback.py @@ -16,7 +16,11 @@ from droste import Budget, RLMConfig, run_rlm from droste.exceptions import RLMError from droste.execution import create_execution_context -from droste.loop.rlm import _extract_final_answer, _is_deadline_error +from droste.loop.rlm import ( + _EXTRACT_HOST_CONTEXT_CHARS, + _extract_final_answer, + _is_deadline_error, +) from droste.loop.trajectory import EXECUTION_STATUS_SUCCESS, IterationRecord from droste.prompts import load_builtin_prompt_catalog, resolve_prompt_pack from droste.protocols.llm_client import TokenUsage @@ -243,3 +247,100 @@ def explode() -> bool: # Treated as "no work" rather than propagating: a terminal recovery path # must never be able to end the run it exists to rescue. assert result.extracted is False + + +# --- What the extract pass is allowed to see ----------------------------- +# +# Unlocking the extract call was necessary but not sufficient: the prompt is +# built from draft + code + stdout, so a run whose data exists only in the host +# ledger handed the extract pass failing code and nothing else, and it returned +# InsufficientEvidence. The host can now contribute what its accessors actually +# returned. + + +def _captured_extract_prompt(**config_kwargs): + """Run to a terminal handoff and return the extract call's user prompt.""" + seen: list[str] = [] + + class CapturingClient(MockLLMClient): + def responses_create(self, messages, *args, **kwargs): # type: ignore[override] + seen.append(str(messages[-1].get("content", ""))) + return super().responses_create(messages, *args, **kwargs) + + responses = [MockResponse(text=_raising_code(), usage=_usage()) for _ in range(12)] + run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=CapturingClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig(budget=Budget(max_iterations=2), **config_kwargs), + ) + return seen[-1] if seen else "" + + +def test_host_observations_reach_the_extract_prompt() -> None: + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=lambda: ( + "[evidence-1] query(...) -> 84 rows\nSarah | dinner friday" + ), + ) + + assert "Data retrieved during the run" in prompt + assert "Sarah | dinner friday" in prompt + + +def test_observations_are_bounded_so_recovery_cannot_eat_its_own_budget() -> None: + """The extract call is deliberately uncached, so every character is paid at + full prefill and an unbounded host could spend the whole recovery call.""" + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=lambda: "x" * 50_000, + ) + + assert prompt.count("x") == _EXTRACT_HOST_CONTEXT_CHARS + + +def test_a_raising_context_provider_still_lets_extraction_run() -> None: + def explode() -> str: + raise RuntimeError("ledger unavailable") + + with pytest.warns(RuntimeWarning, match="extract-context provider failed"): + prompt = _captured_extract_prompt( + extractable_work_probe=lambda: True, + extract_context_provider=explode, + ) + + # Degraded, not fatal: the extract call still happened, just without them. + assert prompt + assert "Data retrieved during the run" not in prompt + + +def test_a_run_the_engine_could_already_extract_is_left_untouched() -> None: + """The narrow gate. A run whose draft or successful step the engine can see + gets the prompt it got before this hook existed — same answer, same spend, + same benchmark score. Only runs that produce nothing today change.""" + provider_calls: list[int] = [] + + responses = [ + MockResponse( + text="```python\nanswer['content'] = 'partial finding'\n```", + usage=_usage(), + ), + MockResponse(text="Best-effort synthesis.", usage=_usage()), + ] + result = run_rlm( + question="test", + environment=MockEnvironment(), + root_llm=MockLLMClient(responses=responses), + subcalls=MockSubcallClient(), + config=RLMConfig( + budget=Budget(max_iterations=1), + extractable_work_probe=lambda: True, + extract_context_provider=lambda: provider_calls.append(1) or "should not appear", + ), + ) + + assert result.extracted is True + # The engine saw the draft, so the host was never consulted at all. + assert provider_calls == []