diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae2b943..5a869cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,23 +154,25 @@ jobs: import sys from pathlib import Path - from droste.testing import ( - runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, - ) + from importlib.resources import files + + from droste.testing import conformance_fixture_names + # Enumerated, never named: an ABI rename must not have to be found + # here. Naming the helpers is what broke the v9 and v10 migrations. source = Path(sys.argv[1]) - assert trace_v9_execution_ndjson() == (source / "trace-v9-execution.ndjson").read_bytes() - assert trace_v9_lifecycle_ndjson() == (source / "trace-v9-lifecycle.ndjson").read_bytes() - assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() + names = conformance_fixture_names() + assert names, "conformance corpus is empty" + for name in names: + packaged = files("droste.testing").joinpath("fixtures", name).read_bytes() + assert packaged == (source / name).read_bytes(), name PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v9-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v9-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v9-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v9-execution.ndjson" + for f in $(cd src/droste/testing/fixtures && ls *.ndjson | sort); do + cmp "src/droste/testing/fixtures/$f" \ + "$tmp/$sdist_root/src/droste/testing/fixtures/$f" + done cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" @@ -368,23 +370,25 @@ jobs: import sys from pathlib import Path - from droste.testing import ( - runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, - ) + from importlib.resources import files + + from droste.testing import conformance_fixture_names + # Enumerated, never named: an ABI rename must not have to be found + # here. Naming the helpers is what broke the v9 and v10 migrations. source = Path(sys.argv[1]) - assert trace_v9_execution_ndjson() == (source / "trace-v9-execution.ndjson").read_bytes() - assert trace_v9_lifecycle_ndjson() == (source / "trace-v9-lifecycle.ndjson").read_bytes() - assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes() + names = conformance_fixture_names() + assert names, "conformance corpus is empty" + for name in names: + packaged = files("droste.testing").joinpath("fixtures", name).read_bytes() + assert packaged == (source / name).read_bytes(), name PY sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')" tar xzf dist/droste-*.tar.gz -C "$tmp" - cmp src/droste/testing/fixtures/trace-v9-lifecycle.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v9-lifecycle.ndjson" - cmp src/droste/testing/fixtures/trace-v9-execution.ndjson \ - "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v9-execution.ndjson" + for f in $(cd src/droste/testing/fixtures && ls *.ndjson | sort); do + cmp "src/droste/testing/fixtures/$f" \ + "$tmp/$sdist_root/src/droste/testing/fixtures/$f" + done cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \ "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson" diff --git a/UPGRADING.md b/UPGRADING.md index 7417016..a58749d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -11,6 +11,36 @@ Ordered newest first. "Embedder" means anything that builds on the engine beyond the `droste` CLI: hosts calling `run_rlm` in-process, `droste_runner` consumers, and Pyodide-substrate integrations staging the Deno relay. +## 0.25.0 + +### Trace ABI v10: a run reports what it did without + +Every result now carries `degradations`, a list that is empty on a clean run +and otherwise names each thing the run continued without: `site`, +`error_type`, `detail`, and — the field that matters to a reader — +`consequence`, which records not that a callback raised but what the run then +did without it. + +Several boundaries here deliberately refuse to let a host-supplied callback end +a run: a broken logging sink or observer must not destroy a user's work. That +resilience is correct. It was paid for with a `warnings.warn` that reaches no +consumer of the result, so a run that lost a budget event, ran its terminal +extract without host context, or discarded salvageable work returned something +byte-identical to a clean run. The loss was real and unobservable. + +Recorded today at `budget_event_sink`, `extract_context_provider`, and +`extractable_work_probe`. `BudgetLedger.dropped_events()` exposes the same +facts for a directly-held ledger. + +**Hosts should treat a non-empty `degradations` as a degraded answer** — worth +surfacing, logging, or refusing, depending on what the run lost. Nothing new +can end a run; this only makes the existing fallbacks visible. + +The v9 -> v10 rename moves the `trace_v9_*` helpers and `trace-v9-*` fixtures +to `trace_v10_*` / `trace-v10-*`. Response builders in `droste_runner` emit +`degradations: []` on every shape, so a consumer reads one field rather than +treating a missing key as "nothing was lost". + ## 0.24.0 ### Trace ABI v9 reports which ready-time gates a run armed diff --git a/docs/trace-abi.md b/docs/trace-abi.md index 0f1f8d1..45e0e94 100644 --- a/docs/trace-abi.md +++ b/docs/trace-abi.md @@ -230,12 +230,12 @@ and sdist. Python consumers load them through package resources: ```python from droste.testing import ( runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, + trace_v10_execution_ndjson, + trace_v10_lifecycle_ndjson, ) -execution_lines = trace_v9_execution_ndjson().splitlines() -event_lines = trace_v9_lifecycle_ndjson().splitlines() +execution_lines = trace_v10_execution_ndjson().splitlines() +event_lines = trace_v10_lifecycle_ndjson().splitlines() pre_admission_refusal = runner_v10_refusal_ndjson() ``` diff --git a/examples/pyodide-host/e2e_test.ts b/examples/pyodide-host/e2e_test.ts index c8c0843..9b83346 100644 --- a/examples/pyodide-host/e2e_test.ts +++ b/examples/pyodide-host/e2e_test.ts @@ -33,7 +33,7 @@ const RUNNER_REFUSAL_FIXTURE = new URL( import.meta.url, ); const TRACE_LIFECYCLE_FIXTURE = new URL( - "../../src/droste/testing/fixtures/trace-v9-lifecycle.ndjson", + "../../src/droste/testing/fixtures/trace-v10-lifecycle.ndjson", import.meta.url, ); const TEST_BUDGET = { diff --git a/pyodide/event_channel_probe.ts b/pyodide/event_channel_probe.ts index e820574..42a4079 100644 --- a/pyodide/event_channel_probe.ts +++ b/pyodide/event_channel_probe.ts @@ -4,7 +4,7 @@ const mode = Deno.args[0]; const channel = eventChannelFromEnvironment(); const fixture = await Deno.readTextFile( new URL( - "../src/droste/testing/fixtures/trace-v9-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v10-lifecycle.ndjson", import.meta.url, ), ); diff --git a/pyodide/event_channel_test.ts b/pyodide/event_channel_test.ts index 769911d..37fc371 100644 --- a/pyodide/event_channel_test.ts +++ b/pyodide/event_channel_test.ts @@ -8,7 +8,7 @@ import { import { isRlmEvent } from "../src/droste/substrates/_relay/events.ts"; const TRACE_LIFECYCLE_FIXTURE = new URL( - "../src/droste/testing/fixtures/trace-v9-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v10-lifecycle.ndjson", import.meta.url, ); diff --git a/pyodide/events_test.ts b/pyodide/events_test.ts index aa1d328..cad8827 100644 --- a/pyodide/events_test.ts +++ b/pyodide/events_test.ts @@ -120,6 +120,7 @@ const BODIES: Record> = { status: "success", ready: true, extracted: false, + degradations: [], iterations: 1, usage: { kind: "resolved", @@ -170,7 +171,7 @@ function wire( run_id: "run-1", seq: 1, timestamp: "2026-07-14T00:00:00Z", - version: 9, + version: 10, persistence_class: persistence ?? PERSISTENCE_BY_TYPE[type], depth: 0, ...body, @@ -384,7 +385,7 @@ Deno.test("successful output beginning ERROR remains an output event", () => { Deno.test("Python and relay accept the same execution golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v9-execution.ndjson", + "../src/droste/testing/fixtures/trace-v10-execution.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); @@ -428,7 +429,7 @@ Deno.test("Python and relay accept the same execution golden NDJSON", async () = Deno.test("Python and relay accept the same lifecycle golden NDJSON", async () => { const fixture = new URL( - "../src/droste/testing/fixtures/trace-v9-lifecycle.ndjson", + "../src/droste/testing/fixtures/trace-v10-lifecycle.ndjson", import.meta.url, ); const lines = (await Deno.readTextFile(fixture)).trim().split("\n"); diff --git a/pyodide/heartbeat_test.ts b/pyodide/heartbeat_test.ts index 056f9d4..f82a0a6 100644 --- a/pyodide/heartbeat_test.ts +++ b/pyodide/heartbeat_test.ts @@ -22,7 +22,7 @@ const { isRlmEvent, PERSISTENCE_BY_TYPE, RLM_EVENT_TYPES } = await import( function wire(body: Record): string { return JSON.stringify({ type: "heartbeat", - version: 9, + version: 10, run_id: "run-1", seq: 4, timestamp: "2026-08-03T00:00:00Z", @@ -97,7 +97,7 @@ Deno.test("a heartbeat a live subcall produced is forwarded", () => { depth: 1, seq: 2, timestamp: "2026-08-03T21:46:54.805Z", - version: 9, + version: 10, persistence_class: "transient", }; diff --git a/pyproject.toml b/pyproject.toml index 968de70..9598003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "droste" -version = "0.24.0" +version = "0.25.0" description = "Recursive analysis engine for data too large for a context window, built with Recursive Language Models (RLMs)" readme = "README.md" requires-python = ">=3.11" diff --git a/src/droste/execution/budget.py b/src/droste/execution/budget.py index 3a2d4cf..f4acc15 100644 --- a/src/droste/execution/budget.py +++ b/src/droste/execution/budget.py @@ -200,6 +200,10 @@ class BudgetLedger: _closed: bool = field(default=False, init=False, repr=False) _event_journal: list[dict[str, Any]] = field(default_factory=list, init=False, repr=False) _emitted_events: int = field(default=0, init=False, repr=False) + # Events this ledger produced but could not deliver. A failing sink must + # not end the run, but the loss must not vanish either: without this the + # stream is short by one event and nothing anywhere says so. + _dropped_events: list[dict[str, str]] = field(default_factory=list, init=False, repr=False) _emit_lock: RLock = field(default_factory=RLock, init=False, repr=False) def __post_init__(self) -> None: @@ -538,6 +542,12 @@ def _queue_event_locked(self, action: str, resource: str, amount: int, call_id: } ) + def dropped_events(self) -> tuple[dict[str, str], ...]: + """Budget events this ledger produced but failed to deliver.""" + + with self._lock: + return tuple(dict(item) for item in self._dropped_events) + def _drain_events(self) -> None: """Emit the ledger journal in mutation order without holding its lock.""" @@ -560,6 +570,14 @@ def _drain_events(self) -> None: RuntimeWarning, stacklevel=2, ) + with self._lock: + self._dropped_events.append( + { + "error_type": type(exc).__name__, + "detail": str(exc), + "event": str(event.get("event", "budget")), + } + ) def _plain_json(value: Any) -> Any: diff --git a/src/droste/execution/context.py b/src/droste/execution/context.py index 6f945f2..5b14b77 100644 --- a/src/droste/execution/context.py +++ b/src/droste/execution/context.py @@ -24,6 +24,35 @@ ) +@dataclass(frozen=True) +class RunDegradation: + """One thing the run silently did without, recorded so it cannot stay silent. + + Several boundaries here deliberately refuse to let a host-supplied callback + end a run -- a broken logging sink or observer must not destroy a user's + work. That resilience is correct, but it used to be paid for with a + ``warnings.warn`` that reaches no consumer of the result, so a run that + lost an event, ran an extract pass without its context, or discarded + salvageable work returned something indistinguishable from a clean run. + + ``consequence`` is the part that matters to a reader: not that a callback + raised, but what the run then did without. + """ + + site: str + error_type: str + detail: str + consequence: str + + def as_dict(self) -> dict[str, str]: + return { + "site": self.site, + "error_type": self.error_type, + "detail": self.detail, + "consequence": self.consequence, + } + + @dataclass class ExecutionContext: """Context for tracking recursive LLM calls within sandbox execution.""" @@ -34,6 +63,7 @@ class ExecutionContext: ledger: BudgetLedger = field(default_factory=lambda: BudgetLedger(Budget())) _emission_lock: RLock = field(default_factory=RLock, init=False, repr=False) _iteration: int = field(default=0, init=False, repr=False) + _degradations: list[RunDegradation] = field(default_factory=list, init=False, repr=False) def __post_init__(self) -> None: if self.ledger.budget != self.config.budget: @@ -50,6 +80,30 @@ def emit_progress(self, status: str) -> None: self.config.on_progress(status) self.emit_event(progress_event(status)) + def record_degradation(self, *, site: str, error: BaseException, consequence: str) -> None: + """Record that the run continued without something it should have had. + + Callers keep whatever fallback they already had; this only ensures the + fallback is visible in the run's own output instead of a warning the + host never sees. + """ + + with self._emission_lock: + self._degradations.append( + RunDegradation( + site=site, + error_type=type(error).__name__, + detail=str(error), + consequence=consequence, + ) + ) + + def degradations(self) -> tuple[RunDegradation, ...]: + """Everything this run did without, in the order it happened.""" + + with self._emission_lock: + return tuple(self._degradations) + def emit_event(self, event: dict[str, Any]) -> None: """Deliver a structured loop event (#1) to the attached sink. diff --git a/src/droste/execution/report.py b/src/droste/execution/report.py index c2cede6..a472c0d 100644 --- a/src/droste/execution/report.py +++ b/src/droste/execution/report.py @@ -53,6 +53,9 @@ def project_result( "subcalls": result.sub_calls_made, "successful_subcalls": int(getattr(result, "sub_calls_succeeded", 0)), "extracted": bool(getattr(result, "extracted", False)), + # Always present, empty on a clean run: a consumer must never have to + # read silence as "nothing was lost". + "degradations": [dict(item) for item in getattr(result, "degradations", ())], "error": error_payload(result.error, include_details=include_error_details), "extract_error": error_payload( getattr(result, "extract_error", None), include_details=include_error_details diff --git a/src/droste/execution/trace.py b/src/droste/execution/trace.py index 185fca9..53f42ae 100644 --- a/src/droste/execution/trace.py +++ b/src/droste/execution/trace.py @@ -17,7 +17,7 @@ from typing import Any, Callable, Mapping from uuid import uuid4 -TRACE_ABI_VERSION = 9 +TRACE_ABI_VERSION = 10 class PersistenceClass(str, Enum): @@ -177,6 +177,9 @@ class PersistenceClass(str, Enum): "status": str, "ready": bool, "extracted": bool, + # Same field as the result event: a host that reads only the + # terminal event must still learn what the run did without. + "degradations": list, "iterations": int, "usage": Mapping, "budget": Mapping, @@ -464,6 +467,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: "subcalls": int, "successful_subcalls": int, "extracted": bool, + # Always present, empty on a clean run: a consumer must never + # have to read silence as "nothing was lost". + "degradations": list, "error": (Mapping, _NONE_TYPE), "extract_error": (Mapping, _NONE_TYPE), "recovered_error": (Mapping, _NONE_TYPE), diff --git a/src/droste/loop/rlm.py b/src/droste/loop/rlm.py index b8d08af..8db5530 100644 --- a/src/droste/loop/rlm.py +++ b/src/droste/loop/rlm.py @@ -573,7 +573,7 @@ def _has_extractable_work(answer: dict[str, Any], has_successful_step: bool) -> return has_successful_step -def _host_extract_context(cfg: "RLMConfig") -> str: +def _host_extract_context(cfg: "RLMConfig", context: ExecutionContext) -> str: """Host-rendered observations to append to the terminal extract prompt. Bounded here rather than trusting the host: this rides on an extract call @@ -597,13 +597,18 @@ def _host_extract_context(cfg: "RLMConfig") -> str: RuntimeWarning, stacklevel=2, ) + context.record_degradation( + site="extract_context_provider", + error=exc, + consequence="terminal extract ran without host-held observations", + ) return "" if not isinstance(rendered, str) or not rendered.strip(): return "" return rendered[:_EXTRACT_HOST_CONTEXT_CHARS] -def _host_reports_extractable_work(cfg: "RLMConfig") -> bool: +def _host_reports_extractable_work(cfg: "RLMConfig", context: ExecutionContext) -> bool: """Whether the host says its own state holds work worth extracting. The engine's test above sees only what the engine owns. Generated code that @@ -628,6 +633,11 @@ def _host_reports_extractable_work(cfg: "RLMConfig") -> bool: RuntimeWarning, stacklevel=2, ) + context.record_degradation( + site="extractable_work_probe", + error=exc, + consequence="run treated as having no salvageable work; extract skipped", + ) return False @@ -1449,7 +1459,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr # their benchmark scores. The behavior change is confined to runs that # produce nothing today. engine_sees_work = _has_extractable_work(answer, has_successful_step) - host_sees_work = not engine_sees_work and _host_reports_extractable_work(cfg) + host_sees_work = not engine_sees_work and _host_reports_extractable_work(cfg, context) if ( not answer.get("ready") and terminal_handoff @@ -1467,7 +1477,7 @@ def call_live_root(live_messages: list[dict[str, str]]) -> tuple[str, Any, RLMEr cfg, context, resolved_prompt_pack.pack, - host_context=_host_extract_context(cfg) if host_sees_work else "", + host_context=_host_extract_context(cfg, context) if host_sees_work else "", ) if extracted: context.emit_event(extract_event(iterations, "completion")) diff --git a/src/droste/loop/step.py b/src/droste/loop/step.py index c776f45..7c74041 100644 --- a/src/droste/loop/step.py +++ b/src/droste/loop/step.py @@ -176,6 +176,11 @@ class RLMResult: tokens_used: int sub_calls_made: int trajectory: list[IterationRecord] + # Everything the run continued without. Empty on a clean run; non-empty + # means the answer was produced by a run that lost an event, skipped a + # salvage, or ran a pass without host context. A host that cares can + # refuse it; every host can at least see it. + degradations: tuple[dict[str, str], ...] = () error: RLMError | None = None # True when the answer came from the post-exhaustion extract pass rather # than the model marking answer['ready'] — hosts may surface it as a @@ -887,6 +892,18 @@ def finalize( tokens_used=context.stats.total_tokens, sub_calls_made=context.stats.calls_made, trajectory=trajectory, + degradations=( + *(item.as_dict() for item in context.degradations()), + *( + { + "site": "budget_event_sink", + "error_type": drop["error_type"], + "detail": drop["detail"], + "consequence": "a budget event was produced but never delivered", + } + for drop in context.ledger.dropped_events() + ), + ), sub_calls_succeeded=context.stats.successful_calls, error=error, extracted=extracted, @@ -973,6 +990,7 @@ def terminal_error(value: RLMError | None) -> dict[str, Any] | None: scaffold_manifest.schema_version if scaffold_manifest is not None else None ), "stdout_chars": result.stdout_chars, + "degradations": [dict(item) for item in result.degradations], } result.run_record = context.finish_trace(terminal) return result diff --git a/src/droste/substrates/_relay/events.ts b/src/droste/substrates/_relay/events.ts index 6f9f0a9..eb8b7e9 100644 --- a/src/droste/substrates/_relay/events.ts +++ b/src/droste/substrates/_relay/events.ts @@ -388,6 +388,7 @@ function validBody(type: string, body: Record): boolean { "status", "ready", "extracted", + "degradations", "iterations", "usage", "budget", @@ -401,6 +402,7 @@ function validBody(type: string, body: Record): boolean { ) && ["success", "error", "cancelled"].includes(String(body.status)) && typeof body.ready === "boolean" && typeof body.extracted === "boolean" && + Array.isArray(body.degradations) && integerField("iterations") && Number(body.iterations) >= 0 && isObject(body.usage) && validBody("usage", body.usage) && isObject(body.budget) && @@ -439,7 +441,7 @@ export function isRlmEvent(line: string): boolean { Number.isInteger(o.seq) && o.seq > 0 && typeof o.timestamp === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(o.timestamp) && - o.version === 9 && + o.version === 10 && o.persistence_class === PERSISTENCE_BY_TYPE[o.type] && Number.isInteger(o.depth) && o.depth >= 0 && (o.depth === 0 diff --git a/src/droste/substrates/_relay/relay.ts b/src/droste/substrates/_relay/relay.ts index 69e9d9e..6fc8198 100644 --- a/src/droste/substrates/_relay/relay.ts +++ b/src/droste/substrates/_relay/relay.ts @@ -455,7 +455,7 @@ function writeRelayEvent(obj: Record): void { depth: 1, seq: ++relaySeq, timestamp: new Date().toISOString(), - version: 9, + version: 10, persistence_class: "transient", }; eventChannel.writeFrame(JSON.stringify(event)); diff --git a/src/droste/testing/__init__.py b/src/droste/testing/__init__.py index 8b2c195..b845644 100644 --- a/src/droste/testing/__init__.py +++ b/src/droste/testing/__init__.py @@ -38,16 +38,16 @@ def conformance_fixture_names() -> tuple[str, ...]: ) -def trace_v9_lifecycle_ndjson() -> bytes: +def trace_v10_lifecycle_ndjson() -> bytes: """Return the shared Trace ABI v7 lifecycle conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v9-lifecycle.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v10-lifecycle.ndjson").read_bytes() -def trace_v9_execution_ndjson() -> bytes: +def trace_v10_execution_ndjson() -> bytes: """Return the shared Trace ABI v7 response/code/output/error conformance corpus.""" - return files(__package__).joinpath("fixtures/trace-v9-execution.ndjson").read_bytes() + return files(__package__).joinpath("fixtures/trace-v10-execution.ndjson").read_bytes() def runner_v10_refusal_ndjson() -> bytes: @@ -73,6 +73,6 @@ def runner_v10_refusal_ndjson() -> bytes: "require_unknown_completion", "run_while_blocked", "conformance_fixture_names", - "trace_v9_execution_ndjson", - "trace_v9_lifecycle_ndjson", + "trace_v10_execution_ndjson", + "trace_v10_lifecycle_ndjson", ] diff --git a/src/droste/testing/_trace_fixtures.py b/src/droste/testing/_trace_fixtures.py index 8fb2137..3536e39 100644 --- a/src/droste/testing/_trace_fixtures.py +++ b/src/droste/testing/_trace_fixtures.py @@ -22,7 +22,7 @@ def _ndjson(events: tuple[RunEvent, ...]) -> bytes: ) -def build_trace_v9_execution_ndjson() -> bytes: +def build_trace_v10_execution_ndjson() -> bytes: """Build the deterministic code/output/error projection corpus.""" started_at = datetime(2026, 7, 16, tzinfo=timezone.utc) diff --git a/src/droste/testing/fixtures/runner-v10-refusal.ndjson b/src/droste/testing/fixtures/runner-v10-refusal.ndjson index 7cae732..4b0a7b5 100644 --- a/src/droste/testing/fixtures/runner-v10-refusal.ndjson +++ b/src/droste/testing/fixtures/runner-v10-refusal.ndjson @@ -1 +1 @@ -{"answer":"","answer_metadata":{},"ready":false,"iterations":0,"tokens_used":0,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"protocol_version_missing","message":"request has no protocol_version; this engine speaks 10 — add \"protocol_version\": 10 to the request","code":"protocol_version_missing","details":{"requested":null,"supported":10}},"extract_error":null,"recovered_error":null,"prompt_pack":null,"scaffold_manifest":null,"stdout_chars":0,"run_record":null,"run_id":null,"status":"refusal","operation":null,"protocol_version":10,"provider":"","response_id":"","stop_reason":"","model":"","data_source_requests":null} +{"answer":"","answer_metadata":{},"ready":false,"iterations":0,"tokens_used":0,"subcalls":0,"successful_subcalls":0,"extracted":false,"degradations":[],"error":{"type":"protocol_version_missing","message":"request has no protocol_version; this engine speaks 10 — add \"protocol_version\": 10 to the request","code":"protocol_version_missing","details":{"requested":null,"supported":10}},"extract_error":null,"recovered_error":null,"prompt_pack":null,"scaffold_manifest":null,"stdout_chars":0,"run_record":null,"run_id":null,"status":"refusal","operation":null,"protocol_version":10,"provider":"","response_id":"","stop_reason":"","model":"","data_source_requests":null} diff --git a/src/droste/testing/fixtures/trace-v9-execution.ndjson b/src/droste/testing/fixtures/trace-v10-execution.ndjson similarity index 60% rename from src/droste/testing/fixtures/trace-v9-execution.ndjson rename to src/droste/testing/fixtures/trace-v10-execution.ndjson index 470905f..1b31284 100644 --- a/src/droste/testing/fixtures/trace-v9-execution.ndjson +++ b/src/droste/testing/fixtures/trace-v10-execution.ndjson @@ -1,9 +1,9 @@ -{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":9,"persistence_class":"configurable","depth":0} -{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":9,"persistence_class":"configurable","depth":0} -{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":9,"persistence_class":"configurable","depth":0} -{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":9,"persistence_class":"configurable","depth":0} -{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":9,"persistence_class":"configurable","depth":0} -{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":9,"persistence_class":"configurable","depth":0} -{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":9,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":9,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} -{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":9,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"response":"```python\nprint('first iteration')\n```","run_id":"golden-execution-root","seq":1,"timestamp":"2026-07-16T00:00:00Z","type":"llm_response","version":10,"persistence_class":"configurable","depth":0} +{"iteration":1,"code":"print('first iteration')","run_id":"golden-execution-root","seq":2,"timestamp":"2026-07-16T00:00:01Z","type":"code","version":10,"persistence_class":"configurable","depth":0} +{"iteration":1,"stdout":"ERROR: ordinary successful stdout\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":34,"run_id":"golden-execution-root","seq":3,"timestamp":"2026-07-16T00:00:02Z","type":"output","version":10,"persistence_class":"configurable","depth":0} +{"iteration":2,"response":"```python\nraise ValueError('synthetic failure')\n```","run_id":"golden-execution-root","seq":4,"timestamp":"2026-07-16T00:00:03Z","type":"llm_response","version":10,"persistence_class":"configurable","depth":0} +{"iteration":2,"code":"raise ValueError('synthetic failure')","run_id":"golden-execution-root","seq":5,"timestamp":"2026-07-16T00:00:04Z","type":"code","version":10,"persistence_class":"configurable","depth":0} +{"iteration":2,"error_type":"ValueError","message":"synthetic execution failure","run_id":"golden-execution-root","seq":6,"timestamp":"2026-07-16T00:00:05Z","type":"execution_error","version":10,"persistence_class":"configurable","depth":0} +{"iteration":1,"response":"```python\nprint('child run')\n```","run_id":"golden-execution-child","seq":1,"timestamp":"2026-07-16T00:00:06Z","type":"llm_response","version":10,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"code":"print('child run')","run_id":"golden-execution-child","seq":2,"timestamp":"2026-07-16T00:00:07Z","type":"code","version":10,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} +{"iteration":1,"stdout":"child output\n","calls_made":0,"answer_ready":false,"answer_content_chars":0,"stdout_chars":13,"run_id":"golden-execution-child","seq":3,"timestamp":"2026-07-16T00:00:08Z","type":"output","version":10,"persistence_class":"configurable","parent_run_id":"golden-execution-root","depth":1} diff --git a/src/droste/testing/fixtures/trace-v10-lifecycle.ndjson b/src/droste/testing/fixtures/trace-v10-lifecycle.ndjson new file mode 100644 index 0000000..38eb666 --- /dev/null +++ b/src/droste/testing/fixtures/trace-v10-lifecycle.ndjson @@ -0,0 +1,67 @@ +{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":10,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3} +{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} +{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} +{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} +{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} +{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} +{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} +{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} +{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":10,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} +{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":10,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} +{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":10,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} +{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":10,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"degradations":[],"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9"},"stdout_chars":13}} +{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":10,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"degradations":[],"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3,"stdout_chars":13} +{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} +{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":10,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":10,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} +{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":10,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":10,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":10,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":10,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"degradations":[],"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":10,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"degradations":[],"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} +{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} +{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":10,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} +{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":10,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} +{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":10,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":10,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"degradations":[],"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775"},"stdout_chars":4097}} +{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":10,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"degradations":[],"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775","scaffold_manifest_version":3,"stdout_chars":4097} +{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} +{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} +{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} +{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} +{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} +{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":10,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} +{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":10,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} +{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} +{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":10,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} +{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":10,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} +{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":10,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} +{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":10,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"degradations":[],"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} +{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":10,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"degradations":[],"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} +{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} +{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} +{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":10,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} +{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":10,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} +{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} +{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":10,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} +{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":10,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} +{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":10,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} +{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":10,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} +{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":10,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} +{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":10,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"degradations":[],"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d"},"stdout_chars":0}} +{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":10,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"degradations":[],"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d","scaffold_manifest_version":3,"stdout_chars":0} diff --git a/src/droste/testing/fixtures/trace-v9-lifecycle.ndjson b/src/droste/testing/fixtures/trace-v9-lifecycle.ndjson deleted file mode 100644 index 29f28e8..0000000 --- a/src/droste/testing/fixtures/trace-v9-lifecycle.ndjson +++ /dev/null @@ -1,67 +0,0 @@ -{"run_id":"golden-success","seq":1,"timestamp":"2026-07-15T00:00:00Z","type":"startup","version":9,"persistence_class":"transient","depth":0,"engine_version":"0.17.0","runner_protocol":10,"provider_protocol":4,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3} -{"run_id":"golden-success","seq":2,"timestamp":"2026-07-15T00:00:01Z","type":"iteration_start","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-success","seq":3,"timestamp":"2026-07-15T00:00:02Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"start","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":4,"timestamp":"2026-07-15T00:00:03Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"completion","kind":"missing_code","iteration":1} -{"run_id":"golden-success","seq":5,"timestamp":"2026-07-15T00:00:04Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"unary-ok","operation":"llm_query","iteration":1,"reservation":{"tokens":20,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-success","seq":6,"timestamp":"2026-07-15T00:00:05Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"unary-ok","operation":"llm_query","iteration":1,"checkpoint":{"tokens":8,"subcalls":1}} -{"run_id":"golden-success","seq":7,"timestamp":"2026-07-15T00:00:06Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"batch-failed","operation":"llm_batch","iteration":1,"reservation":{"tokens":40,"subcalls":2,"wall_ms":985,"depth":0},"batch_count":2} -{"run_id":"golden-success","seq":8,"timestamp":"2026-07-15T00:00:07Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"batch-failed","operation":"llm_batch","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"batch_count":2,"error":{"code":"handler_error","type":"RuntimeError"}} -{"run_id":"golden-success","seq":9,"timestamp":"2026-07-15T00:00:08Z","type":"output","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"answer ready\n","calls_made":3,"answer_ready":true,"answer_content_chars":2,"stdout_chars":13} -{"run_id":"golden-success","seq":10,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":10} -{"run_id":"golden-success","seq":11,"timestamp":"2026-07-15T00:00:09Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18} -{"run_id":"golden-success","seq":12,"timestamp":"2026-07-15T00:00:09Z","type":"usage","version":9,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320} -{"run_id":"golden-success","seq":13,"timestamp":"2026-07-15T00:00:10Z","type":"budget","version":9,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}} -{"run_id":"golden-success","seq":14,"timestamp":"2026-07-15T00:00:11Z","type":"policy","version":9,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"passed","violation_type":null} -{"run_id":"golden-success","seq":15,"timestamp":"2026-07-15T00:00:12Z","type":"result","version":9,"persistence_class":"configurable","depth":0,"result":{"answer":"42","answer_metadata":{},"ready":true,"iterations":1,"tokens_used":18,"subcalls":3,"successful_subcalls":1,"extracted":false,"error":null,"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":10,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9"},"stdout_chars":13}} -{"run_id":"golden-success","seq":16,"timestamp":"2026-07-15T00:00:13Z","type":"done","version":9,"persistence_class":"durable","depth":0,"status":"success","ready":true,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":6,"cache_read_tokens":2,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":10,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":6,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":2,"total_tokens":8,"requests":3,"successes":1,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":18,"wall_time_ms":320},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":10,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":58,"subcalls":3,"wall_ms":320,"depth":0},"remaining":{"tokens":942,"subcalls":7,"wall_ms":680,"depth":1}},"policy":{"contract_enforced":true,"outcome":"passed","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:1be656c1c8a79c157e40e84b5f7c2ec6909d5911ed1c5fc6754acbaa6077fae9","scaffold_manifest_version":3,"stdout_chars":13} -{"run_id":"golden-recovered","seq":1,"timestamp":"2026-07-15T00:01:00Z","type":"iteration_start","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-recovered","seq":2,"timestamp":"2026-07-15T00:01:01Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-recovered","seq":3,"timestamp":"2026-07-15T00:01:02Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-recovered","seq":4,"timestamp":"2026-07-15T00:01:03Z","type":"output","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":14,"stdout_chars":0} -{"run_id":"golden-recovered","seq":5,"timestamp":"2026-07-15T00:01:04Z","type":"execution_error","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-recovered","seq":6,"timestamp":"2026-07-15T00:01:05Z","type":"extract","version":9,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-recovered","seq":7,"timestamp":"2026-07-15T00:01:06Z","type":"extract","version":9,"persistence_class":"configurable","depth":0,"phase":"completion","iteration":1} -{"run_id":"golden-recovered","seq":8,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":9,"timestamp":"2026-07-15T00:01:07Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-recovered","seq":10,"timestamp":"2026-07-15T00:01:07Z","type":"usage","version":9,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-recovered","seq":11,"timestamp":"2026-07-15T00:01:08Z","type":"budget","version":9,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-recovered","seq":12,"timestamp":"2026-07-15T00:01:09Z","type":"policy","version":9,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-recovered","seq":13,"timestamp":"2026-07-15T00:01:10Z","type":"result","version":9,"persistence_class":"configurable","depth":0,"result":{"answer":"best-effort evidence","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":true,"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1}},"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} -{"run_id":"golden-recovered","seq":14,"timestamp":"2026-07-15T00:01:11Z","type":"done","version":9,"persistence_class":"durable","depth":0,"status":"success","ready":false,"extracted":true,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":null,"extract_error":null,"recovered_error":{"type":"PolicyError"},"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-output-limit","seq":1,"timestamp":"2026-07-15T00:02:00Z","type":"iteration_start","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-output-limit","seq":2,"timestamp":"2026-07-15T00:02:01Z","type":"execution_error","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"SandboxError","message":"Sandbox output exceeded 4096 characters (attempted 4097). Summarize or aggregate results instead of printing raw rows."} -{"run_id":"golden-output-limit","seq":3,"timestamp":"2026-07-15T00:02:02Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-output-limit","seq":4,"timestamp":"2026-07-15T00:02:03Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-output-limit","seq":5,"timestamp":"2026-07-15T00:02:04Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16} -{"run_id":"golden-output-limit","seq":6,"timestamp":"2026-07-15T00:02:04Z","type":"usage","version":9,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200} -{"run_id":"golden-output-limit","seq":7,"timestamp":"2026-07-15T00:02:05Z","type":"budget","version":9,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}} -{"run_id":"golden-output-limit","seq":8,"timestamp":"2026-07-15T00:02:06Z","type":"policy","version":9,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-output-limit","seq":9,"timestamp":"2026-07-15T00:02:07Z","type":"result","version":9,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint('x' * 4097)\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":16,"subcalls":0,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":50,"subcalls":0,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":50},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":4097,"execution_timeout_ms":0,"output_chars":4096},"id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775"},"stdout_chars":4097}} -{"run_id":"golden-output-limit","seq":10,"timestamp":"2026-07-15T00:02:08Z","type":"done","version":9,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":12,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":4,"total_tokens":16,"requests":2,"successes":1,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":16,"wall_time_ms":200},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":0,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":50,"max_iterations":30},"consumed":{"tokens":416,"subcalls":0,"wall_ms":200,"depth":0},"remaining":{"tokens":584,"subcalls":0,"wall_ms":800,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:a949075b0c302bdf05188f98f97b4024669f2ebdf89e07f6c712267a9ef2b775","scaffold_manifest_version":3,"stdout_chars":4097} -{"run_id":"golden-extract-failed","seq":1,"timestamp":"2026-07-15T00:03:00Z","type":"iteration_start","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":500} -{"run_id":"golden-extract-failed","seq":2,"timestamp":"2026-07-15T00:03:01Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"reservation":{"tokens":30,"subcalls":2,"wall_ms":990,"depth":0},"batch_count":2} -{"run_id":"golden-extract-failed","seq":3,"timestamp":"2026-07-15T00:03:02Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"completion","call_id":"semantic-batch-failed-extract","operation":"llm_batch_with_errors","iteration":1,"checkpoint":{"tokens":15,"subcalls":2},"batch_count":2} -{"run_id":"golden-extract-failed","seq":4,"timestamp":"2026-07-15T00:03:03Z","type":"output","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"stdout":"","calls_made":2,"answer_ready":false,"answer_content_chars":17,"stdout_chars":0} -{"run_id":"golden-extract-failed","seq":5,"timestamp":"2026-07-15T00:03:04Z","type":"execution_error","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence remains unresolved (1 failed item(s) across 1 batch request(s)); rerun each exact request successfully before confirming the answer."} -{"run_id":"golden-extract-failed","seq":6,"timestamp":"2026-07-15T00:03:05Z","type":"extract","version":9,"persistence_class":"configurable","depth":0,"phase":"start","iteration":1} -{"run_id":"golden-extract-failed","seq":7,"timestamp":"2026-07-15T00:03:06Z","type":"extract","version":9,"persistence_class":"configurable","depth":0,"phase":"failure","iteration":1,"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far."}} -{"run_id":"golden-extract-failed","seq":8,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"root","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":9,"timestamp":"2026-07-15T00:03:07Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30} -{"run_id":"golden-extract-failed","seq":10,"timestamp":"2026-07-15T00:03:07Z","type":"usage","version":9,"persistence_class":"durable","depth":0,"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450} -{"run_id":"golden-extract-failed","seq":11,"timestamp":"2026-07-15T00:03:08Z","type":"budget","version":9,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}} -{"run_id":"golden-extract-failed","seq":12,"timestamp":"2026-07-15T00:03:09Z","type":"policy","version":9,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"} -{"run_id":"golden-extract-failed","seq":13,"timestamp":"2026-07-15T00:03:10Z","type":"result","version":9,"persistence_class":"configurable","depth":0,"result":{"answer":"Error: Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":30,"subcalls":2,"successful_subcalls":2,"extracted":false,"error":{"type":"PolicyError","message":"Policy violation: incomplete structured semantic batch evidence cannot be cleared within the subcall budget: clearing all 1 unresolved recorded exact request(s) requires at least 2 call(s), but only 1 remain.","code":null,"details":{"reason":"semantic_exact_retry_budget_exhausted","required_subcalls":2,"remaining_subcalls":1,"unresolved_batches":1,"unresolved_items":1,"withheld_content":"retained evidence"}},"extract_error":{"type":"InsufficientEvidence","message":"Unable to determine from the work so far.","code":null,"details":null},"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":100,"subcall_output_tokens":10,"subcalls":3,"tokens":500,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":100,"subcall_tokens":10},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47"},"stdout_chars":0}} -{"run_id":"golden-extract-failed","seq":14,"timestamp":"2026-07-15T00:03:11Z","type":"done","version":9,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"resolved","root":{"input_tokens":20,"cache_read_tokens":5,"cache_creation_tokens":2,"output_tokens":10,"total_tokens":30,"requests":2,"successes":2,"complete":true},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":2,"successes":2,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":30,"wall_time_ms":450},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":500,"subcalls":3,"depth":1,"wall_ms":1000,"root_output_tokens":100,"subcall_output_tokens":10,"max_iterations":30},"consumed":{"tokens":45,"subcalls":2,"wall_ms":450,"depth":0},"remaining":{"tokens":455,"subcalls":1,"wall_ms":550,"depth":1}},"policy":{"contract_enforced":true,"outcome":"violated","violation_type":"PolicyError"},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"PolicyError"},"extract_error":{"type":"InsufficientEvidence"},"recovered_error":null,"scaffold_manifest_id":"sha256:d0e6bb585d002c3670d549e93fd9c87179753b280d01875e0aa40cf0565c2c47","scaffold_manifest_version":3,"stdout_chars":0} -{"run_id":"golden-cancelled","seq":1,"timestamp":"2026-07-15T00:04:00Z","type":"iteration_start","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"remaining_tokens":1000} -{"run_id":"golden-cancelled","seq":2,"timestamp":"2026-07-15T00:04:01Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"start","call_id":"cancelled-call","operation":"llm_query","iteration":1,"reservation":{"tokens":10,"subcalls":1,"wall_ms":990,"depth":0}} -{"run_id":"golden-cancelled","seq":3,"timestamp":"2026-07-15T00:04:02Z","type":"subcall","version":9,"persistence_class":"configurable","depth":0,"phase":"failure","call_id":"cancelled-call","operation":"llm_query","iteration":1,"checkpoint":{"tokens":0,"subcalls":0},"error":{"code":"cancelled","type":"CapabilityCancelled"}} -{"run_id":"golden-cancelled","seq":4,"timestamp":"2026-07-15T00:04:03Z","type":"execution_error","version":9,"persistence_class":"configurable","depth":0,"iteration":1,"error_type":"CapabilityCallError","message":"CapabilityCancelled: capability call was cancelled"} -{"run_id":"golden-cancelled","seq":5,"timestamp":"2026-07-15T00:04:04Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"start","kind":"execution_error","iteration":1} -{"run_id":"golden-cancelled","seq":6,"timestamp":"2026-07-15T00:04:05Z","type":"repair","version":9,"persistence_class":"configurable","depth":0,"phase":"failure","kind":"execution_error","iteration":1,"error":{"type":"RuntimeError","message":"repair provider unavailable"}} -{"run_id":"golden-cancelled","seq":7,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"root","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":0,"successes":0,"complete":true},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":8,"timestamp":"2026-07-15T00:04:06Z","type":"usage_progress","version":9,"persistence_class":"transient","depth":0,"boundary":"subcall","kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12} -{"run_id":"golden-cancelled","seq":9,"timestamp":"2026-07-15T00:04:06Z","type":"usage","version":9,"persistence_class":"durable","depth":0,"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150} -{"run_id":"golden-cancelled","seq":10,"timestamp":"2026-07-15T00:04:07Z","type":"budget","version":9,"persistence_class":"durable","depth":0,"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}} -{"run_id":"golden-cancelled","seq":11,"timestamp":"2026-07-15T00:04:08Z","type":"policy","version":9,"persistence_class":"durable","depth":0,"contract_enforced":true,"outcome":"not_evaluated","violation_type":null} -{"run_id":"golden-cancelled","seq":12,"timestamp":"2026-07-15T00:04:09Z","type":"result","version":9,"persistence_class":"configurable","depth":0,"result":{"answer":"```python\nprint(llm_query('cancel me'))\n```","answer_metadata":{},"ready":false,"iterations":1,"tokens_used":12,"subcalls":1,"successful_subcalls":0,"extracted":false,"error":{"type":"RuntimeError","message":"repair provider unavailable","code":null,"details":null},"extract_error":null,"recovered_error":null,"prompt_pack":{"id":"droste.generic.full","revision":"1.0.3","profile":"full","resolution_tier":"generic","model_family":"generic","provenance_source":"droste","provenance_benchmark":null,"provenance_score":null,"content_sha256":"61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7"},"scaffold_manifest":{"schema_version":3,"abis":{"capability":1,"kernel":1,"prompt_pack":1,"provider":4,"runner":10,"trace":7},"budget":{"depth":1,"root_output_tokens":50,"subcall_output_tokens":5,"subcalls":1,"tokens":1000,"wall_ms":1000,"max_iterations":30},"capabilities":{"manifest_hash":"sha256:33a8de001c12c2f7ba57236980c4aafab7a1b8bfcc60ac4bb891a3a3c0089579","model_visible_globals":["answer"]},"contracts":{"overrides":{"refinement_prompt":null,"system_prompt":null,"system_prompt_additions":null,"user_prompt":null},"subcall_identity":"capability-call-parent-v1","templates":{"error_repair":"sha256:1056c9f99b9fcbc3fd021775a80c18f38b2454cb92cb062cb089aa86b8067913","extract_system":"sha256:6616daeb153049f23365d9c04cac1e255dbb76a7f81d807d6c830e2720dc6af8","extract_user":"sha256:81c0370baeceb252547baca669cd6c82d55ac858c3283d3cd41554d7082c0ec1","missing_code_repair":"sha256:4033a5d7901934c6b6de279d5b21ced97354ccff4bcb1fb10c211dba89a57173","refinement":"sha256:9b4700c0db49443f39d1b43c0dc0f43d9aa1d809cba0585fdcb5ea3d399c8046"},"terminal":"answer-ready-v1"},"engine":{"source_revision":null,"version":"0.17.0"},"inference":{"concurrency":5,"input_capacity":{"subcall":{"state":"unknown","tokens":null}},"output_limits":{"root_tokens":50,"subcall_tokens":5},"root":{"id":"root-model","revision":null},"root_sampling":{},"seed":null,"subcall":{"id":"root-model","revision":null},"subcall_sampling":{}},"parent_child":{"identity":"capability-call-parent-v1","trace_depth":"root-zero-child-increment"},"prompt_pack":{"content_hash":"sha256:61da9b9cfae4f2b6c0efddc4c9232cbbfdd9e8187f977874a4ba62aefc9bcfb7","id":"droste.generic.full","profile":"full","revision":"1.0.3"},"sandbox":{"capture_output_chars":25000,"execution_timeout_ms":0,"output_chars":25000},"id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d"},"stdout_chars":0}} -{"run_id":"golden-cancelled","seq":13,"timestamp":"2026-07-15T00:04:10Z","type":"done","version":9,"persistence_class":"durable","depth":0,"status":"error","ready":false,"extracted":false,"iterations":1,"usage":{"kind":"partial","root":{"input_tokens":8,"cache_read_tokens":3,"cache_creation_tokens":1,"output_tokens":4,"total_tokens":12,"requests":2,"successes":1,"complete":false},"subcall":{"input_tokens":0,"cache_read_tokens":0,"cache_creation_tokens":0,"output_tokens":0,"total_tokens":0,"requests":1,"successes":0,"complete":false},"unattributed":{"total_tokens":0},"total_tokens":12,"wall_time_ms":150},"budget":{"kind":"snapshot","source":"budget_ledger","configured":{"tokens":1000,"subcalls":1,"depth":1,"wall_ms":1000,"root_output_tokens":50,"subcall_output_tokens":5,"max_iterations":30},"consumed":{"tokens":372,"subcalls":1,"wall_ms":150,"depth":0},"remaining":{"tokens":628,"subcalls":0,"wall_ms":850,"depth":1}},"policy":{"contract_enforced":true,"outcome":"not_evaluated","violation_type":null},"retention":{"policy_id":"conformance-all-content","retain":["execution_error","extract","iteration_start","output","repair","result","subcall"],"expires_at":null,"host_managed_expiry":false,"replay_retained":false},"error":{"type":"RuntimeError"},"extract_error":null,"recovered_error":null,"scaffold_manifest_id":"sha256:b689ad95c39f7bd038640a287fb25600b7951badbd9af332ff9310a72fbba99d","scaffold_manifest_version":3,"stdout_chars":0} diff --git a/src/droste_runner/protocol.py b/src/droste_runner/protocol.py index ec0f5da..6d8ede6 100644 --- a/src/droste_runner/protocol.py +++ b/src/droste_runner/protocol.py @@ -63,6 +63,9 @@ def build_response( "subcalls": 0, "successful_subcalls": 0, "extracted": False, + # Same key on every response shape: a consumer reads one field to know + # what was lost, and never has to treat a missing key as "nothing". + "degradations": [], "error": error, "extract_error": None, "recovered_error": None, diff --git a/tests/test_answer_checkpoints.py b/tests/test_answer_checkpoints.py index f89e312..042de78 100644 --- a/tests/test_answer_checkpoints.py +++ b/tests/test_answer_checkpoints.py @@ -72,8 +72,8 @@ def _valid_body(**overrides: object) -> dict[str, object]: # --- Layer 1: the wire contract ---------------------------------------------- -def test_trace_abi_is_version_nine_and_knows_checkpoint() -> None: - assert TRACE_ABI_VERSION == 9 +def test_trace_abi_is_version_ten_and_knows_checkpoint() -> None: + assert TRACE_ABI_VERSION == 10 assert "checkpoint" in EVENT_TYPES @@ -167,7 +167,7 @@ def test_checkpoint_follows_each_executed_step_whose_draft_moved() -> None: assert [event["ready"] for event in checkpoints] == [False, True] assert [event["iteration"] for event in checkpoints] == [1, 2] assert all(event["payload"] is None for event in checkpoints) - assert all(event["version"] == 9 for event in checkpoints) + assert all(event["version"] == 10 for event in checkpoints) assert all(event["persistence_class"] == "configurable" for event in checkpoints) diff --git a/tests/test_environment_factory.py b/tests/test_environment_factory.py index 5c5e23e..43b55a1 100644 --- a/tests/test_environment_factory.py +++ b/tests/test_environment_factory.py @@ -206,9 +206,36 @@ def test_native_and_pyodide_deliver_the_same_ordered_terminal_event_lifecycle() context=context, ) assert result.answer == "done" - event_orders.append(require_ordered_terminal_events(events)) - - assert event_orders[0] == event_orders[1] + try: + event_orders.append(require_ordered_terminal_events(events)) + except AssertionError as exc: + raise AssertionError( + f"{config.kind} substrate emitted an invalid lifecycle: {exc}\n" + f"observed: {[event.get('type') for event in events]}" + ) from exc + + native_order, pyodide_order = event_orders + if native_order != pyodide_order: + # Both substrates ran the same mock provider over the same code, so a + # divergence names the offending substrate and index rather than + # printing two anonymous tuples. Written this way because a rare + # failure of this assertion is the only chance to diagnose it: it has + # been seen once, on a heavily loaded machine, and never reproduced. + index = next( + ( + i + for i, (native, pyodide) in enumerate(zip(native_order, pyodide_order)) + if native != pyodide + ), + min(len(native_order), len(pyodide_order)), + ) + raise AssertionError( + "native and pyodide emitted different lifecycles; they must agree\n" + f"first difference at index {index}: " + f"{native_order[index : index + 1]} vs {pyodide_order[index : index + 1]}\n" + f"native ({len(native_order)}): {native_order}\n" + f"pyodide ({len(pyodide_order)}): {pyodide_order}" + ) def test_native_rejects_pyodide_only_safety_declarations() -> None: diff --git a/tests/test_runner_subcall_reporting.py b/tests/test_runner_subcall_reporting.py index 38680bf..50d9614 100644 --- a/tests/test_runner_subcall_reporting.py +++ b/tests/test_runner_subcall_reporting.py @@ -253,7 +253,7 @@ def __init__(self, **kwargs: Any) -> None: "seed": 17, } assert manifest["abis"]["runner"] == 10 - assert manifest["abis"]["trace"] == 9 + assert manifest["abis"]["trace"] == 10 assert manifest["engine"]["source_revision"] == "commit-a" assert manifest["id"].startswith("sha256:") assert "trajectory" not in response @@ -295,7 +295,7 @@ def __init__(self, **kwargs: Any) -> None: ] assert len({event["call_id"] for event in live_subcalls}) == 1 assert live_subcalls[1]["checkpoint"] == {"tokens": 5, "subcalls": 1} - assert all(event["iteration"] == 1 and event["version"] == 9 for event in live_subcalls) + assert all(event["iteration"] == 1 and event["version"] == 10 for event in live_subcalls) assert capability["outcome"]["capability_id"]["operation"] == "llm_query" assert "params" not in capability["outcome"] assert "result" not in capability["outcome"] diff --git a/tests/test_subcall_input_capacity.py b/tests/test_subcall_input_capacity.py index 6ed160d..6a34e06 100644 --- a/tests/test_subcall_input_capacity.py +++ b/tests/test_subcall_input_capacity.py @@ -349,7 +349,7 @@ def test_runner_preflight_records_declared_capacity() -> None: } -def test_runner_v5_is_refused_before_trace_v9_can_be_ignored() -> None: +def test_runner_v5_is_refused_before_trace_v10_can_be_ignored() -> None: response = run_worker( { "protocol_version": 5, diff --git a/tests/test_trace_abi.py b/tests/test_trace_abi.py index aa31413..f6c4b4e 100644 --- a/tests/test_trace_abi.py +++ b/tests/test_trace_abi.py @@ -33,10 +33,10 @@ MockResponse, MockSubcallClient, runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, + trace_v10_execution_ndjson, + trace_v10_lifecycle_ndjson, ) -from droste.testing._trace_fixtures import build_trace_v9_execution_ndjson +from droste.testing._trace_fixtures import build_trace_v10_execution_ndjson from droste_runner.run import run as run_worker @@ -124,6 +124,7 @@ def _terminal() -> dict[str, object]: "status": "success", "ready": True, "extracted": False, + "degradations": [], "iterations": 0, "usage": usage, "budget": budget, @@ -233,7 +234,7 @@ def test_usage_progress_emits_cumulative_role_boundaries_without_estimating() -> assert progress[1]["root"]["total_tokens"] == 10 assert progress[1]["subcall"]["total_tokens"] == 7 assert progress[1]["persistence_class"] == "transient" - assert progress[1]["version"] == 9 + assert progress[1]["version"] == 10 def test_concurrent_subcall_usage_progress_is_serialized_and_monotonic() -> None: @@ -582,7 +583,7 @@ def test_run_record_allows_repeated_durable_budget_mutations() -> None: ] -def test_parser_requires_v9_envelope_and_rejects_false_classification() -> None: +def test_parser_requires_v10_envelope_and_rejects_false_classification() -> None: with pytest.raises(ValueError, match="missing envelope fields"): parse_event({"type": "code", "iteration": 1, "code": "print(1)"}) @@ -592,7 +593,7 @@ def test_parser_requires_v9_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 9, + "version": 10, "persistence_class": "configurable", "depth": 0, "iteration": 1, @@ -607,7 +608,7 @@ def test_parser_requires_v9_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 2, "timestamp": "2026-07-14T00:00:00Z", - "version": 9, + "version": 10, "persistence_class": "transient", "depth": 0, "engine_version": "0.10.6", @@ -624,7 +625,7 @@ def test_parser_requires_v9_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T00:00:00Z", - "version": 9, + "version": 10, "persistence_class": "durable", "depth": 0, "code": "print(1)", @@ -638,7 +639,7 @@ def test_parser_requires_v9_envelope_and_rejects_false_classification() -> None: "run_id": "run", "seq": 1, "timestamp": "2026-07-14T01:00:00+01:00", - "version": 9, + "version": 10, "persistence_class": "transient", "depth": 0, "status": "working", @@ -673,7 +674,7 @@ def test_event_bodies_reject_missing_unknown_and_wrong_primitive_fields() -> Non recorder.finish(terminal) -def test_trace_v9_budget_snapshot_requires_max_iterations() -> None: +def test_trace_v10_budget_snapshot_requires_max_iterations() -> None: recorder = TraceRecorder(run_id="strict-budget") budget = json.loads(json.dumps(_terminal()["budget"])) del budget["configured"]["max_iterations"] @@ -844,11 +845,11 @@ def test_trace_abi_v3_is_rejected_instead_of_dropping_cache_usage() -> None: ) -def _trace_v9_golden_runs() -> dict[str, list[RunEvent]]: +def _trace_v10_golden_runs() -> dict[str, list[RunEvent]]: runs: dict[str, list[RunEvent]] = {} previous_run_id: str | None = None closed: set[str] = set() - for line in trace_v9_lifecycle_ndjson().decode("utf-8").splitlines(): + for line in trace_v10_lifecycle_ndjson().decode("utf-8").splitlines(): event = parse_event(json.loads(line)) if event.run_id != previous_run_id: if event.run_id in closed: @@ -860,9 +861,9 @@ def _trace_v9_golden_runs() -> dict[str, list[RunEvent]]: return runs -def test_trace_v9_execution_fixture_is_canonical_and_exact() -> None: - fixture = trace_v9_execution_ndjson() - assert fixture == build_trace_v9_execution_ndjson() +def test_trace_v10_execution_fixture_is_canonical_and_exact() -> None: + fixture = trace_v10_execution_ndjson() + assert fixture == build_trace_v10_execution_ndjson() events = [parse_event(json.loads(line)) for line in fixture.decode("utf-8").splitlines()] root = events[:6] @@ -894,8 +895,8 @@ def _error_type(value: object) -> object: return value.get("type") if isinstance(value, Mapping) else None -def test_trace_v9_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: - runs = _trace_v9_golden_runs() +def test_trace_v10_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> None: + runs = _trace_v10_golden_runs() assert set(runs) == { "golden-success", "golden-recovered", @@ -991,8 +992,8 @@ def test_trace_v9_lifecycle_golden_ndjson_is_strict_ordered_and_terminal() -> No assert terminal["usage"]["subcall"]["cache_creation_tokens"] == 1 -def test_trace_v9_golden_corpus_covers_each_discriminated_lifecycle() -> None: - events = [event for run in _trace_v9_golden_runs().values() for event in run] +def test_trace_v10_golden_corpus_covers_each_discriminated_lifecycle() -> None: + events = [event for run in _trace_v10_golden_runs().values() for event in run] subcalls = [event for event in events if event.type == "subcall"] repairs = [event for event in events if event.type == "repair"] extracts = [event for event in events if event.type == "extract"] @@ -1019,7 +1020,7 @@ def test_trace_v9_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert {event.body["boundary"] for event in usage_progress} == {"root", "subcall"} assert all(event.persistence_class is PersistenceClass.TRANSIENT for event in usage_progress) - output_limit = _trace_v9_golden_runs()["golden-output-limit"] + output_limit = _trace_v10_golden_runs()["golden-output-limit"] assert not any(event.type == "output" for event in output_limit) assert any( event.type == "execution_error" and event.body["error_type"] == "SandboxError" @@ -1043,7 +1044,7 @@ def test_trace_v9_golden_corpus_covers_each_discriminated_lifecycle() -> None: event.body["message"] for event in output_limit if event.type == "execution_error" ) assert f"exceeded {output_manifest['sandbox']['output_chars']} characters" in output_error - cancelled = _trace_v9_golden_runs()["golden-cancelled"] + cancelled = _trace_v10_golden_runs()["golden-cancelled"] assert cancelled[-1].body["status"] == "error" assert any( event.type == "execution_error" and event.body["error_type"] == "CapabilityCallError" @@ -1057,7 +1058,7 @@ def test_trace_v9_golden_corpus_covers_each_discriminated_lifecycle() -> None: assert cancelled[-1].body["usage"]["root"]["complete"] is False assert cancelled[-1].body["usage"]["subcall"]["complete"] is False - extract_failed = _trace_v9_golden_runs()["golden-extract-failed"] + extract_failed = _trace_v10_golden_runs()["golden-extract-failed"] result = next(event.body["result"] for event in extract_failed if event.type == "result") assert result["answer"] == f"Error: {result['error']['message']}" assert result["error"]["details"]["withheld_content"] == "retained evidence" @@ -1361,8 +1362,8 @@ def test_conformance_corpus_is_enumerated_not_listed() -> None: from droste.testing import ( conformance_fixture_names, runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, + trace_v10_execution_ndjson, + trace_v10_lifecycle_ndjson, ) names = conformance_fixture_names() @@ -1374,12 +1375,133 @@ def test_conformance_corpus_is_enumerated_not_listed() -> None: # driven by it stages exactly what the helpers can read. assert set(names) == { "runner-v10-refusal.ndjson", - "trace-v9-execution.ndjson", - "trace-v9-lifecycle.ndjson", + "trace-v10-execution.ndjson", + "trace-v10-lifecycle.ndjson", } for reader in ( runner_v10_refusal_ndjson, - trace_v9_execution_ndjson, - trace_v9_lifecycle_ndjson, + trace_v10_execution_ndjson, + trace_v10_lifecycle_ndjson, ): assert reader(), "every enumerated fixture must be non-empty" + + +def test_a_failing_budget_sink_reports_the_event_it_lost() -> None: + """A dropped event must not be inferable only from a shorter stream. + + The ledger deliberately refuses to let a host's sink end the run, but it + used to pay for that with a warning no consumer of the result ever sees: + the stream came back one event short and nothing said so. + """ + + import warnings + + from droste.execution.budget import Budget, BudgetLedger, BudgetRequest + + calls = {"n": 0} + + def flaky_sink(event: dict) -> None: + calls["n"] += 1 + if calls["n"] == 2: + raise RuntimeError("transient sink failure") + + ledger = BudgetLedger(budget=Budget(), on_event=flaky_sink) + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + for index in range(4): + ledger.reserve(f"call-{index}", BudgetRequest(tokens=10)) + + (dropped,) = ledger.dropped_events() + assert dropped["error_type"] == "RuntimeError" + assert "transient sink failure" in dropped["detail"] + + +def test_a_clean_run_reports_no_degradations() -> None: + from droste.execution.context import ExecutionContext + + assert ExecutionContext().degradations() == () + + +def test_a_recorded_degradation_names_what_the_run_did_without() -> None: + from droste.execution.context import ExecutionContext + + context = ExecutionContext() + context.record_degradation( + site="extract_context_provider", + error=RuntimeError("host blew up"), + consequence="terminal extract ran without host-held observations", + ) + + (degradation,) = context.degradations() + assert degradation.site == "extract_context_provider" + assert degradation.error_type == "RuntimeError" + # The consequence, not just the exception, is what a reader needs. + assert "without host-held observations" in degradation.consequence + + +def test_a_degraded_run_says_so_on_the_wire() -> None: + """The whole point: a consumer of the event stream can see the loss. + + Recording a degradation the host never receives would be the same defect + one layer up, so assert it reaches the emitted result event. + """ + + from droste import EnvironmentConfig, RLMConfig, create_environment, run_rlm + from droste.environments import create_environment_context + from droste.protocols.llm_client import TokenUsage + from droste.testing import MockLLMClient, MockResponse, MockSubcallClient + + events: list[dict] = [] + config = EnvironmentConfig(kind="native") + context = create_environment_context(config, on_event=events.append, run_id="degraded") + subcalls = MockSubcallClient() + environment = create_environment( + config, context={}, registry=None, subcalls=subcalls, execution_context=context + ) + context.record_degradation( + site="extract_context_provider", + error=RuntimeError("host blew up"), + consequence="terminal extract ran without host-held observations", + ) + result = run_rlm( + "question", + environment=environment, + root_llm=MockLLMClient( + [ + MockResponse( + "```python\nanswer['content'] = 'done'\nanswer['ready'] = True\n```", + TokenUsage(prompt_tokens=1, completion_tokens=1, total_tokens=2, exact=True), + ) + ] + ), + subcalls=subcalls, + config=RLMConfig(run_id="degraded"), + context=context, + ) + + assert len(result.degradations) == 1 + (wire_result,) = [e["result"] for e in events if e["type"] == "result"] + (reported,) = wire_result["degradations"] + assert reported["site"] == "extract_context_provider" + assert reported["consequence"] == ("terminal extract ran without host-held observations") + + +def test_a_clean_run_still_reports_the_field_rather_than_omitting_it() -> None: + """Absence must never be how a consumer learns nothing was lost.""" + + from droste.execution.report import project_result + from droste.loop.step import RLMResult + + projected = project_result( + RLMResult( + answer="a", + ready=True, + iterations=1, + tokens_used=0, + sub_calls_made=0, + trajectory=[], + ), + include_trajectory=False, + ) + + assert projected["degradations"] == [] diff --git a/uv.lock b/uv.lock index 80b4dcd..e6bf16e 100644 --- a/uv.lock +++ b/uv.lock @@ -562,7 +562,7 @@ wheels = [ [[package]] name = "droste" -version = "0.24.0" +version = "0.25.0" source = { editable = "." } [package.optional-dependencies]