From 286390cb7a68b30c82080c4166e1732b7271e698 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Mon, 22 Jun 2026 18:48:55 -0400 Subject: [PATCH 1/5] feat: add unified transcript resume state --- README.md | 38 +++- docs/behavior.md | 15 ++ docs/docs.md | 21 +- docs/site/about/index.html | 2 +- docs/site/explainer/index.html | 18 +- tests/test_approvals.py | 33 ++- tests/test_providers.py | 94 +++++++- tests/test_resume.py | 243 +++++++++++++++++--- thinharness/approvals.py | 11 +- thinharness/providers.py | 398 +++++++++++++++++++++++++-------- 10 files changed, 725 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index 52b216a..586b881 100644 --- a/README.md +++ b/README.md @@ -292,7 +292,7 @@ Streaming emits coarse run, model, tool, background, retry, limit, and subagent - **Subagents:** opt-in delegation through a built-in `subagent` tool and explicit `SubAgentConfig`. - **Parallel LLM:** opt-in `parallel_llm` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable tools with explicit model, path, prompt, and retry settings. - **Skills:** explicit `skill_read` and `skill_run` tools for selected skill directories, with Python, shell, JavaScript, and Go script runners. -- **Resume:** clean new-turn continuation through opaque provider session state. +- **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models. - **MCP:** optional MCP client support with lazy tool discovery and collision checks. - **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. - **Background tools:** opt-in long-running tool calls return a start notice immediately, keep the agent loop moving, and deliver completion back to the model when ready. @@ -302,12 +302,48 @@ Streaming emits coarse run, model, tool, background, retry, limit, and subagent - **Limits and notices:** configured request, tool-call, output-retry, and tool-retry budgets bound each run; near-limit guidance can warn the model before request or tool-call budgets are exhausted. - **Tracing:** local plaintext JSONL traces plus OpenTelemetry-compatible spans for runs, provider calls, tools, and subagents. +## Resume + +Cleanly completed runs return `HarnessResult.resume_state`, a JSON-serializable transcript that can be passed back as `resume_from` with the next user message. + +```python +first = await harness.run("Summarize this repository.") +second = await harness.run("Now turn that into a checklist.", resume_from=first.resume_state) +``` + +For the built-in OpenAI, Anthropic, and OpenRouter adapters, resume state is self-contained and provider-agnostic. A run captured with one built-in provider or model can be replayed by another built-in provider or model, subject to provider wire-format acceptance for tool-call ids and argument JSON. The live harness config supplies the system prompt and tool schemas on resume; captured system prompts are not stored. + +Provider-specific reasoning chains are not preserved in version 2 resume state. OpenAI runs seeded manually with `previous_response_id` still work, but later resume state captures only the new transcript entries, not the externally seeded prior turns. + ## Tracing Local tracing is on by default. It writes full plaintext JSONL traces under `~/.thinharness/traces//`, including prompts, model outputs, tool arguments, and tool results, so treat that directory as sensitive local data. Set `local_tracing=False` or `THINHARNESS_DISABLE_LOCAL_TRACING=1` to disable local trace files. External tracing is generic OpenTelemetry: pass any tracer with `start_as_current_span(...)` or `start_span(...)` in `TracingOptions`, and each sink keeps its own capture policy. +## Examples + +Three agents built on ThinHarness, from a self-contained demo to a benchmark run to one I use live. + +### 1. Web Research Report + +A market-landscape research agent that plans, runs batched Exa search, triages and fetches sources, extracts structured source notes with `parallel_llm`, drafts a report, and runs a `citation_critic` subagent before finalizing. + +It isn't meant to be a state-of-the-art research agent; it's a worked example showing the harness drive a non-trivial agentic loop correctly — real multi-tool use across 15 model turns, ending in a reasonable cited report. The full run is browsable on the [docs site](https://ryanbbrown.com/thinharness/examples). + +### 2. LongMemEval-V2 Reproduction + +I ran ThinHarness on a 127-question subset of a real long-term-memory retrieval benchmark, and did a local reproduction of the benchmark's optimized harness on the same subset. + +- **Performance:** Matched-or-better accuracy (74.0% vs 72.4% on the 127 dynamic questions) with ~46% less token usage (62M vs. 116M). See [my fork](https://github.com/ryanbbrown/LongMemEval-V2) for more details. +- **Simpler Setup:** ThinHarness only had its built-in filesystem tools (with `jsonl_search` doing the heavy lifting), while the benchmark harness was a full Codex instance with shell and a custom Python tool designed for the task. + +### 3. Personal Opinions Agent + +An agent I run live for myself, inspired by [this Substack post](https://blog.kunchenguid.com/p/everyone-should-have-an-opinionsmd). On a schedule, it reads my Readwise highlights + surrounding context, draws on what it's learned in past runs, and proposes conceptual changes to a durable `OPINIONS.md`. + +Approval happens over Telegram, and it can be simple accept/reject or involve multi-turn revision + discussion. Under the hood it exercises filesystem tools over a JSONL corpus, native structured output, long-term memory, a custom validation tool, and resumable conversations that pick up across each round. See [here](https://github.com/ryanbbrown/opinions-agent) for the code. + ## Status Pre-1.0. APIs may shift, but I don't expect dramatic changes. Forking is a real option, not just a theoretical one: the codebase is small enough that pulling upstream changes into your fork by hand stays cheap. Each major feature (MCP, subagents, jsonl_search, parallel_llm, background tools, skills) lives in its own file with no hidden dependencies. If you don't use one, that's even less code to worry about. If you want to delete it entirely, that's a one-shot 10-word prompt to a coding agent. diff --git a/docs/behavior.md b/docs/behavior.md index b42f2b1..ea2fd5b 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -62,3 +62,18 @@ Use this section only when ordering, lifecycle, concurrency, retries, streaming, - JSONL-TYPED-EQUALITY-3: Date equality filters compare ISO-like date and datetime strings, compare date-only values by calendar date, and treat aware/naive datetime mismatches as non-comparable. - JSONL-TYPED-EQUALITY-4: Non-comparable row values do not match either `eq` or `ne` and increment `compare_warnings` once per candidate row where a typed equality comparison was attempted. - JSONL-TYPED-EQUALITY-5: Invalid typed equality filter definitions fail before scanning rows with `invalid where filter`. + +## Resume State + +### Purpose + +Built-in provider resume state is a self-contained, provider-agnostic transcript that can be replayed by any built-in provider or model while preserving the run lifecycle rules for when resume state is available. + +### Requirements + +- RESUME-1: `resume_state` is a provider-agnostic transcript; resume across built-in providers and across built-in models is supported. +- RESUME-2: `resume_state` is self-contained and does not depend on provider continuation tokens such as OpenAI `previous_response_id`; an OpenAI run that never received a response id is still resumable. +- RESUME-3: Resume state version 2 preserves reasoning as visible transcript text only and does not preserve provider-specific reasoning chains. +- RESUME-4: Built-in provider resume state uses `version` 2; version 1 state and old provider-native `kind` values are rejected with a regenerate error. +- RESUME-5: On resume, the live system prompt from the resuming harness config is re-injected; captured system prompts are not stored or restored. +- RESUME-6: A session seeded via `OpenAIResponsesSession.start(previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. diff --git a/docs/docs.md b/docs/docs.md index c6ba132..b76e6ce 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -245,11 +245,11 @@ Set `requires_approval=True` on a custom `ToolSpec` when the host application mu The paused result includes: - `pending_approvals`: call id, tool name, and raw JSON arguments for each approval-required call. -- `resume_state`: one JSON-serializable approval envelope that wraps provider resume state, the full paused tool batch, run history, usage, metadata, and accounting needed to continue the same logical run. +- `resume_state`: one JSON-serializable approval envelope that wraps the provider resume payload, the full paused tool batch, run history, usage, metadata, and accounting needed to continue the same logical run. For built-in providers, the nested `provider_state` is the full neutral transcript. Resume with `resume_approvals(...)`, `stream_approvals(...)`, or `resume_approvals_sync(...)` and one `ApprovalDecision` per pending approval. Approved calls execute through the normal tool machinery, including hooks, tracing, retry accounting, and stream events. Rejected calls do not execute or fire tool hooks; the model receives a failed tool result with `error_type="ApprovalRejected"` and can explain, recover, or request another tool. -Approval-required tools need a resumable model because the harness must continue the exact provider session after the paused assistant tool-call turn. They cannot use background execution, and they are not supported inside child subagent harnesses. Built-in tools remain non-approval tools in this version; wrap built-in behavior in a custom `ToolSpec` when host review is required. +Approval-required tools need a resumable model because the harness must continue after the paused assistant tool-call turn. They cannot use background execution, and they are not supported inside child subagent harnesses. Built-in tools remain non-approval tools in this version; wrap built-in behavior in a custom `ToolSpec` when host review is required. ### Bash Prototype Tool @@ -559,21 +559,24 @@ The contract: - Save `result.resume_state` exactly as JSON. - Pass it back as `resume_from` with the next user message. -- Use the same provider, model, system prompt, and tools as the run that produced it. +- Built-in provider state is a self-contained transcript and can be resumed by any built-in provider or model. +- The resuming harness supplies the live system prompt and tool schemas; captured system prompts are not stored or restored. - Expect no state after failed, cancelled, partial, or exhausted runs. -- Treat the contents as provider-owned details; do not read or construct them. +- Treat the contents as harness-owned details; persist them exactly, but do not construct them by hand. `resume_from` is a new-turn API. The prior run completed, and the next call appends a new user message. It is not a retry mechanism, interrupted-tool-call recovery, or a way to continue the assistant's previous response. Approval pauses use a separate resume path. When `stop_reason == "approval_required"`, persist the returned `resume_state` envelope and call `resume_approvals(...)` with approval decisions instead of passing that envelope to `run(..., resume_from=...)`. The post-resume result carries the full logical run history: pre-pause responses, tool records, usage counters, and metadata are restored before the approved or rejected batch is processed. -Budgets span the pause. The paused batch counts against `usage.tool_calls` exactly once at pause time, and a resumed run can immediately hit `limit_reached` if the logical run was already at its configured model-request or tool-call limit. The approval envelope also includes raw provider responses, so its stored size grows with run length. +Budgets span the pause. The paused batch counts against `usage.tool_calls` exactly once at pause time, and a resumed run can immediately hit `limit_reached` if the logical run was already at its configured model-request or tool-call limit. The approval envelope also includes the full nested provider transcript and raw provider responses, so its stored size grows with run length. `APPROVAL_ENVELOPE_VERSION` remains independent from the nested provider-state version; old nested provider states fail when the provider resume step validates them. -Provider behavior differs internally: +Built-in provider resume details: -- OpenAI Responses stores conversation state server-side. `resume_state` contains the previous response id. -- Anthropic Messages stores the full message transcript in `resume_state`. -- OpenRouter chat completions stores the full chat transcript in `resume_state`. +- `resume_state["kind"] == "transcript"` and `version == 2`. +- The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. +- Provider-specific reasoning chains are not preserved; visible reasoning text, ordinary assistant text, tool calls, user messages, tool results, and harness notices are replayed. +- Cross-provider resume is supported by the built-in renderers, but real providers may reject foreign-format tool-call ids or malformed tool-call argument JSON. +- `OpenAIResponsesSession.start(previous_response_id=...)` remains available as a low-level escape hatch, but later resume state captures only the new prompt onward, not the externally seeded prior turns. The same `resume_state` can be reused for sequential branching: diff --git a/docs/site/about/index.html b/docs/site/about/index.html index 0e495f5..bc7ab6f 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -193,7 +193,7 @@

Features

Subagents

Opt-in delegation through a built-in subagent tool and explicit SubAgentConfig.

Parallel LLM

Opt-in parallel_llm fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable tools with explicit model, path, prompt, and retry settings.

Skills

Explicit skill_read and skill_run tools for selected skill directories, with Python, shell, JavaScript, and Go script runners.

-
Resume

Clean new-turn continuation through opaque provider session state.

+
Resume

Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models.

MCP

Optional MCP client support with lazy tool discovery and collision checks.

Parallel tool calls

Same-turn tool batches run concurrently when every called tool is parallel-safe.

Background tools

Opt-in long-running tool calls return a start notice immediately, keep the agent loop moving, and deliver completion back to the model when ready.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index 9fcccaa..a827866 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -205,7 +205,7 @@

Provider-neutral objects

NameMeaning ModelProtocol for reusable model configuration. It creates isolated ModelSession objects. - ModelSessionPer-run provider conversation state. Anthropic and OpenRouter sessions store message lists; OpenAI Responses stores previous_response_id. All expose start, continuation, correction, resume, and dump_state methods. + ModelSessionPer-run provider conversation state. Built-in sessions keep native in-run state plus a parallel neutral transcript; dump_state returns the transcript for provider-agnostic resume. All expose start, continuation, correction, resume, and dump_state methods. ModelTurnNormalized provider response: assistant text, requested ModelToolCall entries, raw provider JSON. ModelToolCallNormalized tool request with id, name, and raw JSON argument string. ToolOutputTool result sent back to the provider so the model can continue after a tool call: call id plus model-visible output string. @@ -386,21 +386,21 @@

Providers and Sessions

OpenAI Responses OpenAIProvider OpenAIResponsesModel - OpenAIResponsesSession.previous_response_id + Live previous_response_id chaining plus neutral transcript state native: ask OpenAI directly for JSON-schema output. Anthropic Messages AnthropicProvider AnthropicMessagesModel - system plus full messages transcript + Live system/messages plus neutral transcript state tool: use the harness-created final_result tool because Anthropic native JSON-schema output is not supported here. OpenRouter chat completions OpenRouterProvider OpenRouterModel - full chat messages transcript + Live chat messages plus neutral transcript state tool by default; explicit native mode is passed through as OpenRouter response_format. @@ -655,7 +655,7 @@

Implementation Deep Dive

RunContext.pause_for_approval() - Captures provider resume state, pending tool batch, usage, responses, tool records, emitted limit-warning keys, metadata, and any background-task cancellations before emitting RunCompletedEvent. + Captures provider resume payload, pending tool batch, usage, responses, tool records, emitted limit-warning keys, metadata, and any background-task cancellations before emitting RunCompletedEvent. Harness.resume_approvals() @@ -675,7 +675,7 @@

Implementation Deep Dive

The paused batch counts against usage.tool_calls at pause time and is not counted again during resume. The post-resume result contains the whole logical run history, not only the second half of the run. - This is why approval envelopes are larger than plain provider resume state: they carry prior responses and + This is why approval envelopes are larger than plain resume state: they carry provider transcript state, prior responses, and accounting as well as the provider checkpoint.

@@ -898,13 +898,13 @@

Implementation Deep Dive

Resume state - Provider-specific resume state copied into HarnessResult.resume_state while building the final result. - OpenAI stores a previous_response_id; Anthropic and OpenRouter store their message transcript plus provider metadata. Callers can store and pass it back, but should not edit or construct it. + Provider-agnostic transcript state copied into HarnessResult.resume_state while building the final result. + Built-in providers emit kind="transcript", version=2, origin diagnostics, and neutral user/assistant/tool entries. Callers can store and pass it back, but should not edit or construct it. Approval pause state Harness-level approval_pause envelope copied into HarnessResult.resume_state when stop_reason="approval_required". - Wraps provider state plus pending batch, run history, usage, emitted limit-warning keys, metadata, and background cancellations. It must be resumed with resume_approvals(), not resume_from. + Wraps provider transcript state plus pending batch, run history, usage, emitted limit-warning keys, metadata, and background cancellations. It must be resumed with resume_approvals(), not resume_from. diff --git a/tests/test_approvals.py b/tests/test_approvals.py index f8c37ad..47797ce 100644 --- a/tests/test_approvals.py +++ b/tests/test_approvals.py @@ -597,9 +597,12 @@ async def test_openai_approval_pause_round_trips_provider_state(tmp_path: Path) assert result.text == "done" assert called == [{"path": "hello.txt"}] - assert paused.resume_state["provider_state"] == {"kind": "openai", "version": 1, "model": "test-model", "previous_response_id": "resp_1"} - assert client.payloads[1]["previous_response_id"] == "resp_1" - assert client.payloads[1]["input"][0] == { + assert paused.resume_state["provider_state"]["kind"] == "transcript" + assert paused.resume_state["provider_state"]["version"] == 2 + assert [entry["role"] for entry in paused.resume_state["provider_state"]["entries"]] == ["user", "assistant"] + assert "previous_response_id" not in client.payloads[1] + assert [item["type"] for item in client.payloads[1]["input"]] == ["message", "function_call", "function_call_output"] + assert client.payloads[1]["input"][-1] == { "type": "function_call_output", "call_id": "call_1", "output": result.tool_call_records[-1]["output"], @@ -734,6 +737,30 @@ async def test_approval_resume_labels_inner_provider_state_errors(tmp_path: Path await harness.resume_approvals(state, [ApprovalDecision(call_id="call_1", approved=True)]) +async def test_approval_resume_labels_builtin_provider_state_errors(tmp_path: Path) -> None: + client = FakeClient() + model = _fake_openai(client) + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=model, + tools=[ + ToolSpec( + "read", + "Read something.", + {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}, + lambda _args: "read-ok", + requires_approval=True, + ) + ], + ) + paused = await harness.run("read") + state = json.loads(json.dumps(paused.resume_state)) + state["provider_state"]["version"] = 1 + + with pytest.raises(HarnessError, match="approval state provider_state version 1 is not supported"): + await harness.resume_approvals(state, [ApprovalDecision(call_id="call_1", approved=True)]) + + async def test_limit_warning_dedup_keys_survive_approval_round_trip(tmp_path: Path) -> None: first = ScriptedSession( start_turn=ModelTurn( diff --git a/tests/test_providers.py b/tests/test_providers.py index 484730f..3b7cd52 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -103,7 +103,13 @@ async def test_openai_appends_notices_to_string_and_tool_inputs() -> None: notices=[notice], ) await session.continue_with_user_message("fix this", instructions="system", tools=[], notices=[notice]) - resumed = model.resume_session({"kind": "openai", "version": 1, "model": "gpt-test", "previous_response_id": "resp_existing"}) + resumed = model.resume_session({ + "kind": "transcript", + "version": 2, + "origin_provider": "openai", + "origin_model": "gpt-test", + "entries": [{"role": "user", "content": "prior", "notice": False}], + }) await resumed.continue_with_user_prompt("follow-up", instructions="system", tools=[], notices=[notice]) assert client.payloads[0]["input"].endswith("\nFinal request.\n") @@ -116,8 +122,19 @@ async def test_openai_appends_notices_to_string_and_tool_inputs() -> None: assert client.payloads[2]["input"] == f"fix this\n\n{_notice_text()}" assert client.payloads[1]["instructions"] == "system" assert client.payloads[2]["instructions"] == "system" - assert client.payloads[3]["input"] == f"follow-up\n\n{_notice_text()}" - assert client.payloads[3]["previous_response_id"] == "resp_existing" + assert client.payloads[3]["input"] == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "prior"}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": f"follow-up\n\n{_notice_text()}"}], + }, + ] + assert "previous_response_id" not in client.payloads[3] async def test_openai_no_notice_payloads_are_unchanged() -> None: client = FakeClient() @@ -176,6 +193,77 @@ async def test_openrouter_appends_notices_to_messages() -> None: assert provider.payloads[2]["messages"][-1]["content"] == f"fix this\n\n{_notice_text()}" assert provider.payloads[3]["messages"][-1]["content"] == f"follow-up\n\n{_notice_text()}" +async def test_resume_replays_preserved_tool_notices() -> None: + tools = [{"type": "function", "name": "echo", "description": "Echo", "parameters": {"type": "object", "properties": {}}}] + notice = _notice() + + anthropic_provider = FakeAnthropicProvider() + anthropic_session = AnthropicMessagesModel("claude-test", provider=anthropic_provider).new_session() + anthropic_first = await anthropic_session.start(prompt="hi", instructions="system", tools=tools) + await anthropic_session.continue_with_tools([ToolOutput(anthropic_first.tool_calls[0].id, "ok")], tools=tools, notices=[notice]) + anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) + assert anthropic_state == json.loads(json.dumps(anthropic_state)) + anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_provider).resume_session(anthropic_state) + await anthropic_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert anthropic_provider.payloads[2]["messages"][2]["content"][-1] == {"type": "text", "text": _notice_text()} + + openai_capture = FakeClient() + openai_session = OpenAIResponsesModel("gpt-test", provider=openai_capture).new_session() + openai_first = await openai_session.start(prompt="hi", instructions="system", tools=tools) + await openai_session.continue_with_tools([ToolOutput(openai_first.tool_calls[0].id, "ok")], instructions="system", tools=tools, notices=[notice]) + openai_state = json.loads(json.dumps(openai_session.dump_state())) + openai_replay = FakeClient() + openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) + await openai_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": _notice_text()}], + } in openai_replay.payloads[0]["input"] + + openrouter_provider = FakeOpenRouterProvider() + openrouter_session = OpenRouterModel("openai/test", provider=openrouter_provider).new_session() + openrouter_first = await openrouter_session.start(prompt="hi", instructions="system", tools=tools) + await openrouter_session.continue_with_tools([ToolOutput(openrouter_first.tool_calls[0].id, "ok")], tools=tools, notices=[notice]) + openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) + openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_provider).resume_session(openrouter_state) + await openrouter_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert {"role": "user", "content": _notice_text()} in openrouter_provider.payloads[2]["messages"] + +async def test_resume_replays_preserved_user_notices() -> None: + tools = [{"type": "function", "name": "echo", "description": "Echo", "parameters": {"type": "object", "properties": {}}}] + notice = _notice() + + anthropic_capture = FakeAnthropicProvider() + anthropic_session = AnthropicMessagesModel("claude-test", provider=anthropic_capture).new_session() + anthropic_first = await anthropic_session.start(prompt="hi", instructions="system", tools=tools, notices=[notice]) + await anthropic_session.continue_with_tools([ToolOutput(anthropic_first.tool_calls[0].id, "ok")], tools=tools) + anthropic_state = json.loads(json.dumps(anthropic_session.dump_state())) + anthropic_replay = FakeAnthropicProvider() + anthropic_resumed = AnthropicMessagesModel("claude-test", provider=anthropic_replay).resume_session(anthropic_state) + await anthropic_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert anthropic_replay.payloads[0]["messages"][0]["content"] == f"hi\n\n{_notice_text()}" + + openai_capture = FakeClient() + openai_session = OpenAIResponsesModel("gpt-test", provider=openai_capture).new_session() + openai_first = await openai_session.start(prompt="hi", instructions="system", tools=tools, notices=[notice]) + await openai_session.continue_with_tools([ToolOutput(openai_first.tool_calls[0].id, "ok")], instructions="system", tools=tools) + openai_state = json.loads(json.dumps(openai_session.dump_state())) + openai_replay = FakeClient() + openai_resumed = OpenAIResponsesModel("gpt-test", provider=openai_replay).resume_session(openai_state) + await openai_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert openai_replay.payloads[0]["input"][0]["content"][0]["text"] == f"hi\n\n{_notice_text()}" + + openrouter_capture = FakeOpenRouterProvider() + openrouter_session = OpenRouterModel("openai/test", provider=openrouter_capture).new_session() + openrouter_first = await openrouter_session.start(prompt="hi", instructions="system", tools=tools, notices=[notice]) + await openrouter_session.continue_with_tools([ToolOutput(openrouter_first.tool_calls[0].id, "ok")], tools=tools) + openrouter_state = json.loads(json.dumps(openrouter_session.dump_state())) + openrouter_replay = FakeOpenRouterProvider() + openrouter_resumed = OpenRouterModel("openai/test", provider=openrouter_replay).resume_session(openrouter_state) + await openrouter_resumed.continue_with_user_prompt("next", instructions="system", tools=tools) + assert openrouter_replay.payloads[0]["messages"][1]["content"] == f"hi\n\n{_notice_text()}" + def test_openai_native_structured_output_overrides_extra_body_text() -> None: model = OpenAIResponsesModel( "gpt-test", diff --git a/tests/test_resume.py b/tests/test_resume.py index 31d0929..8fc9827 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -27,7 +27,33 @@ from thinharness.providers import ModelSession, ProviderError -async def test_openai_resume_uses_previous_response_id_only_for_followup(tmp_path: Path) -> None: +class _TerminalOpenAIProvider(OpenAIProvider): + def __init__(self) -> None: + super().__init__(api_key="fake") + self.payloads = [] + + async def create_response(self, payload): + self.payloads.append(payload) + return {"id": f"resp_{len(self.payloads)}", "output_text": "done"} + + +class _MultiToolAnthropicProvider(FakeAnthropicProvider): + async def create_message(self, payload): + """Capture payloads and request two echo tool calls on the first user turn.""" + self.payloads.append(json.loads(json.dumps(payload))) + last = payload["messages"][-1] + if isinstance(last["content"], str): + return { + "content": [ + {"type": "tool_use", "id": "toolu_a", "name": "echo", "input": {"value": "a"}}, + {"type": "tool_use", "id": "toolu_b", "name": "echo", "input": {"value": "b"}}, + ], + "stop_reason": "tool_use", + } + return {"content": [{"type": "text", "text": "done"}], "stop_reason": "end_turn"} + + +async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path) -> None: (tmp_path / "hello.txt").write_text("hello", encoding="utf-8") client = FakeClient() harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=["read"]), model=OpenAIResponsesModel("gpt-test", provider=client)) @@ -36,14 +62,26 @@ async def test_openai_resume_uses_previous_response_id_only_for_followup(tmp_pat state = json.loads(json.dumps(first.resume_state)) second = await harness.run("follow-up", resume_from=state) - assert first.resume_state == {"kind": "openai", "version": 1, "model": "gpt-test", "previous_response_id": "resp_2"} + assert first.resume_state["kind"] == "transcript" + assert first.resume_state["version"] == 2 + assert first.resume_state["origin_provider"] == "openai" + assert first.resume_state["origin_model"] == "gpt-test" + assert [entry["role"] for entry in first.resume_state["entries"]] == ["user", "assistant", "tool", "assistant"] assert second.text == "done" assert client.payloads[1]["previous_response_id"] == "resp_1" assert client.payloads[1]["instructions"] == harness.system_instructions() - assert client.payloads[2]["input"] == "follow-up" - assert client.payloads[2]["previous_response_id"] == "resp_2" + assert "previous_response_id" not in client.payloads[2] + assert [item["type"] for item in client.payloads[2]["input"]] == [ + "message", + "function_call", + "function_call_output", + "message", + "message", + ] + assert client.payloads[2]["input"][0]["content"][0]["text"] == "first" + assert client.payloads[2]["input"][-1]["content"][0]["text"] == "follow-up" assert client.payloads[2]["instructions"] == harness.system_instructions() - assert "first" not in json.dumps(client.payloads[2]) + assert "first" in json.dumps(client.payloads[2]) async def test_anthropic_resume_replays_transcript_and_appends_new_user_turn(tmp_path: Path) -> None: @@ -83,25 +121,129 @@ async def test_openrouter_resume_replays_transcript_and_appends_new_user_turn(tm assert assistant_tool_call["type"] == "function" assert tool_message["role"] == "tool" assert tool_message["tool_call_id"] == assistant_tool_call["id"] + assert provider.payloads[2]["messages"][4] == {"role": "assistant", "content": "done"} + + +async def test_cross_provider_resume_after_tool_round_trip(tmp_path: Path) -> None: + source_provider = FakeAnthropicProvider() + source = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) + state = json.loads(json.dumps((await source.run("first")).resume_state)) + + openai_provider = _TerminalOpenAIProvider() + openai_result = await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=OpenAIResponsesModel("gpt-test", provider=openai_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=state) + openai_input = openai_provider.payloads[0]["input"] + assert openai_result.text == "done" + assert [item["type"] for item in openai_input[:3]] == ["message", "function_call", "function_call_output"] + assert openai_input[1]["call_id"] == "toolu_1" + assert openai_input[2]["call_id"] == "toolu_1" + + openrouter_provider = FakeOpenRouterProvider() + openrouter_result = await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=OpenRouterModel("openai/test", provider=openrouter_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=state) + replay = openrouter_provider.payloads[0]["messages"] + assert openrouter_result.text == "done" + assert replay[2]["tool_calls"][0]["id"] == "toolu_1" + assert replay[3]["tool_call_id"] == "toolu_1" + + +async def test_multi_tool_batch_replay_shapes(tmp_path: Path) -> None: + source_provider = _MultiToolAnthropicProvider() + source = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=source_provider), tools=[echo_tool()]) + state = json.loads(json.dumps((await source.run("first")).resume_state)) + assert json.loads(json.dumps(state)) == state + + anthropic_provider = FakeAnthropicProvider() + await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=AnthropicMessagesModel("claude-test", provider=anthropic_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=state) + tool_result_blocks = anthropic_provider.payloads[0]["messages"][2]["content"] + assert [block["type"] for block in tool_result_blocks] == ["tool_result", "tool_result"] + assert [block["tool_use_id"] for block in tool_result_blocks] == ["toolu_a", "toolu_b"] + + openrouter_provider = FakeOpenRouterProvider() + await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=OpenRouterModel("openai/test", provider=openrouter_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=state) + roles = [message["role"] for message in openrouter_provider.payloads[0]["messages"]] + assert roles[:6] == ["system", "user", "assistant", "tool", "tool", "assistant"] + + +async def test_resume_rederives_live_system_prompt(tmp_path: Path) -> None: + anthropic_source = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()), + tools=[echo_tool()], + ) + anthropic_state = json.loads(json.dumps((await anthropic_source.run("first")).resume_state)) + anthropic_provider = FakeAnthropicProvider() + await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + model=AnthropicMessagesModel("claude-test", provider=anthropic_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=anthropic_state) + assert anthropic_provider.payloads[0]["system"].startswith("new system") + assert "old system" not in anthropic_provider.payloads[0]["system"] + + openrouter_source = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + model=OpenRouterModel("openai/test", provider=FakeOpenRouterProvider()), + tools=[echo_tool()], + ) + openrouter_state = json.loads(json.dumps((await openrouter_source.run("first")).resume_state)) + openrouter_provider = FakeOpenRouterProvider() + await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + model=OpenRouterModel("openai/test", provider=openrouter_provider), + tools=[echo_tool()], + ).run("follow-up", resume_from=openrouter_state) + assert openrouter_provider.payloads[0]["messages"][0]["content"].startswith("new system") + assert "old system" not in openrouter_provider.payloads[0]["messages"][0]["content"] + + openai_source = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="old system"), + model=OpenAIResponsesModel("gpt-test", provider=_TerminalOpenAIProvider()), + ) + openai_state = json.loads(json.dumps((await openai_source.run("first")).resume_state)) + openai_provider = _TerminalOpenAIProvider() + await Harness( + HarnessConfig(root=tmp_path, builtin_tools=[], system_prompt="new system"), + model=OpenAIResponsesModel("gpt-test", provider=openai_provider), + ).run("follow-up", resume_from=openai_state) + assert openai_provider.payloads[0]["instructions"].startswith("new system") + assert "old system" not in openai_provider.payloads[0]["instructions"] + + +def test_resume_allows_provider_model_mismatches_and_rejects_bad_versions_and_keys(tmp_path: Path) -> None: + class NoToolOpenAIProvider(OpenAIProvider): + def __init__(self) -> None: + super().__init__(api_key="fake") + async def create_response(self, payload): + return {"id": "resp_text", "output_text": "done"} -def test_resume_rejects_provider_model_version_and_unknown_key_mismatches(tmp_path: Path) -> None: - def openai_harness() -> Harness: + def openai_harness(model_name: str = "gpt-test") -> Harness: """Create a fresh OpenAI resume harness.""" - return Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=FakeClient())) + return Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel(model_name, provider=NoToolOpenAIProvider())) state = openai_harness().run_sync("first").resume_state anthropic = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider())) - with pytest.raises(HarnessError, match="resume_from kind"): - anthropic.run_sync("follow-up", resume_from=state) - - wrong_model = {**state, "model": "other"} - with pytest.raises(HarnessError, match="resume_from model"): - openai_harness().run_sync("follow-up", resume_from=wrong_model) + assert anthropic.run_sync("follow-up", resume_from=state).text == "done" + assert openai_harness("other").run_sync("follow-up", resume_from=state).text == "done" - wrong_version = {**state, "version": 2} - with pytest.raises(HarnessError, match="resume_from version 2 is not supported"): + wrong_version = {**state, "version": 1} + with pytest.raises(HarnessError, match="resume_from version 1 is not supported"): openai_harness().run_sync("follow-up", resume_from=wrong_version) missing_version = {key: value for key, value in state.items() if key != "version"} @@ -112,6 +254,9 @@ def openai_harness() -> Harness: with pytest.raises(HarnessError, match="unknown keys.*foo"): openai_harness().run_sync("follow-up", resume_from=unknown) + with pytest.raises(HarnessError, match="resume_from kind 'openai' is not supported"): + openai_harness().run_sync("follow-up", resume_from={"kind": "openai", "version": 1, "model": "gpt-test", "previous_response_id": "resp_1"}) + def test_resume_rejects_malformed_shapes_before_hooks_fire(tmp_path: Path) -> None: events: list[str] = [] @@ -129,26 +274,42 @@ def harness() -> Harness: with pytest.raises(HarnessError, match="resume_from must be a dict"): harness().run_sync("follow-up", resume_from="resp_abc") # type: ignore[arg-type] - with pytest.raises(HarnessError, match="resume_from kind None does not match 'anthropic'"): - harness().run_sync("follow-up", resume_from={"version": 1, "model": "claude-test", "system": "", "messages": []}) - with pytest.raises(HarnessError, match="field 'previous_response_id' has wrong type"): - Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=FakeClient())).run_sync( + base_state = {"kind": "transcript", "version": 2, "origin_provider": "anthropic", "origin_model": "claude-test"} + with pytest.raises(HarnessError, match="resume_from kind None is not supported"): + harness().run_sync("follow-up", resume_from={"version": 2, "origin_provider": "anthropic", "origin_model": "claude-test", "entries": []}) + with pytest.raises(HarnessError, match="missing required field: 'entries'"): + harness().run_sync("follow-up", resume_from=base_state) + with pytest.raises(HarnessError, match="field 'entries' has wrong type"): + harness().run_sync("follow-up", resume_from={**base_state, "entries": "bad"}) + with pytest.raises(HarnessError, match="entry 'user' has wrong keys"): + harness().run_sync("follow-up", resume_from={**base_state, "entries": [{"role": "user", "content": "hi"}]}) + with pytest.raises(HarnessError, match="assistant tool call has wrong type"): + harness().run_sync( "follow-up", - resume_from={"kind": "openai", "version": 1, "model": "gpt-test", "previous_response_id": 123}, + resume_from={**base_state, "entries": [{"role": "assistant", "text": "", "tool_calls": [{"id": 1, "name": "x", "arguments": "{}"}]}]}, ) - with pytest.raises(HarnessError, match="missing required field: 'system'"): - harness().run_sync("follow-up", resume_from={"kind": "anthropic", "version": 1, "model": "claude-test"}) - with pytest.raises(HarnessError, match="field 'messages' has wrong type"): - harness().run_sync("follow-up", resume_from={"kind": "anthropic", "version": 1, "model": "claude-test", "system": "", "messages": ["bad"]}) with pytest.raises(HarnessError, match="JSON-serializable"): harness().run_sync( "follow-up", - resume_from={"kind": "anthropic", "version": 1, "model": "claude-test", "system": "", "messages": [{"bad": datetime.now()}]}, + resume_from={**base_state, "entries": [{"role": "user", "content": datetime.now(), "notice": False}]}, ) assert events == [] +def test_anthropic_resume_rejects_non_json_tool_arguments() -> None: + model = AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()) + + with pytest.raises(HarnessError, match="resume_from assistant tool call arguments must be JSON"): + model.resume_session({ + "kind": "transcript", + "version": 2, + "origin_provider": "openrouter", + "origin_model": "openai/test", + "entries": [{"role": "assistant", "text": "", "tool_calls": [{"id": "call_1", "name": "echo", "arguments": "{bad"}]}], + }) + + def test_structured_output_final_result_omits_resume_state(tmp_path: Path) -> None: class Answer(BaseModel): value: str @@ -245,17 +406,29 @@ def add_context(ctx) -> None: assert resumed_session.notice_calls[0][1][0].content == "Final request: produce the answer now; do not request tools." -def test_no_openai_response_id_omits_resume_state(tmp_path: Path) -> None: +def test_no_openai_response_id_still_produces_resume_state(tmp_path: Path) -> None: class NoIdProvider(OpenAIProvider): def __init__(self) -> None: super().__init__(api_key="fake") + self.payloads = [] async def create_response(self, payload): + self.payloads.append(payload) return {"output_text": "done"} - harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=NoIdProvider())) + provider = NoIdProvider() + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=provider)) - assert harness.run_sync("first").resume_state is None + first = harness.run_sync("first") + assert first.resume_state is not None + assert first.resume_state["kind"] == "transcript" + resumed = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=OpenAIResponsesModel("gpt-test", provider=provider)).run_sync( + "follow-up", + resume_from=json.loads(json.dumps(first.resume_state)), + ) + assert resumed.text == "done" + assert "previous_response_id" not in provider.payloads[1] + assert [item["type"] for item in provider.payloads[1]["input"]] == ["message", "message", "message"] def test_non_clean_exits_omit_resume_state(tmp_path: Path) -> None: @@ -365,10 +538,10 @@ async def test_resume_state_is_detached_outbound_and_inbound(tmp_path: Path) -> first = await harness.run("first") stashed = json.loads(json.dumps(first.resume_state)) - first.resume_state["messages"][1]["content"] = "mutated" + first.resume_state["entries"][0]["content"] = "mutated" inbound = json.loads(json.dumps(stashed)) result = await harness.run("follow-up", resume_from=inbound) - inbound["messages"][1]["content"] = "mutated after call" + inbound["entries"][0]["content"] = "mutated after call" assert result.text == "done" assert provider.payloads[2]["messages"][1]["content"] == "first" @@ -393,7 +566,13 @@ def test_adapter_validation_does_not_mutate_state_on_failure() -> None: model = AnthropicMessagesModel("claude-test", provider=FakeAnthropicProvider()) with pytest.raises(HarnessError): - model.resume_session({"kind": "anthropic", "version": 1, "model": "claude-test", "system": "", "messages": ["bad"]}) + model.resume_session({ + "kind": "transcript", + "version": 2, + "origin_provider": "anthropic", + "origin_model": "claude-test", + "entries": [{"role": "user", "content": "missing notice flag"}], + }) session = model.new_session() assert session.messages == [] diff --git a/thinharness/approvals.py b/thinharness/approvals.py index 396d292..ea8b454 100644 --- a/thinharness/approvals.py +++ b/thinharness/approvals.py @@ -40,7 +40,11 @@ class ApprovalToolCall: @dataclass(frozen=True) class ApprovalPause: - """Validated approval pause envelope data.""" + """Validated approval pause envelope data. + + provider_state is the provider resume payload. Built-in providers store the + full neutral transcript there; the approval envelope version is independent. + """ provider_state: Json batch: list[ApprovalToolCall] @@ -68,8 +72,11 @@ def build_approval_envelope( metadata: Json, ) -> Json: """Build an isolated JSON approval pause envelope.""" + # Built-in provider_state contains the full neutral transcript. The outer + # envelope version is unchanged because only the nested provider payload + # changed; old nested payloads fail later when the provider resumes them. # Raw provider responses make the post-resume result a full logical-run result. - # They also make approval envelopes grow with run length; host docs call this out. + # Both provider_state and responses make approval envelopes grow with run length. envelope: Json = { "kind": APPROVAL_ENVELOPE_KIND, "version": APPROVAL_ENVELOPE_VERSION, diff --git a/thinharness/providers.py b/thinharness/providers.py index 8c925ca..8865be9 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -38,6 +38,33 @@ class ModelTurn: finalized_output_mode: str | None = None +@dataclass +class AssistantEntry: + """Provider-neutral assistant transcript entry.""" + + text: str + tool_calls: list[ModelToolCall] + + +@dataclass +class UserEntry: + """Provider-neutral user transcript entry.""" + + content: str + notice: bool = False + + +@dataclass +class ToolResultEntry: + """Provider-neutral tool-result transcript entry.""" + + call_id: str + output: str + + +TranscriptEntry = AssistantEntry | UserEntry | ToolResultEntry + + @dataclass class ToolOutput: """A normalized local tool output.""" @@ -181,37 +208,113 @@ def __init__(self, message: str, *, status_code: int | None = None) -> None: self.status_code = status_code -_BASE_RESUME_KEYS = frozenset({"kind", "version", "model"}) +_TRANSCRIPT_RESUME_KEYS = frozenset({"kind", "version", "origin_provider", "origin_model", "entries"}) +_TRANSCRIPT_ENTRY_KEYS = { + "assistant": frozenset({"role", "text", "tool_calls"}), + "user": frozenset({"role", "content", "notice"}), + "tool": frozenset({"role", "call_id", "output"}), +} -def _validate_resume_state( - state: dict[str, Any], - *, - expected_kind: str, - expected_model: str, - required_fields: dict[str, type | tuple[type, ...]], -) -> None: - """Validate resume state shape before any session mutation.""" +def _validate_resume_state(state: dict[str, Any]) -> list[TranscriptEntry]: + """Validate built-in provider resume state shape before any session mutation.""" if not isinstance(state, dict): raise HarnessError("resume_from must be a dict") - if state.get("kind") != expected_kind: - raise HarnessError(f"resume_from kind {state.get('kind')!r} does not match {expected_kind!r}") - if state.get("version") != 1: + try: + json.dumps(state) + except (TypeError, ValueError) as exc: + raise HarnessError("resume_from must be JSON-serializable") from exc + if state.get("kind") != "transcript": + raise HarnessError(f"resume_from kind {state.get('kind')!r} is not supported; regenerate resume_state") + if state.get("version") != 2: raise HarnessError(f"resume_from version {state.get('version')!r} is not supported") - if state.get("model") != expected_model: - raise HarnessError(f"resume_from model {state.get('model')!r} does not match current model {expected_model!r}") - for field_name, expected_type in required_fields.items(): + unknown = set(state) - _TRANSCRIPT_RESUME_KEYS + if unknown: + raise HarnessError(f"resume_from has unknown keys: {sorted(unknown)!r}") + for field_name, expected_type in {"origin_provider": str, "origin_model": str, "entries": list}.items(): if field_name not in state: raise HarnessError(f"resume_from missing required field: {field_name!r}") if not isinstance(state[field_name], expected_type): raise HarnessError(f"resume_from field {field_name!r} has wrong type") - unknown = set(state) - (_BASE_RESUME_KEYS | required_fields.keys()) - if unknown: - raise HarnessError(f"resume_from has unknown keys: {sorted(unknown)!r}") - try: - json.dumps(state) - except (TypeError, ValueError) as exc: - raise HarnessError("resume_from must be JSON-serializable") from exc + return [_transcript_entry_from_dict(entry) for entry in state["entries"]] + + +def _transcript_state(*, model: Model, entries: list[TranscriptEntry]) -> dict[str, Any]: + """Return the neutral transcript resume envelope.""" + return { + "kind": "transcript", + "version": 2, + "origin_provider": provider_prefix(model.provider.name), + "origin_model": model.model, + "entries": [_transcript_entry_to_dict(entry) for entry in entries], + } + + +def _transcript_entry_to_dict(entry: TranscriptEntry) -> Json: + if isinstance(entry, UserEntry): + return {"role": "user", "content": entry.content, "notice": entry.notice} + if isinstance(entry, ToolResultEntry): + return {"role": "tool", "call_id": entry.call_id, "output": entry.output} + return { + "role": "assistant", + "text": entry.text, + "tool_calls": [ + {"id": call.id, "name": call.name, "arguments": call.arguments} + for call in entry.tool_calls + ], + } + + +def _transcript_entry_from_dict(value: Any) -> TranscriptEntry: + if not isinstance(value, dict): + raise HarnessError("resume_from entries must be dicts") + role = value.get("role") + if role not in _TRANSCRIPT_ENTRY_KEYS: + raise HarnessError(f"resume_from entry role {role!r} is not supported") + if set(value) != _TRANSCRIPT_ENTRY_KEYS[role]: + raise HarnessError(f"resume_from entry {role!r} has wrong keys") + if role == "user": + if not isinstance(value["content"], str) or type(value["notice"]) is not bool: + raise HarnessError("resume_from user entry has wrong type") + return UserEntry(content=value["content"], notice=value["notice"]) + if role == "tool": + if not isinstance(value["call_id"], str) or not isinstance(value["output"], str): + raise HarnessError("resume_from tool entry has wrong type") + return ToolResultEntry(call_id=value["call_id"], output=value["output"]) + if not isinstance(value["text"], str) or not isinstance(value["tool_calls"], list): + raise HarnessError("resume_from assistant entry has wrong type") + return AssistantEntry( + text=value["text"], + tool_calls=[_model_tool_call_from_dict(call) for call in value["tool_calls"]], + ) + + +def _model_tool_call_from_dict(value: Any) -> ModelToolCall: + if not isinstance(value, dict) or set(value) != {"id", "name", "arguments"}: + raise HarnessError("resume_from assistant tool call has wrong shape") + if not isinstance(value["id"], str) or not isinstance(value["name"], str) or not isinstance(value["arguments"], str): + raise HarnessError("resume_from assistant tool call has wrong type") + return ModelToolCall(id=value["id"], name=value["name"], arguments=value["arguments"]) + + +def _append_tool_results(transcript: list[TranscriptEntry], outputs: list[ToolOutput], notice_text: str) -> None: + transcript.extend(ToolResultEntry(call_id=output.call_id, output=output.output) for output in outputs) + if notice_text: + transcript.append(UserEntry(content=notice_text, notice=True)) + + +def _append_assistant_turn(transcript: list[TranscriptEntry], turn: ModelTurn) -> None: + transcript.append(AssistantEntry(text=turn.text, tool_calls=copy.deepcopy(turn.tool_calls))) + + +def _validate_anthropic_tool_arguments(entries: list[TranscriptEntry]) -> None: + for entry in entries: + if isinstance(entry, AssistantEntry): + for call in entry.tool_calls: + try: + json.loads(call.arguments) + except ValueError as exc: + raise HarnessError("resume_from assistant tool call arguments must be JSON for Anthropic resume") from exc # ============================================================================= @@ -381,16 +484,10 @@ def new_session(self) -> ModelSession: def resume_session(self, state: dict[str, Any]) -> ModelSession: """Create an isolated Responses API session from resume state.""" - _validate_resume_state( - state, - expected_kind=self.resume_kind, - expected_model=self.model, - required_fields={"previous_response_id": str}, - ) - if not state["previous_response_id"]: - raise HarnessError("resume_from field 'previous_response_id' must be non-empty") + entries = _validate_resume_state(state) session = OpenAIResponsesSession(self) - session.previous_response_id = state["previous_response_id"] + session.transcript = copy.deepcopy(entries) + session._pending_replay = copy.deepcopy(entries) return session def build_payload( @@ -422,6 +519,8 @@ class OpenAIResponsesSession: def __init__(self, model: OpenAIResponsesModel) -> None: self.model = model self.previous_response_id: str | None = None + self.transcript: list[TranscriptEntry] = [] + self._pending_replay: list[TranscriptEntry] | None = None async def start( self, @@ -436,8 +535,10 @@ async def start( ) -> ModelTurn: """Start a Responses API run.""" self.previous_response_id = previous_response_id + input_text = append_notices_to_text(prompt, notices) + self.transcript = [UserEntry(content=input_text)] payload = self.model.build_payload( - input_payload=append_notices_to_text(prompt, notices), + input_payload=input_text, instructions=instructions, tools=tools, metadata=metadata, @@ -469,8 +570,10 @@ async def continue_with_tools( "role": "user", "content": [{"type": "input_text", "text": notice_text}], }) + _append_tool_results(self.transcript, outputs, notice_text) + replay_input = self._prepend_replay(input_payload) payload = self.model.build_payload( - input_payload=input_payload, + input_payload=replay_input, instructions=instructions, tools=tools, metadata=metadata, @@ -491,8 +594,10 @@ async def continue_with_user_message( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue a Responses API run with a corrective user message.""" + input_text = append_notices_to_text(message, notices) + self.transcript.append(UserEntry(content=input_text)) payload = self.model.build_payload( - input_payload=append_notices_to_text(message, notices), + input_payload=self._prepend_replay(input_text), instructions=instructions, tools=tools, metadata=metadata, @@ -513,8 +618,10 @@ async def continue_with_user_prompt( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue a resumed Responses API run with a new user prompt.""" + input_text = append_notices_to_text(prompt, notices) + self.transcript.append(UserEntry(content=input_text)) payload = self.model.build_payload( - input_payload=append_notices_to_text(prompt, notices), + input_payload=self._prepend_replay(input_text), instructions=instructions, tools=tools, metadata=metadata, @@ -525,21 +632,25 @@ async def continue_with_user_prompt( return await self._complete(payload) def dump_state(self) -> dict[str, Any] | None: - """Serialize the latest Responses API continuation token.""" - if not self.previous_response_id: - return None - return { - "kind": self.model.resume_kind, - "version": 1, - "model": self.model.model, - "previous_response_id": self.previous_response_id, - } + """Serialize the neutral transcript for resume.""" + return _transcript_state(model=self.model, entries=self.transcript) async def _complete(self, payload: Json) -> ModelTurn: """Send a Responses API payload and normalize the response.""" response = await self.model.provider.create_response(payload) self.previous_response_id = response.get("id") or self.previous_response_id - return ModelTurn(text=_extract_responses_text(response), tool_calls=_extract_responses_tool_calls(response), raw=response) + turn = ModelTurn(text=_extract_responses_text(response), tool_calls=_extract_responses_tool_calls(response), raw=response) + _append_assistant_turn(self.transcript, turn) + return turn + + def _prepend_replay(self, input_payload: str | list[Json]) -> str | list[Json]: + if self._pending_replay is None: + return input_payload + replay = _render_openai_transcript(self._pending_replay) + self._pending_replay = None + if isinstance(input_payload, str): + return [*replay, _openai_user_item(input_payload)] + return [*replay, *input_payload] class AnthropicMessagesModel: @@ -572,17 +683,11 @@ def new_session(self) -> ModelSession: def resume_session(self, state: dict[str, Any]) -> ModelSession: """Create an isolated Anthropic Messages session from resume state.""" - _validate_resume_state( - state, - expected_kind=self.resume_kind, - expected_model=self.model, - required_fields={"system": (str, list), "messages": list}, - ) - if not all(isinstance(message, dict) for message in state["messages"]): - raise HarnessError("resume_from field 'messages' has wrong type") + entries = _validate_resume_state(state) + _validate_anthropic_tool_arguments(entries) session = AnthropicMessagesSession(self) - session.system = state["system"] - session.messages = copy.deepcopy(state["messages"]) + session.transcript = copy.deepcopy(entries) + session._resume_entries = copy.deepcopy(entries) return session @@ -593,6 +698,8 @@ def __init__(self, model: AnthropicMessagesModel) -> None: self.model = model self.messages: list[Json] = [] self.system = "" + self.transcript: list[TranscriptEntry] = [] + self._resume_entries: list[TranscriptEntry] | None = None async def start( self, @@ -611,7 +718,9 @@ async def start( if previous_response_id: raise ProviderError("previous_response_id is only supported by OpenAI Responses") self.system = instructions - self.messages = [{"role": "user", "content": append_notices_to_text(prompt, notices)}] + content = append_notices_to_text(prompt, notices) + self.messages = [{"role": "user", "content": content}] + self.transcript = [UserEntry(content=content)] return await self._complete(tools=tools, metadata=metadata) async def continue_with_tools( @@ -631,6 +740,8 @@ async def continue_with_tools( notice_text = render_model_notices(notices) if notice_text: content.append({"type": "text", "text": notice_text}) + _append_tool_results(self.transcript, outputs, notice_text) + self._apply_resume(instructions) self.messages.append({ "role": "user", "content": content, @@ -650,7 +761,10 @@ async def continue_with_user_message( """Continue an Anthropic Messages run with a corrective user message.""" if structured_output is not None: raise ProviderError("Anthropic does not support native structured output") - self.messages.append({"role": "user", "content": append_notices_to_text(message, notices)}) + content = append_notices_to_text(message, notices) + self.transcript.append(UserEntry(content=content)) + self._apply_resume(instructions) + self.messages.append({"role": "user", "content": content}) return await self._complete(tools=tools, metadata=metadata) async def continue_with_user_prompt( @@ -666,18 +780,15 @@ async def continue_with_user_prompt( """Continue a resumed Anthropic Messages run with a new user prompt.""" if structured_output is not None: raise ProviderError("Anthropic does not support native structured output") - self.messages.append({"role": "user", "content": append_notices_to_text(prompt, notices)}) + content = append_notices_to_text(prompt, notices) + self.transcript.append(UserEntry(content=content)) + self._apply_resume(instructions) + self.messages.append({"role": "user", "content": content}) return await self._complete(tools=tools, metadata=metadata) def dump_state(self) -> dict[str, Any] | None: - """Serialize the Anthropic transcript for resume.""" - return { - "kind": self.model.resume_kind, - "version": 1, - "model": self.model.model, - "system": self.system, - "messages": copy.deepcopy(self.messages), - } + """Serialize the neutral transcript for resume.""" + return _transcript_state(model=self.model, entries=self.transcript) async def _complete(self, *, tools: list[Json], metadata: Json | None = None) -> ModelTurn: """Send a Messages API request and normalize the response.""" @@ -695,7 +806,16 @@ async def _complete(self, *, tools: list[Json], metadata: Json | None = None) -> payload.update(self.model.settings.extra_body) response = await self.model.provider.create_message(payload) self.messages.append({"role": "assistant", "content": response.get("content", [])}) - return ModelTurn(text=_extract_anthropic_text(response), tool_calls=_extract_anthropic_tool_calls(response), raw=response) + turn = ModelTurn(text=_extract_anthropic_text(response), tool_calls=_extract_anthropic_tool_calls(response), raw=response) + _append_assistant_turn(self.transcript, turn) + return turn + + def _apply_resume(self, instructions: str | None) -> None: + if self._resume_entries is None: + return + self.system = instructions or "" + self.messages = _render_anthropic_transcript(self._resume_entries) + self._resume_entries = None class OpenRouterModel: @@ -725,16 +845,10 @@ def new_session(self) -> ModelSession: def resume_session(self, state: dict[str, Any]) -> ModelSession: """Create an isolated OpenRouter session from resume state.""" - _validate_resume_state( - state, - expected_kind=self.resume_kind, - expected_model=self.model, - required_fields={"messages": list}, - ) - if not all(isinstance(message, dict) for message in state["messages"]): - raise HarnessError("resume_from field 'messages' has wrong type") + entries = _validate_resume_state(state) session = OpenRouterSession(self) - session.messages = copy.deepcopy(state["messages"]) + session.transcript = copy.deepcopy(entries) + session._resume_entries = copy.deepcopy(entries) return session @@ -744,6 +858,8 @@ class OpenRouterSession: def __init__(self, model: OpenRouterModel) -> None: self.model = model self.messages: list[Json] = [] + self.transcript: list[TranscriptEntry] = [] + self._resume_entries: list[TranscriptEntry] | None = None async def start( self, @@ -759,10 +875,12 @@ async def start( """Start an OpenRouter run.""" if previous_response_id: raise ProviderError("previous_response_id is only supported by OpenAI Responses") + content = append_notices_to_text(prompt, notices) self.messages = [ {"role": "system", "content": instructions}, - {"role": "user", "content": append_notices_to_text(prompt, notices)}, + {"role": "user", "content": content}, ] + self.transcript = [UserEntry(content=content)] return await self._complete(tools=tools, metadata=metadata, structured_output=structured_output) async def continue_with_tools( @@ -776,9 +894,11 @@ async def continue_with_tools( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue an OpenRouter run with tool messages.""" + notice_text = render_model_notices(notices) + _append_tool_results(self.transcript, outputs, notice_text) + self._apply_resume(instructions) for output in outputs: self.messages.append({"role": "tool", "tool_call_id": output.call_id, "content": output.output}) - notice_text = render_model_notices(notices) if notice_text: self.messages.append({"role": "user", "content": notice_text}) return await self._complete(tools=tools, metadata=metadata, structured_output=structured_output) @@ -794,7 +914,10 @@ async def continue_with_user_message( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue an OpenRouter run with a corrective user message.""" - self.messages.append({"role": "user", "content": append_notices_to_text(message, notices)}) + content = append_notices_to_text(message, notices) + self.transcript.append(UserEntry(content=content)) + self._apply_resume(instructions) + self.messages.append({"role": "user", "content": content}) return await self._complete(tools=tools, metadata=metadata, structured_output=structured_output) async def continue_with_user_prompt( @@ -808,17 +931,15 @@ async def continue_with_user_prompt( notices: list[ModelNotice] | None = None, ) -> ModelTurn: """Continue a resumed OpenRouter run with a new user prompt.""" - self.messages.append({"role": "user", "content": append_notices_to_text(prompt, notices)}) + content = append_notices_to_text(prompt, notices) + self.transcript.append(UserEntry(content=content)) + self._apply_resume(instructions) + self.messages.append({"role": "user", "content": content}) return await self._complete(tools=tools, metadata=metadata, structured_output=structured_output) def dump_state(self) -> dict[str, Any] | None: - """Serialize the OpenRouter transcript for resume.""" - return { - "kind": self.model.resume_kind, - "version": 1, - "model": self.model.model, - "messages": copy.deepcopy(self.messages), - } + """Serialize the neutral transcript for resume.""" + return _transcript_state(model=self.model, entries=self.transcript) async def _complete( self, @@ -843,7 +964,15 @@ async def _complete( response = await self.model.provider.create_chat_completion(payload) message = ((response.get("choices") or [{}])[0].get("message") or {}) self.messages.append(message) - return ModelTurn(text=str(message.get("content") or ""), tool_calls=_extract_chat_tool_calls(message), raw=response) + turn = ModelTurn(text=str(message.get("content") or ""), tool_calls=_extract_chat_tool_calls(message), raw=response) + _append_assistant_turn(self.transcript, turn) + return turn + + def _apply_resume(self, instructions: str | None) -> None: + if self._resume_entries is None: + return + self.messages = [{"role": "system", "content": instructions or ""}, *_render_openrouter_transcript(self._resume_entries)] + self._resume_entries = None # ============================================================================= @@ -928,6 +1057,99 @@ def append_notices_to_text(text: str, notices: list[ModelNotice] | None) -> str: return text if not notice_text else f"{text}\n\n{notice_text}" +def _render_anthropic_transcript(entries: list[TranscriptEntry]) -> list[Json]: + """Render neutral transcript entries as Anthropic Messages history.""" + messages: list[Json] = [] + index = 0 + while index < len(entries): + entry = entries[index] + if isinstance(entry, UserEntry): + if entry.notice: + messages.append({"role": "user", "content": [{"type": "text", "text": entry.content}]}) + else: + messages.append({"role": "user", "content": entry.content}) + index += 1 + continue + if isinstance(entry, AssistantEntry): + content: list[Json] = [] + if entry.text: + content.append({"type": "text", "text": entry.text}) + content.extend({ + "type": "tool_use", + "id": call.id, + "name": call.name, + "input": json.loads(call.arguments), + } for call in entry.tool_calls) + messages.append({"role": "assistant", "content": content}) + index += 1 + continue + content = [] + while index < len(entries) and isinstance(entries[index], ToolResultEntry): + tool_entry = entries[index] + assert isinstance(tool_entry, ToolResultEntry) + content.append({"type": "tool_result", "tool_use_id": tool_entry.call_id, "content": tool_entry.output}) + index += 1 + if index < len(entries): + notice_entry = entries[index] + if isinstance(notice_entry, UserEntry) and notice_entry.notice: + content.append({"type": "text", "text": notice_entry.content}) + index += 1 + messages.append({"role": "user", "content": content}) + return messages + + +def _render_openrouter_transcript(entries: list[TranscriptEntry]) -> list[Json]: + """Render neutral transcript entries as OpenRouter chat history.""" + messages: list[Json] = [] + for entry in entries: + if isinstance(entry, UserEntry): + messages.append({"role": "user", "content": entry.content}) + elif isinstance(entry, ToolResultEntry): + messages.append({"role": "tool", "tool_call_id": entry.call_id, "content": entry.output}) + else: + message: Json = {"role": "assistant"} + if entry.text: + message["content"] = entry.text + if entry.tool_calls: + message["tool_calls"] = [ + { + "id": call.id, + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + for call in entry.tool_calls + ] + if not entry.text and not entry.tool_calls: + message["content"] = "" + messages.append(message) + return messages + + +def _openai_user_item(text: str) -> Json: + """Render one Responses API user message item.""" + return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} + + +def _render_openai_transcript(entries: list[TranscriptEntry]) -> list[Json]: + """Render neutral transcript entries as Responses API input items.""" + items: list[Json] = [] + for entry in entries: + if isinstance(entry, UserEntry): + items.append(_openai_user_item(entry.content)) + elif isinstance(entry, ToolResultEntry): + items.append({"type": "function_call_output", "call_id": entry.call_id, "output": entry.output}) + else: + if entry.text: + items.append({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": entry.text}]}) + items.extend({ + "type": "function_call", + "call_id": call.id, + "name": call.name, + "arguments": call.arguments, + } for call in entry.tool_calls) + return items + + def _responses_tool_to_anthropic(tool: Json) -> Json: """Convert a Responses API function tool to Anthropic format.""" return { From 707b86c9f155c4c43e426587827dd722323efdd5 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Mon, 22 Jun 2026 21:32:26 -0400 Subject: [PATCH 2/5] refactor: project tracing from transcript deltas --- .plans/29-unified-transcript-log.md | 301 ++++++++++++++++++++++++++++ .plans/30-tracing-consolidation.md | 142 +++++++++++++ .plans/image-inputs.md | 26 +++ docs/behavior.md | 15 ++ docs/docs.md | 2 +- docs/site/about/index.html | 18 +- docs/site/explainer/index.html | 6 +- docs/site/index.html | 2 +- docs/table.md | 38 ++-- tests/test_background_tools.py | 6 +- tests/test_streaming.py | 10 + tests/test_tracing.py | 111 +++++++--- thinharness/events.py | 1 - thinharness/projections.py | 133 ++++++++++++ thinharness/runtime.py | 67 +++++-- thinharness/tracing.py | 127 ++---------- 16 files changed, 811 insertions(+), 194 deletions(-) create mode 100644 .plans/29-unified-transcript-log.md create mode 100644 .plans/30-tracing-consolidation.md create mode 100644 .plans/image-inputs.md create mode 100644 thinharness/projections.py diff --git a/.plans/29-unified-transcript-log.md b/.plans/29-unified-transcript-log.md new file mode 100644 index 0000000..0a7ea59 --- /dev/null +++ b/.plans/29-unified-transcript-log.md @@ -0,0 +1,301 @@ +# Unified neutral transcript (one canonical log) — plan v3 + +Introduce a single provider-agnostic, JSON-serializable conversation transcript as the harness's canonical record of a run, and make `resume_state` (and the approval-pause `provider_state`) a projection of it instead of a per-provider native blob. + +Revised after two multi-review rounds (`.reviews/plans/unified-transcript-log/*-v{1,2}.md`). All reviewers endorsed the core design across both rounds; v2 fixed the v1 correctness bugs (OpenAI deferred replay, system re-injection, Anthropic coalescing, notice preservation, version bump), and v3 resolves the v2 implementation-spec precision items (chiefly: the backing store is explicitly **dual-store / additive**, not a replacement). Findings-resolution tables for both rounds are at the end. + +## Goal + +A run's durable state should have **one representation**, not three. Today the conversation exists as: + +1. Provider-native session state (`AnthropicMessagesSession.messages` in Anthropic block format, `OpenRouterSession.messages` in chat format, `OpenAIResponsesSession.previous_response_id` as a server-side pointer). +2. `HarnessResult.resume_state` — whatever (1) serializes to, tagged by provider `kind`. +3. The live `StreamEvent` queue and OTel spans — separate ephemeral projections. + +This plan collapses (1) and (2) into a neutral `Transcript` that every provider renders to/from. After this change: + +- `resume_state` is provider-agnostic: a run captured on `anthropic:claude-...` can be resumed on `openai:gpt-...` (with documented graceful degradation), and vice versa. +- `resume_state` is self-contained and no longer depends on OpenAI server-side response retention (the ~30-day expiry footgun in `.plans/08-resume.md`). +- The approval-pause envelope's `provider_state` is the same neutral transcript, so both resume paths unify. +- The neutral transcript is positioned to become the single source the event stream and tracing derive from (deferred — see "Out of scope"). + +This is the architecture pydantic-ai, strands, and agno converged on. The cost is bounded because thinharness has only three built-in providers and is already half-neutral on the harness↔provider boundary. + +## What this is and is not + +**Is:** a neutral *transcript / log* — one durable representation of what happened — added *alongside* the existing provider-native in-run request builders, which are unchanged. + +**Is not:** a neutral *capability surface* (LiteLLM-style lowest-common-denominator), and **not** a replacement of the in-run request path. Provider-specific request settings (`ModelSettings.extra_body`, Anthropic `max_tokens`, structured-output modes) keep flowing through unchanged. The transcript records the conversation, not the request knobs, and is consulted only to produce/restore durable state. + +## Current state (file:line) + +The harness↔provider boundary is **already neutral in both directions**: + +- **Out:** every provider returns `ModelTurn` (`providers.py:31` — `text`, `tool_calls: list[ModelToolCall]`, `raw`). The run loop never reads provider-native responses; it appends `turn.raw` to `run_ctx.responses` (`core.py:526`) and reads `turn.text` / `turn.tool_calls`. +- **In:** the loop drives sessions with neutral inputs — `start(prompt=...)`, `continue_with_tools(outputs: list[ToolOutput])`, `continue_with_user_message`, `continue_with_user_prompt`, plus `ModelNotice` / `StructuredOutputRequest` (`providers.py:115-173`). + +What is **not** neutral is only the accumulated transcript each session keeps, appended after each completion: + +- `AnthropicMessagesSession` — Anthropic block format; appends at `providers.py:614` (start user), `:634` (tool_result batch), `:653` (corrective user), `:669` (resumed user prompt), `:697` (assistant turn). Notices are baked into user content via `append_notices_to_text` at `:614/:653/:669` and as a trailing text block in the tool batch at `:632-633`. +- `OpenRouterSession` — chat format; appends at `:762/:780/:797/:811/:845`. Notices baked in at `:764/:797/:811` and a trailing user message at `:781-783`. +- `OpenAIResponsesSession` — no client transcript; only `previous_response_id`, set from each response id (`:541`). Notices appended to input text at `:440/:495/:517` and as a user message item in the tool path at `:467-471`. + +`TurnDriver` can attach `ModelNotice`s to `start`, resumed prompts, corrective messages, and tool outputs (`runtime.py:67/:81/:122`), so notices are model-visible on every entry point, not just tool continuations. + +`dump_state()` (`providers.py:171`) serializes whichever native blob the session kept, tagged with a provider `kind`. That tag is the entire reason resume is provider-bound. Both durable-state consumers funnel through `dump_state()`: + +- `HarnessResult.resume_state` via `run_ctx.finalize(..., require_dump_state=model_supports_resume)` (`core.py:532-538`). +- The approval-pause envelope's `provider_state` (built in `run_ctx.pause_for_approval`, `core.py:577`; restored via `self._resume_approval_session(approval_pause.provider_state)` → `resume_session(...)`, `core.py:465`, `:720`). + +So neutralizing `dump_state`/`resume_session` fixes **both** durable paths. + +The validation/lifecycle rules from `.plans/08-resume.md` (clean-`end_turn`-only, no resume after `final_result`, no transcript repair) stay in force. + +## Design + +### 1. The neutral `Transcript` type (`providers.py`, new leaf types) + +An ordered list of neutral entries. Pure data, JSON-serializable via an explicit to-dict/from-dict mapping with a `role` discriminator (dataclasses are not directly `json.dumps`-able, and the durable path does `json.loads(json.dumps(state))` — `runtime.py`/`approvals.py:87`). + +```python +@dataclass +class AssistantEntry: + text: str + tool_calls: list[ModelToolCall] # reuse existing leaf type + +@dataclass +class UserEntry: + content: str # fully rendered, provider-neutral user text + notice: bool = False # True only for batch-accompanying notice text (grouping hint) + +@dataclass +class ToolResultEntry: + call_id: str + output: str + +TranscriptEntry = AssistantEntry | UserEntry | ToolResultEntry +``` + +**Serialization (load-bearing — specify exactly).** `dump_state` emits plain dicts with a `role` discriminator; the validator/renderers reconstruct from those dicts: + +```python +{"role": "user", "content": "...", "notice": false} +{"role": "assistant", "text": "...", "tool_calls": [{"id": "...", "name": "...", "arguments": "..."}]} +{"role": "tool", "call_id": "...", "output": "..."} +``` + +`ModelToolCall.arguments` stays a JSON string in the transcript. Renderers needing a parsed object (Anthropic `tool_use.input`) call `json.loads(arguments)` (mirroring `_extract_anthropic_tool_calls`'s `json.dumps(input)` at `providers.py:994`). + +**Notice text is captured in `UserEntry.content`, faithfully (no notice is dropped).** Because notices are model-visible on *all* entry points (current state above), v3 records them wherever they appear: + +- `start` / `continue_with_user_prompt` / `continue_with_user_message`: the `UserEntry.content` stores the **notice-appended** text — exactly the value `append_notices_to_text(text, notices)` produces and that current in-run tests pin (`test_providers.py:148/:154-155/:172`) — not the raw prompt. `notice=False` (it is a normal user turn that happens to carry appended guidance). +- `continue_with_tools`: tool results become `ToolResultEntry`s, and any rendered notice text becomes a trailing `UserEntry(notice=True)` so the Anthropic renderer can fold it back into the tool-result user message (role alternation; §3). + +Dropping stale notices from the durable transcript is a separate, intentional behavior change deferred to a follow-up; v3 preserves current behavior. + +Notes: + +- **No `system` in the stream.** The system prompt and tool schemas are the run's "versioned constant" (supplied by `HarnessConfig` each turn via `instructions`/`tools`), not part of the transcript. Resume re-derives them from live config — see §3. +- **v1 stores reasoning as text only.** No opaque provider blobs (Anthropic thinking `signature`, OpenAI `encrypted_content`/reasoning item ids). Same-provider reasoning fidelity is deferred (Out of scope). +- **No per-entry `provider_name` and no `_parse_extras` in v1.** Envelope-level `origin_provider` covers diagnostics; per-entry provenance is only needed by deferred fidelity work. +- **Assistant content ordering is intentionally flattened.** `AssistantEntry` stores concatenated `text` + a separate `tool_calls` list, matching what the harness already sees (`ModelTurn.text` is the concatenation of all text blocks via `_extract_anthropic_text`, `providers.py:998`). A `text → tool_use → text` native turn round-trips as `text + tool_use`. This is consistent with every existing downstream consumer; an accepted, tested degradation, not a regression. (See §2 — it affects only `dump_state`, never the in-run request.) +- `ModelToolCall.id` is the portable correlation key between an `AssistantEntry.tool_calls[i]` and the matching `ToolResultEntry.call_id`. + +### 2. Provider contract: dual store (native in-run builder + parallel neutral transcript) + +**This is Option A and it is explicit: the native in-run request builders are retained; the neutral transcript is purely additive.** Concretely: + +- `AnthropicMessagesSession.messages`/`.system`, `OpenRouterSession.messages`, and `OpenAIResponsesSession.previous_response_id` **stay** and remain the source of every in-run request, byte-for-byte as today (OpenRouter still stores the raw assistant message at `providers.py:845`, preserving any `reasoning`/`reasoning_details` during the live run; OpenAI still chains `previous_response_id` and sends only new items). +- Each session **also** maintains a neutral `Transcript`, appending entries as it already receives/produces them: `start`/`continue_with_user_*` append a `UserEntry` (notice-appended content per §1); `continue_with_tools` appends `ToolResultEntry`s + optional `UserEntry(notice=True)`; `_complete` appends an `AssistantEntry(text, tool_calls)` from the existing `_extract_*` helpers (`providers.py:968-1009`). +- `_render_input(entries, *, instructions)` is invoked **only on the first post-resume turn**, never during a normal in-run turn. It is gated on a per-session resume flag (`_resume_entries` for stateless providers, `_pending_replay` for OpenAI) that is set by `resume_session` and consumed exactly once. + +Consequence (and the implementation guard): **all in-run payload tests must remain byte-identical** — `test_harness.py:226-242`, `test_providers.py:60-74`, and `test_providers.py:92-175` (minus the resume sub-case at `:106-120`). If any in-run payload test changes, the dual-store boundary leaked (Option B crept in) and must be corrected. The assistant-text flattening and any per-turn re-rendering apply to resumed turns only. + +The four `continue_with_*` bodies stay distinct — each renders to its native wire format and gains one neutral-append line. They do not "collapse." + +### 3. Provider-agnostic `dump_state` / `resume_session` lifecycle + +```python +# neutral envelope — same shape for every provider; allowed keys are exactly this set +{"kind": "transcript", "version": 2, + "origin_provider": "anthropic", # normalized model-ref prefix (see below); required + "origin_model": "claude-...", # required + "entries": [ {"role": "user", ...}, {"role": "assistant", ...}, {"role": "tool", ...}, ... ]} +``` + +- `dump_state()` serializes the neutral transcript. `origin_provider` is the **normalized lowercase prefix** via `provider_prefix(self.model.provider.name)` (`providers.py:888-895`) — `self.model.provider.name` is capitalized (`"Anthropic"`/`"OpenAI"`/`"OpenRouter"`, `:279/:303/:329`), so it must be normalized to match model-ref prefixes. `origin_provider`/`origin_model` are **required** envelope fields; the allowed-key set is exactly `{kind, version, origin_provider, origin_model, entries}`. +- `resume_session(state)` (any provider) validates the neutral envelope and prepares the session for full replay. It **must not make a model request** (it runs before hooks/`_running`), so the rendered replay is deferred to the first post-resume turn, not produced inside `resume_session`. + +**Replay injection is entry-point-agnostic.** A resumed run's first provider call is `continue_with_user_prompt` (normal resume) or `continue_with_tools` (approval resume — `core.py:492` sets `skip_user_prompt`; `_resume_approval_batch` enters through `send_tool_outputs` → `continue_with_tools`, `core.py:778`). The first such turn renders the transcript prefix and injects it, then proceeds normally; the resume flag is consumed so subsequent turns use the native in-run path. + +- **Anthropic / OpenRouter (stateless).** `resume_session` stores `self._resume_entries = entries` (it cannot render yet — no `instructions`). The first post-resume turn renders entries into `self.messages`, sets/prepends the system prompt from the live `instructions`, appends the new turn's content, completes, and clears `_resume_entries`. +- **OpenAI (server-side).** `resume_session` stores `self._pending_replay = entries`, leaves `previous_response_id = None`. The first post-resume turn renders entries into Responses `input` items and produces the final `input` as `[*replay_items, *entry_point_items]`, sends with **no** `previous_response_id`, then clears `_pending_replay`. **`continue_with_user_prompt` builds a *string* `input`** (`providers.py:516`); when `_pending_replay` is set, that string must first be wrapped as a `{type:message, role:user, content:[{type:input_text, text}]}` item before prepending replay items (you cannot prepend list items to a string). `continue_with_tools` already builds a list, so it prepends directly. Once the response yields an `id`, chaining resumes. + +**System prompt re-injection** (reverses plan-08's "instructions ignored on resume" — `.plans/08-resume.md:226/:236`): + +- **Anthropic** — the first resumed turn sets `self.system = instructions` (today it ignores `instructions`, relying on rehydrated `self.system`, `providers.py:656-670`). Gated on the resume flag so non-resumed in-run turns are untouched. +- **OpenRouter** — the resume render prepends `{"role": "system", "content": instructions}` as `messages[0]` (today system is the rehydrated `messages[0]`, `providers.py:800-812`). +- **OpenAI** — already passes `instructions` live every turn (`providers.py:407`); no change. + +**Per-provider render (`_render_input`):** + +- **Anthropic** — entries → `messages`. `AssistantEntry` → one assistant message with `{type:tool_use, id, name, input: json.loads(arguments)}` per call, **plus a leading `{type:text}` block only when `text` is non-empty** (an empty text block would shift `content[0]` off the `tool_use` block that `test_resume.py:61` pins). **A maximal run of consecutive `ToolResultEntry`s + an immediately-following `UserEntry(notice=True)` coalesces into one `user` message** with N `tool_result` blocks (+ trailing `text` block) — separate user messages would violate role alternation. A `UserEntry(notice=True)` with *zero* preceding tool results (defensive: shouldn't occur) renders as a plain user message with a text block. Plain `UserEntry` → its own user message. +- **OpenRouter** — entries → chat messages. `AssistantEntry` → `{role:assistant, content, tool_calls:[{id, type:function, function:{name, arguments}}]}`; each `ToolResultEntry` → `{role:tool, tool_call_id, content}` (consecutive tool messages allowed); `UserEntry` → `{role:user, content}`. +- **OpenAI Responses** — entries → `input` items: `UserEntry` → `{type:message, role:user, content:[{type:input_text, text}]}`; `AssistantEntry` → optional `{type:message, role:assistant, content:[{type:output_text, text}]}` (omit when text empty) + one `{type:function_call, call_id, name, arguments}` per call; `ToolResultEntry` → `{type:function_call_output, call_id, output}`. + +**Tool-call id portability (claim softened).** Replaying both sides of a call/result pair from the stored `ModelToolCall.id` preserves *internal* correlation, but does **not** guarantee the receiving provider accepts a foreign-format id (e.g. OpenAI may reject a `toolu_*` `call_id`). Acceptance must be verified against the real API (residual risks). Likewise, on a cross-provider replay into Anthropic, `json.loads(arguments)` could raise if a non-Anthropic origin stored non-JSON `arguments` (the OpenAI/OpenRouter extractors pass the raw provider string, `providers.py:973/:1008`); treat a parse failure as a documented cross-provider failure mode adjacent to the foreign-`call_id` risk. + +### 4. Cross-provider / cross-model resume + version + validator coupling + +- `_validate_resume_state`'s model/provider mismatch rejection (`providers.py:197-202`) is **removed**. `origin_provider`/`origin_model` are retained for diagnostics only. +- **`version` bumps to `2`.** The envelope shape changed, so per plan-08's contract the version must bump. The validator rejects old state with a clear, **`"resume_from"`-prefixed** `HarnessError` instructing the caller to regenerate — both old `version: 1` *and* old `kind` values (`"openai"`/`"anthropic"`/`"openrouter"`). thinharness is greenfield (no deployed persisted state); no migration shim. +- **The validator must keep the `"resume_from"` message prefix.** `_resume_approval_session` relabels only errors whose message `startswith("resume_from")` into `"approval state provider_state…"` (`core.py:724-728`); `test_approvals.py:733` depends on this. If the new validator changes the prefix, either preserve it or update `core.py:726` to match — state the coupling. +- Validation still rejects: non-dict, wrong/missing `version`, missing/non-list/malformed `entries`, missing/wrong-typed `origin_*`, unknown top-level keys, non-JSON-serializable payloads, malformed entry dicts (bad `role`/missing fields). `HarnessError`, not `ProviderError`. +- v1 degradation is simple because reasoning is text: switching providers loses nothing structural — message/tool history replays cleanly. + +### 5. `resume_kind`, capability gates, and the custom-model boundary + +`resume_kind` is no longer used for envelope validation (`kind` is fixed to `"transcript"`), but it is **kept as the capability marker** for the two runtime `hasattr` gates that opt a model into resume: `core.py:460` (`model_supports_resume`, also drives finalize's `require_dump_state`) and `core.py:1135` (`_model_supports_approval_resume`, also referenced by approval-tool config validation in `.plans/24-human-in-the-loop.md`). Both gates and the fakes/tests that set `resume_kind` stay. The field is intentionally vestigial for the envelope and load-bearing only for capability detection — documented as such. + +**Custom-model boundary.** The neutral-transcript contract binds the **three built-in providers** (and the real-provider-backed test fakes). Custom `ResumableModel` implementations keep their own opaque `dump_state`/`resume_session` protocol — the harness delegates to `model.resume_session(state)` and does not impose the neutral schema on them. The scripted/sequence test fakes are exactly such custom models (`kind == "scripted"`) and are **left unchanged** (see §Tests). Document this boundary so adapter authors don't assume they must emit `{"kind": "transcript", ...}`. + +### 6. Approval-envelope unification (`approvals.py`, `core.py`) + +`build_approval_envelope`'s `provider_state` already carries `dump_state()` output, restored via `resume_session` (`core.py:465`, `:720-727`). With §3 it becomes the neutral transcript automatically. Required work: + +- Update the field's documented shape; note that the envelope now embeds the full transcript inside `provider_state` (net-new size growth for OpenAI, previously a single id). +- `APPROVAL_ENVELOPE_VERSION` **stays `1`** — intentionally. The outer envelope shape is unchanged; only the *inner* `provider_state` reshapes. The two versions are independent. An approval envelope captured under the old code carries a native `provider_state` that now fails at `resume_session` time (not at `validate_approval_pause_state`, which only checks `provider_state` is a dict, `approvals.py:110`) with the relabeled `"approval state provider_state…"` error — consistent with the greenfield/no-migration stance. State this rather than leaving it silent. +- Fix `test_approval_resume_labels_inner_provider_state_errors` (`test_approvals.py:731-734`): it mutates `provider_state["kind"]="wrong"`; since `kind` is fixed to `"transcript"`, switch the malformed trigger to a bad `version`/`entries` so it still exercises the relabeling path (and the error must keep the `"resume_from"` prefix per §4). The scripted-backed approval envelopes (`test_approvals.py:860/:884`) use the custom protocol and are unaffected. + +### 7. `final_result` / lifecycle parity + +Unchanged from `.plans/08-resume.md`: `resume_state` only on clean `end_turn`, never after the synthetic `final_result` tool, never on non-clean exits. `finalized_via_output_tool` gating in `run_ctx.finalize` (`core.py:536`) stays. + +### 8. Low-level escape hatch — a resume *semantic regression* to document + +`OpenAIResponsesSession.start(previous_response_id=...)` (the documented escape hatch, `providers.py:438`, exercised by `test_providers.py:76-90`) still works in-run. But if a caller seeds a session from an *external* OpenAI response id and the run later dumps state, the neutral transcript captures only the new prompt onward — the externally-seeded prior turns are **absent from a full replay**. Today those turns survive via server-side chaining; after this change they do not. Because this changes resume *semantics* for an existing supported escape hatch (not merely an absent feature), it is surfaced in `behavior.md` (RESUME-6), not just noted here. + +## Behavior changes (update `docs/behavior.md` after review, before implementation) + +Add a `RESUME` section per the template (`docs/behavior.md:6-22`): + +- **RESUME-1** — `resume_state` is a provider-agnostic transcript; resume across providers and across models is supported. +- **RESUME-2** — `resume_state` is self-contained and does not depend on any provider continuation token (e.g. OpenAI server-side response retention); an OpenAI run that never received a response id is still resumable. +- **RESUME-3** — v1 does not preserve provider-specific reasoning chains across resume (reasoning replays as text). +- **RESUME-4** — `resume_state` `version` is `2`; v1-shaped state (old `version` or old provider `kind`) is rejected with a regenerate error. +- **RESUME-5** — on resume, the *live* system prompt (from the resuming harness's config) is re-injected; the captured system prompt is not stored or restored. +- **RESUME-6** — a session seeded via the `start(previous_response_id=…)` escape hatch loses its externally-seeded prior turns on resume (the transcript captures only from the new prompt onward). + +## Implementation steps + +1. **`providers.py`** — add transcript leaf types + dict (de)serialization; add the neutral-envelope validator (drop kind/model rejection, require `version == 2`, keep `"resume_from"` prefix, reject old `kind` values, exact allowed-key set); per session add the parallel transcript (dual-store) + `_resume_entries`/`_pending_replay` flags, neutral-append in `start`/`continue_*`/`_complete`, `_render_input` (Anthropic coalescing + omit-empty-text + system set; OpenAI pending-replay with string→item wrap; OpenRouter system prepend), rewrite `dump_state`/`resume_session` to the neutral envelope with normalized `origin_provider`. Keep native in-run stores and `resume_kind`. +2. **`core.py`** — no structural loop change; confirm `_resume_approval_session` (`:720`) passes the neutral envelope through and the relabel prefix still matches; both capability gates (`:460`, `:1135`) unchanged. +3. **`approvals.py`** — update `provider_state` docs/shape; record version-independence + size-growth notes. +4. **Tests — targeted audit (§"Tests").** Update only real-provider-backed fakes; **leave the scripted/sequence fakes unchanged**. +5. **`docs/behavior.md`** — add `RESUME-1..6` (after review). +6. **Docs surfaces** — update `docs/docs.md` stale contract at `:248` (approval wraps provider resume state), `:252`, `:558`, `:562`, `:564` (same provider/model + provider-owned details), `:570` (size growth), `:572`; **add** a resume section to `README.md` (none exists — `:295` is a one-line bullet); regenerate site artifacts (`scripts/build_site.py`) if part of the docs workflow. +7. **Run** `uv run pyright`, ruff, and the full pytest suite (per project `CLAUDE.md`). + +## Tests + +**Do NOT migrate the scripted/sequence fakes.** `ScriptedModel.resume_session`/`ScriptedSession.dump_state` (`fakes.py:177-183/:210/:240-242`) and `SequenceSession` defaults (`test_mcp.py:102`, `test_streaming.py:62`) model custom resumable models with their own `kind == "scripted"` protocol; they never touch `_render_input` or the neutral validator. `test_stream_resume_from_emits_resume_kind` (`test_streaming.py:352`) **passes unchanged** — drop it from the change list. Migrating these would cascade-break `test_resume.py:179/:201/:223/:264/:305/:346/:405`, `test_tracing.py:358`, `test_mcp.py:102`, `test_streaming.py:62`, `test_approvals.py:860/:884`. Only `FakeClient`/`FakeAnthropicProvider`/`FakeOpenRouterProvider`-backed tests change. + +Existing assertions to **invert or restructure** (not "migrate"): + +- `test_resume.py:30` — resume-turn `previous_response_id` assertions invert; **also** `:43` (`input == "follow-up"` string→list) and `:46` (`"first" not in json.dumps` — now `"first"` *is* present). In-run `payloads[1]["previous_response_id"]` stays. +- `test_resume.py:88` — remove `kind`/`model` rejection branches (cross-provider/cross-model now succeed); keep version + unknown-key branches, retargeted to `version: 2`. +- `test_resume.py:116` (`test_resume_rejects_malformed_shapes_before_hooks_fire`) — **restructure** against the neutral schema. Every sub-case uses dead v1 field names (`:132/:134/:139/:141`), and the JSON-serializable case (`:143-147`) now trips the unknown-key check (which runs before `json.dumps`, `providers.py:208-214`), so it would raise "unknown keys" not "JSON-serializable". Rebuild the cases (and assert the JSON-serializable case still raises the JSON error — adjust validator order if needed). +- `test_resume.py:248` (`test_no_openai_response_id_omits_resume_state`) — **inverts**: a no-id OpenAI run is now resumable (RESUME-2). +- `test_resume.py:361-374` (detachment) — index changes from `["messages"]` to `["entries"]`; the mutated element must be a `UserEntry` (`entries[0]["content"]`), since `entries[1]` is the `AssistantEntry` (no `content` key). +- `test_resume.py:399` (`assert session.messages == []`) — **stays valid** under dual-store (native attribute retained). +- `test_approvals.py:600/:609/:629` — provider-native `provider_state` + OpenAI `previous_response_id`-replay assertions become `kind:"transcript"` + full-replay. +- `test_approvals.py:731-734` — change malformed trigger per §6; keep the `"resume_from"` prefix so relabeling fires. +- `test_providers.py:106-120` — neutral envelope + full-replay shape. + +New cases: + +1. **Cross-provider resume after a real tool round-trip** — capture on an Anthropic-shaped fake *through an actual tool turn*, resume on OpenAI- and OpenRouter-shaped fakes; assert the assistant tool call and its result replay with matching ids. +2. **Cross-model same-provider resume** — previously rejected; now succeeds. +3. **OpenAI approval-resume full replay** — pause on an approval-required tool (OpenAI-shaped fake), resume; assert the first `continue_with_tools` request replays the full prior input (user + assistant `function_call`) + the `function_call_output`, with **no** `previous_response_id`. +4. **OpenAI normal-resume shape** — assert `payloads[2]["input"]` is the list `[{user "first"}, {assistant function_call}, {function_call_output}, {user "follow-up"}]` with no `previous_response_id`, and `payloads[1]["previous_response_id"]` (in-run chaining) preserved. +5. **Multi-tool batch resume** — assistant turn with ≥2 tool calls: Anthropic render = one `user` message with N `tool_result` blocks; OpenRouter = N `role:tool` messages. (Requires fakes emitting multiple tool calls — current `echo_tool` fakes emit one, `fakes.py:133/:154`.) +6. **System-prompt re-derivation on resume** — resume with a harness whose system prompt differs from the capturing one; assert the replayed input carries the *new* system text (Anthropic `system` / OpenRouter `messages[0]` / OpenAI `instructions`), never `""`/absent (RESUME-5). +7. **Notice preservation, tool and non-tool** — (a) resume after a `continue_with_tools` carrying a background-completion/cancellation notice and a limit warning; assert the notice text survives and (Anthropic) is folded into the tool-result user message; (b) attach a limit notice to `start`/`continue_with_user_prompt` (e.g. low `max_model_requests`), resume, assert the rendered `` text survives in the replayed `UserEntry.content`. +8. **Round-trip serialization** — `json.loads(json.dumps(dump_state()))` equals `dump_state()` for a transcript with all three entry kinds and a multi-tool assistant turn; the non-JSON-serializable-dump test (`test_resume.py:402`) still raises. +9. **OpenAI no-id resume** — `resume_state` non-`None` for a no-id run; resume full-replays (RESUME-2). +10. **Version + kind rejection** — v1-shaped state (`version:1` *and* old `kind` values) raises `HarnessError` (RESUME-4). +11. **In-run byte-identical guard** — confirm `test_harness.py:226-242`, `test_providers.py:60-74`, `test_providers.py:92-175` (minus `:106-120`) are unchanged; any change means Option B leaked. +12. **Wire-shape pins still pass** — `test_resume.py:49/:61/:68` (string-content plain user messages; exact `tool_use`/`tool_result` and `type:"function"` shapes). +13. Carry over (retargeted): `final_result` ⇒ no state, non-clean exits ⇒ no state, malformed envelopes ⇒ `HarnessError`, fresh-harness persistence. + +A test fake that **rejects an unpaired `function_call_output`** is recommended so the OpenAI renderer's pairing is proven against something stricter than permissive `FakeClient`. Live notice tests (`test_providers.py:300/:324`, skipped without keys) seed `session.system`/`session.messages` directly on a fresh session — confirm dual-store keeps that working. + +## Out of scope (deferred) + +- **Same-provider reasoning fidelity** — a `provider_extras` field stamped with `origin_provider`, re-emitted only on same-provider resume (Anthropic thinking `signature`, OpenAI `encrypted_content`), with per-entry `provider_name` added then. v1 stores reasoning as text. +- **Dropping stale notices** from the durable transcript — intentional behavior change; v1 preserves current behavior. +- **Tracing/event-stream as projections of the transcript** — pydantic-ai has each canonical part own its OTel serialization; thinharness's `tracing.py`/`events.py` could pull payload-building from the transcript. Separate change. +- **Extended-thinking in-run block-ordering constraints** — current code does not enable extended thinking; revisit if it does. +- **OpenAI same-provider `previous_response_id` fast-path on resume** — v1 favors uniform full replay (intentional; full replay is larger payloads for the common same-provider case, accepted for uniformity + retention-independence). + +## Residual risks + +- **Real-provider divergence from fakes (highest).** The OpenAI render path (full `input`-item replay, foreign `call_id` acceptance, `output_text`/`input_text` content types, `function_call`/`function_call_output` pairing without `previous_response_id`, reasoning-item-free replay against gpt-5-class models) is only exercised against fakes. Gate "RESUME-1 cross-provider supported" (vs experimental) on at least one real-API smoke test per provider pair: capture on Anthropic, resume on OpenAI Responses and OpenRouter; confirm acceptance or capture the exact rejection. +- **Cross-provider malformed `arguments`** — `json.loads(arguments)` in the Anthropic render can raise if a non-Anthropic origin stored non-JSON `arguments`; documented failure mode adjacent to foreign-`call_id`. +- **OpenAI `input` wire shape** changes from string to list on the resumed first turn (handled per §3); tests asserting string `input` update. +- **Size growth** — OpenAI `resume_state`/approval-envelope grows O(1)→O(conversation); also in-run *stored* state for OpenAI grows O(1)→O(n) (the parallel transcript), bounded by `max_model_requests`. Document for OpenAI callers. + +## LOC estimate + +~400–600 net lines added against the 1009-line `providers.py`: per-provider `_render_input`, Anthropic coalescing, OpenAI pending-replay (with string→item wrap), system re-injection, dual-store bookkeeping, and explicit entry (de)serialization. A few hundred net lines, not a doubling. + +## Findings resolution — v1 review round + +| Finding (reviewer) | Resolution | +|---|---| +| OpenAI replay can't run in `resume_session`; approval-resume re-enters via `continue_with_tools` (claude H1, glm H1) | §3 deferred entry-point-agnostic pending-replay; tests 3-4. | +| System prompt empty on Anthropic/OpenRouter resume (claude #3, glm H2) | §3 system re-injection from live `instructions`; RESUME-5; test 6. | +| Anthropic must coalesce consecutive tool results into one user message (claude #2, glm M2) | §3 coalescing rule; test 5. | +| Notices dropped from durable transcript (codex #1) | §1 notice capture; test 7. | +| Entry dataclasses not JSON-serializable / no discriminator (claude #4, glm L2) | §1 explicit dict shape + `json.loads(arguments)`. | +| Test blast radius understated; assertions invert (codex, claude #5, glm H3) | §Tests enumerates inversions. | +| No-id OpenAI run becomes resumable (claude #6) | RESUME-2; test 9. | +| `provider_name`/`_parse_extras` speculative (claude #7, glm L3) | Dropped from v1 (§1). | +| `version` not bumped (glm M1) | Bumped to `2`; v1 rejected (§4); RESUME-4. | +| `resume_kind` / two gates (glm M3) | §5 vestigial capability marker; both gates unchanged. | +| Tool-call id cross-provider acceptance over-claimed (glm M4) | §3 softened; real-API residual risk. | +| Assistant content ordering loss (codex #2) | §1 accepted+documented flattening; test 5. | +| Stale `docs/docs.md`; README has no section (codex #3, glm L1) | Step 6 updates docs.md + adds README section. | +| "Collapse `continue_with_*`" inaccurate (glm L4) | §2 corrected. | +| Wrong append-site citations (glm L5) | "Current state" anchored on real sites. | +| Escape-hatch interaction (claude #9, glm residual) | §8 + RESUME-6. | +| Envelope size growth (claude open-q1, glm residual) | §6 + residual risks. | +| behavior.md should be `RESUME-*` (glm/claude open-q) | RESUME-1..6. | +| OpenAI replay only tested vs fake (claude #8) | Residual risk + strict fake. | + +## Findings resolution — v2 review round + +| Finding (reviewer) | Resolution | +|---|---| +| Backing-store wording self-contradictory; pick Option A (claude #1 High, glm #2) | §2 rewritten: explicit dual-store; in-run byte-identical guard (test 11). | +| OpenAI `continue_with_user_prompt` builds a string `input` (glm #1) | §3 string→message-item wrap before prepending replay items. | +| Notice asymmetry for non-tool paths (codex #1, claude #4) | §1 `UserEntry.content` stores notice-appended text on those paths; test 7b. | +| Don't migrate scripted/sequence fakes (claude #3) | §5 custom-model boundary; §Tests leaves them unchanged; `test_streaming.py:352` dropped from change list. | +| `test_resume.py:116` malformed-shapes needs restructure (validator order) (glm #3) | §Tests restructure + assert JSON-serializable case still raises that error. | +| `test_resume.py:361-374` detachment retarget (`messages`→`entries`, mutate `UserEntry`) (claude #2, glm #3) | §Tests explicit retarget. | +| `test_resume.py:43/:46` invert (glm #3) | §Tests enumerated. | +| `_resume_approval_session` relabel depends on `"resume_from"` prefix (claude #6, glm #4) | §4 validator keeps the prefix (or update `core.py:726`); stated coupling. | +| behavior.md missing system-on-resume change (glm #5) | RESUME-5. | +| Escape-hatch is a semantic regression, belongs in behavior.md (claude #9, glm #5) | RESUME-6 + §8. | +| `origin_provider`/`origin_model` required + normalized (glm #6) | §3 required, exact allowed-key set, `provider_prefix` normalization. | +| Anthropic omit empty assistant text block (glm #7) | §3 leading text block only when non-empty. | +| docs.md enumeration incomplete (claude #7) | Step 6 adds `:248/:562/:564/:570`. | +| Approval envelope version independence (claude #8, glm #10) | §6 stays `1`, independent, old envelopes rejected at resume time. | +| Coalescing zero-tool-results edge (glm #8) | §3 defensive: plain user message. | +| Cross-provider malformed `arguments` (glm #9) | §3 + residual risks documented failure mode. | +| OpenAI in-run memory O(1)→O(n) (glm #11) | Residual risks note. | +| Custom `ResumableModel` schema boundary (codex open-q) | §5 custom-model boundary. | + +## Open questions + +None blocking. Deferred items are listed under "Out of scope." diff --git a/.plans/30-tracing-consolidation.md b/.plans/30-tracing-consolidation.md new file mode 100644 index 0000000..1997a1f --- /dev/null +++ b/.plans/30-tracing-consolidation.md @@ -0,0 +1,142 @@ +# Tracing and event projection from transcript deltas + +## Goal + +Reduce duplicated conversation-shape construction in tracing and runtime now that built-in providers maintain a neutral transcript. The durable transcript is the canonical model-visible conversation state; tracing and streaming should be projections from the same per-request model-visible delta where that makes sense. + +This is intentionally a behavior cleanup, not a backwards-compatibility exercise. Trace input should reflect the exact model-visible content, including rendered `` text. Keeping a separate "logical but not quite what the model saw" trace view is unnecessary complexity. + +## Current state + +The new transcript introduced by `.plans/29-unified-transcript-log.md` is the durable model-visible log: + +- `UserEntry(content, notice=False)` +- `AssistantEntry(text, tool_calls)` +- `ToolResultEntry(call_id, output)` +- `UserEntry(content, notice=True)` for notice text accompanying tool-result batches + +Tracing currently has a parallel request-log model: + +- `ModelTraceSnapshot` in `tracing.py` carries `kind`, `prompt`, `tool_outputs`, `notices`, and `structured_output`. +- `TurnDriver` builds that snapshot independently in `runtime.py` for `start`, `resume`, `send_tool_outputs`, and `send_user_message`. +- `tracing.py` then rebuilds model input payloads again via `_model_request_input()` and `_otel_input_messages()`. +- `tracing.py` rebuilds assistant output again via `_otel_output_messages(turn)`. +- `runtime.py` separately emits `ModelMessageEvent` by mapping `ModelTurn.tool_calls` to `StreamToolCall`. + +The event stream is not just the transcript. It includes operational lifecycle notifications that are not durable model-visible conversation entries: + +- run start/completion/failure +- model request start +- tool call start/completion +- background task start/completion +- retry and limit notifications +- approval resume + +So the right target is not "make events.py equal transcript." The right target is "use one model-visible request/response delta as the source for durable transcript, model tracing payloads, and model-message stream projection." Operational events remain separate. + +## Design sketch + +Introduce an internal `ModelRequestDelta` (name flexible) near the provider/runtime boundary: + +```python +@dataclass(frozen=True) +class ModelRequestDelta: + kind: Literal["start", "resume", "tool_outputs", "approval_resume", "correction", "output_retry_tool", "background_completion"] + entries: list[TranscriptEntry] + notices: list[ModelNotice] = field(default_factory=list) + structured_output: str | None = None +``` + +The delta represents the exact new model-visible input for one provider request, after hooks/background/limit notices have been rendered into user text where applicable. `notices` remains as structured metadata for observability, but the primary trace input should include the rendered notice text because that is what the model saw. + +Examples: + +- start/resume/correction: one `UserEntry(content=append_notices_to_text(prompt, notices))` +- tool output request: N `ToolResultEntry`s plus optional `UserEntry(notice=True)` if notices are sent as user text +- background completion as user message: one `UserEntry(content=...)` +- approval resume: same as tool output request, with `kind="approval_resume"` + +Then project it to: + +- tracing input attributes from `entries` plus structured `notices` metadata (`gen_ai.input.messages`, `gen_ai.prompt`, `langfuse.observation.input`, `thinharness.model.notices`) +- request-kind stream metadata (`ModelRequestStartedEvent.request_kind`) +- provider transcript append logic, if provider sessions can accept already-rendered entries without leaking into their native in-run request builders + +For assistant output, add a helper from `AssistantEntry` / `ModelTurn` to: + +- tracing output attributes (`gen_ai.output.messages`, `langfuse.observation.output`) +- `ModelMessageEvent` +- durable transcript append, if provider sessions can share that helper + +## Boundaries + +Keep these separate: + +- **Conversation log:** model-visible entries only. This is what durable resume uses. +- **Operational stream:** lifecycle notifications. It includes some conversation projections, but also events that happen before or outside model-visible transcript entries. +- **Trace spans:** operational timing plus optional model-visible payload capture. + +Do not add operational metadata to durable transcript entries. Avoid storing timing, stream sequence, tracing capture policy, or output-mode details in `resume_state`. + +Public stream event dataclasses should stay mostly stable, with one intended simplification: remove `StreamOptions.include_model_text`. The stream is an observability/log surface; suppressing assistant text makes it less useful and adds branching that other agent frameworks do not appear to expose as a core option. If callers need redaction, that should be a separate filtering layer outside the core event schema. + +## Implementation steps + +1. Add projection helpers in a small internal module, e.g. `thinharness/projections.py`, so `providers.py` does not learn about tracing or stream-event concerns: + - `trace_input_messages_from_entries(entries)` + - `trace_output_messages_from_assistant(entry_or_turn)` + - `model_request_input_from_delta(delta)` + - `stream_tool_calls_from_assistant(entry_or_turn)` + +2. Replace `ModelTraceSnapshot.prompt/tool_outputs/notices` with `ModelRequestDelta.entries/notices`, or adapt `ModelTraceSnapshot` into this shape as an intermediate step. Do not keep separate logical/model-visible entry lists. + +3. Update `TurnDriver` to build one delta per model request after notices are known: + - It currently builds `ModelTraceSnapshot` before `advance_model`, but final limit/background notices are only known inside `RunContext.advance_model`. + - The likely place to finalize the delta is inside `advance_model`, after `all_notices` is computed and before `annotate_model_request`. + +4. Update `annotate_model_request()` to consume the delta and remove `_otel_input_messages()` branches that duplicate request-kind logic. Trace input should include rendered notices because the delta is model-visible. + +5. Update `annotate_model_span()` to use the same assistant-entry projection helper used by stream `ModelMessageEvent`. + +6. Keep stream lifecycle event emission in `runtime.py` and tool execution code. Use the shared assistant projection for `ModelMessageEvent` and always include assistant text. + +7. Remove `StreamOptions.include_model_text` and the associated runtime branch. Keep `StreamOptions.include_subagents` unless a separate review shows it is also unnecessary. + +8. Leave provider-native in-run request builders untouched. This must preserve the dual-store boundary from plan 29: normal in-run provider payloads should stay byte-identical. + +## Tests + +This is mostly a refactor, with two intentional behavior/API changes: + +- trace input messages now represent exact model-visible input, including rendered notices +- stream events no longer support suppressing assistant text via `StreamOptions.include_model_text` + +Required checks: + +- Existing tracing tests in `tests/test_tracing.py` should be updated where they currently expect notices to be absent from trace input messages. +- Existing streaming tests in `tests/test_streaming.py` and approval streaming tests should be updated if they cover `StreamOptions.include_model_text`. +- Existing provider payload tests should remain byte-identical; any changed in-run payload means transcript projection leaked into provider request construction. +- Full `uv run pyright`, relevant `ruff`, and full `pytest`. + +New or adjusted tests worth adding: + +- One tracing test that asserts a tool-output request with a notice includes the rendered notice in trace input and also preserves structured notice metadata. +- One tracing test that asserts a prompt-path notice includes the rendered notice in trace input. +- One tracing test that asserts assistant text + tool calls uses the same projection as `ModelMessageEvent`. +- One regression test for resume first-turn tracing: replayed transcript itself should not be double-counted as the new request delta unless that is intentionally exposed. +- One streaming test update proving model text is always included in `ModelMessageEvent`. + +## Risks + +- Tracing has sink-specific conventions (`gen_ai.*`, Langfuse) that are not identical to durable transcript shape. The shared helper should project transcript entries into those conventions rather than forcing tracing attributes to become transcript dictionaries. +- Including rendered notices in trace input is a trace-shape behavior change. This is accepted for simplicity and fidelity to what the model saw, but docs/tests should make it explicit. +- Limit notices are computed inside `advance_model`, while request kind/prompt/tool outputs are prepared in `TurnDriver`. Moving delta finalization too early will miss notices; moving too late can obscure the original request kind. +- Stream events include operational timing and partial lifecycle states. Treating the stream as "just the transcript" would lose useful host notifications like `tool_call_started`. + +## Out of scope + +- Broad changes to public stream event schemas beyond removing `StreamOptions.include_model_text`. +- Adding token streaming. +- Making tracing required or changing capture defaults. +- Moving tool execution records into durable transcript. +- Reducing provider-native in-run state or changing the transcript resume envelope. diff --git a/.plans/image-inputs.md b/.plans/image-inputs.md new file mode 100644 index 0000000..81dd4dd --- /dev/null +++ b/.plans/image-inputs.md @@ -0,0 +1,26 @@ +# Image Inputs + +ThinHarness should support images by making model input richer while keeping plain text prompts unchanged. The public API should continue to accept `Harness.run("prompt")`, and additionally accept an ordered sequence of input parts such as text plus image content. + +The core type should be provider-neutral: + +```python +UserInput = str | Sequence[UserInputPart] +UserInputPart = TextPart | ImageUrlPart | BinaryImagePart +``` + +`TextPart` can stay optional at first because bare strings inside the sequence are enough for most callers. `ImageUrlPart` should hold a URL, optional MIME type, optional provider metadata, and an optional `force_download` flag. `BinaryImagePart` should hold bytes, MIME type, and optional provider metadata, with helpers such as `from_path(...)` and `from_data_uri(...)`. + +The harness should normalize prompt handling once near the run boundary. Hooks, tracing, stream events, and model session APIs should receive the neutral input shape instead of assuming a string. Text-only callers should see the same behavior and payloads they see today. + +Provider adapters should own wire-format mapping: + +- OpenAI Responses: map text to `input_text` and images to `input_image`; binary images should use data URIs, URL images should use URLs unless `force_download` is set. +- Anthropic Messages: map text to text blocks and images to image blocks; binary images should use base64 source blocks, URL images should use URL source blocks when supported. +- OpenRouter chat completions: map text and images to chat content parts using `text` and `image_url`; binary images should use data URIs. + +Unsupported image cases should fail loudly with a provider error. The harness should not silently stringify images or drop them. + +Keep the first implementation limited to images. Do not include audio, video, PDFs, uploaded provider files, or prompt-cache markers yet. The type shape should leave room for those later, but the code should not implement speculative modalities. + +Tests should cover backward-compatible text prompts, mixed text/image ordering, URL image mapping, binary image mapping, provider-specific unsupported cases, notice appending with rich input, and tracing redaction or placeholder behavior so raw image bytes are not accidentally written into local traces. diff --git a/docs/behavior.md b/docs/behavior.md index ea2fd5b..f0a0274 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -77,3 +77,18 @@ Built-in provider resume state is a self-contained, provider-agnostic transcript - RESUME-4: Built-in provider resume state uses `version` 2; version 1 state and old provider-native `kind` values are rejected with a regenerate error. - RESUME-5: On resume, the live system prompt from the resuming harness config is re-injected; captured system prompts are not stored or restored. - RESUME-6: A session seeded via `OpenAIResponsesSession.start(previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. + +## Model Observability Projections + +### Purpose + +Tracing and streaming expose projections of the same neutral per-request model-visible input and assistant output without changing provider-native in-run request construction. + +### Requirements + +- MODEL-OBSERVABILITY-1: Model trace input is built from the new model-visible entries for that provider request, including rendered `` text whenever notices are sent to the model. +- MODEL-OBSERVABILITY-2: Structured model notice metadata remains available on model spans separately from rendered input messages. +- MODEL-OBSERVABILITY-3: Replayed resume transcript entries are not counted as new model request input for the first resumed provider request; only the new resume prompt or continuation delta is traced as request input. +- MODEL-OBSERVABILITY-4: `ModelMessageEvent.text` always includes assistant text from the completed provider turn; stream text suppression is not part of `StreamOptions`. +- MODEL-OBSERVABILITY-5: Stream lifecycle events remain operational events and are not stored in durable provider transcript entries. +- MODEL-OBSERVABILITY-6: Core tracing emits OTel/GenAI-oriented attributes and does not include sink-specific display namespaces. diff --git a/docs/docs.md b/docs/docs.md index b76e6ce..98583fe 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -63,7 +63,7 @@ Stream events are high-level workflow events intended for app consumption: - `ToolCallStartedEvent.arguments` includes the model-requested tool arguments. - `ToolCallCompletedEvent.output` and `BackgroundTaskCompletedEvent.output` include model-visible tool output. - Raw provider response JSON is not part of stream events; use `HarnessResult.responses` for raw provider responses after completion. -- `ModelMessageEvent.text` is included by default; set `include_model_text=False` to hide it. +- `ModelMessageEvent.text` includes assistant text from the completed provider turn. - Child subagent events are flattened by default; set `include_subagents=False` to keep only the parent `subagent` tool lifecycle. Workflow UIs should group nested work with `kind`, `run_id`, `parent_run_id`, and `parent_tool_call_id`. The final successful `RunCompletedEvent` carries the same full `HarnessResult` returned by `run()`, including `responses`, `tool_call_records`, and `resume_state`; treat that terminal result as completion data rather than a progress update. diff --git a/docs/site/about/index.html b/docs/site/about/index.html index bc7ab6f..dca0653 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -78,7 +78,7 @@

How small, exactly

ThinHarness
- 8,241 + 8,493 ●●●●●● @@ -93,37 +93,37 @@

How small, exactly

deepagents4
- 17,039 + 17,664 ○●○●●○
AWS Strands
- 28,157 + 32,526 ◑●●○○●
Microsoft Agent Framework
- 40,514 + 41,331 ○●●●○●
Pydantic AI
- 59,034 + 59,087 ●○●○○●
Google ADK
- 64,890 + 65,799 ◑●●●○●
OpenAI Agents SDK
- 73,139 + 73,796 ●●●○○●
Agno
- 111,539 + 113,477 ◑●●●●● @@ -134,7 +134,7 @@

How small, exactly

1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.

2. Tool retries: a documented primitive (e.g. Pydantic AI's ModelRetry) that lets tools signal "model passed bad args — retry with this feedback," distinct from generic exception propagation.

3. Claude Agent SDK shells out to the Claude Code CLI binary, which is 200k+ LOC.

-

4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈111k LOC.

+

4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈112k LOC.

See docs/table.md for per-cell rationale and how the LOC numbers are measured.

diff --git a/docs/site/explainer/index.html b/docs/site/explainer/index.html index a827866..fbf73fb 100644 --- a/docs/site/explainer/index.html +++ b/docs/site/explainer/index.html @@ -51,11 +51,11 @@

A mental model for the whole repository

Snapshot

- 22 + 23 runtime Python files under thinharness/, including the tools package.
- 8,241 + 8,493 README-stated framework LOC, intentionally small enough to inspect, adapt, and fork.
@@ -136,7 +136,7 @@

Payload policy

  • Stream events include high-level prompt, tool argument, and model-visible tool result payloads for app-facing workflow visibility.
  • Raw provider response JSON is not included in stream events. Use the terminal HarnessResult.responses after completion when provider raw responses are needed.
    • -
  • StreamOptions can hide model text or child subagent events, but it does not expose raw provider payloads.
    • +
  • StreamOptions can hide child subagent events, but it does not hide model text or expose raw provider payloads.
  • Nested work is correlated with run_id, parent_run_id, parent_tool_call_id, agent_name, and per-stream sequence.
diff --git a/docs/site/index.html b/docs/site/index.html index d23e268..a386f36 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -36,7 +36,7 @@

A minimal, opinionated agent harness.
install.sh
-
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 22 files · 8,241 LOC
+
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 23 files · 8,493 LOC

diff --git a/docs/table.md b/docs/table.md index 4d4d36e..aaaf714 100644 --- a/docs/table.md +++ b/docs/table.md @@ -12,33 +12,35 @@ The README is the canonical rendered table. The raw README source comments keep the command beside each row. To reproduce locally, clone each upstream repo at the pinned commit and run the command shown below. -Measured 2026-06-15 for upstream libraries. ThinHarness was remeasured from the -current working tree on 2026-06-20. +Measured 2026-06-22 for upstream libraries (pinned commits below). ThinHarness +was measured from the current working tree on 2026-06-23. ## LOC Commands - **ThinHarness** — `tokei thinharness/ -t Python` -- **Claude Agent SDK** — `tokei src/claude_agent_sdk/ -t Python --exclude testing` at `anthropics/claude-agent-sdk-python @ 634c2f6` -- **smolagents** — `tokei src/smolagents/ -t Python --exclude cli.py --exclude gradio_ui.py --exclude vision_web_browser.py` at `huggingface/smolagents @ e8b988d` -- **deepagents** — `tokei libs/deepagents/deepagents/ -t Python` at `langchain-ai/deepagents @ 5975503` -- **AWS Strands** — `tokei strands-py/src/strands/ -t Python --exclude experimental --exclude vended_plugins --exclude multiagent/a2a` at `strands-agents/sdk-python @ a92502f` -- **Microsoft Agent Framework** — `tokei python/packages/core/agent_framework/ -t Python --exclude _evaluation.py --exclude a2a --exclude ag_ui --exclude chatkit --exclude declarative --exclude devui --exclude hyperlight --exclude lab --exclude orchestrations --exclude mem0 --exclude redis --exclude microsoft` at `microsoft/agent-framework @ ed4ff18` -- **Pydantic AI** — `tokei pydantic_ai_slim/pydantic_ai/ -t Python --exclude _a2a.py --exclude ag_ui.py --exclude ui --exclude durable_exec --exclude embeddings --exclude ext` at `pydantic/pydantic-ai @ fabeacc` -- **Google ADK** — `tokei src/google/adk/ -t Python --exclude a2a --exclude apps --exclude cli --exclude cloud --exclude code_executors --exclude environment --exclude evaluation --exclude examples --exclude integrations --exclude optimization --exclude platform` at `google/adk-python @ 22adbe1` -- **OpenAI Agents SDK** — `tokei src/agents/ -t Python --exclude realtime --exclude voice --exclude extensions/experimental --exclude extensions/visualization.py` at `openai/openai-agents-python @ c359c20` -- **Agno** — `tokei libs/agno/agno/{agent,agents,approval,compression,factory,guardrails,hooks,memory,models,reasoning,registry,run,session,skills,team,tools,tracing,utils} -t Python` at `agno-agi/agno @ 5cf1ed7` +- **Claude Agent SDK** — `tokei src/claude_agent_sdk/ -t Python --exclude testing` at `anthropics/claude-agent-sdk-python @ 315df97` +- **smolagents** — `tokei src/smolagents/ -t Python --exclude cli.py --exclude gradio_ui.py --exclude vision_web_browser.py` at `huggingface/smolagents @ 526069c` +- **deepagents** — `tokei libs/deepagents/deepagents/ -t Python` at `langchain-ai/deepagents @ eb9de75` +- **AWS Strands** — `tokei strands-py/src/strands/ -t Python --exclude experimental --exclude vended_plugins --exclude multiagent/a2a` at `strands-agents/sdk-python @ a5a2cf9` +- **Microsoft Agent Framework** — `tokei python/packages/core/agent_framework/ -t Python --exclude _evaluation.py --exclude a2a --exclude ag_ui --exclude chatkit --exclude declarative --exclude devui --exclude hyperlight --exclude lab --exclude orchestrations --exclude mem0 --exclude redis --exclude microsoft` at `microsoft/agent-framework @ 2999f74` +- **Pydantic AI** — `tokei pydantic_ai_slim/pydantic_ai/ -t Python --exclude _a2a.py --exclude ag_ui.py --exclude ui --exclude durable_exec --exclude embeddings --exclude ext` at `pydantic/pydantic-ai @ 53e0641` +- **Google ADK** — `tokei src/google/adk/ -t Python --exclude a2a --exclude apps --exclude cli --exclude cloud --exclude code_executors --exclude environment --exclude evaluation --exclude examples --exclude integrations --exclude optimization --exclude platform` at `google/adk-python @ 8c9fff8` +- **OpenAI Agents SDK** — `tokei src/agents/ -t Python --exclude realtime --exclude voice --exclude extensions/experimental --exclude extensions/visualization.py` at `openai/openai-agents-python @ a9b7b7e` +- **Agno** — `tokei libs/agno/agno/{agent,agents,approval,compression,factory,guardrails,hooks,memory,models,reasoning,registry,run,session,skills,team,tools,tracing,utils} -t Python` at `agno-agi/agno @ 16f33c1` Claude Agent SDK also shells out to the Claude Code CLI binary, which is 200k+ LOC. The table counts the Python SDK package and footnotes that relationship. ## Notes on the Marks -- **Tool retries** — only Pydantic AI (`ModelRetry`) and OpenAI Agents (`ModelRetryAdvice` / `ModelRetrySettings`) ship a documented, named primitive that lets a tool function signal "model passed bad args — please retry with this feedback," distinct from generic exception propagation. AWS Strands has hook-based retry via `AfterToolCallEvent.retry=True`, Google ADK has a `ReflectAndRetryToolPlugin`, and Agno has a `RetryAgentRun` exception that retries the whole agent run rather than a single tool — these are marked `⚠️`. Claude Agent SDK, smolagents, deepagents, and Microsoft Agent Framework have no named primitive and are marked `❌`. -- **Subagents** — Pydantic AI documents an "agent delegation" pattern, where one agent is called inside another's tool function, but ships no class, decorator, or middleware for it. Its own multi-agent docs point users to deepagents for that case, so it is marked `❌`. -- **Structured output** — Claude Agent SDK and deepagents return free-form messages with no built-in validation step, so they are marked `❌`. -- **Skills** — Pydantic AI, OpenAI Agents SDK, smolagents, and AWS Strands have no Markdown/frontmatter skills primitive, so they are marked `❌`. -- **Built-in FS tools** — A `✅` means the project ships a model-facing filesystem toolkit with read/write/edit or search-style primitives. OpenAI Agents SDK is marked `⚠️` because it has hosted `apply_patch` and shell tools, but not a full read/write/search toolkit. Google ADK is marked `⚠️` because its experimental environment tools include `ReadFile`, `WriteFile`, and `EditFile`, but those are outside the strict core LOC scope above. Pydantic AI, smolagents, AWS Strands, and Microsoft Agent Framework are marked `❌` because they do not ship a comparable toolkit in the core package. Generic shell or code-exec tools do not count as full filesystem tools. -- **OTel tracing** — deepagents leans on LangSmith rather than emitting OTel from its own code, so it is marked `❌`. Claude Agent SDK is marked `⚠️` because the Python SDK itself ships no instrumentation beyond W3C traceparent propagation into the CLI subprocess, while the Claude Code CLI it shells out to has beta OTel support. +Marks reflect shipped, documented, first-class capability judged on the public API — independent of whether a feature is gated experimental or lives in a directory excluded from the strict LOC count. + +- **Tool retries** — only Pydantic AI (`ModelRetry`) ships a documented, named primitive that lets a tool function signal "model passed bad args — please retry with this feedback," distinct from generic exception propagation. OpenAI Agents' `ModelRetryAdvice` / `ModelRetrySettings` are runner-managed retries for the *model HTTP call* (network/timeout/HTTP-status backoff), not a tool-feedback primitive. AWS Strands has hook-based retry via `AfterToolCallEvent.retry=True`, Google ADK has a `ReflectAndRetryToolPlugin`, and Agno has a `RetryAgentRun` exception that retries the whole agent run rather than a single tool — these are marked `⚠️`. Claude Agent SDK, smolagents, deepagents, Microsoft Agent Framework, and OpenAI Agents SDK have no named primitive and are marked `❌`. +- **Subagents** — Pydantic AI documents an "agent delegation" pattern, where one agent is called inside another's tool function, but ships no class, decorator, or middleware for it in core (real subagent primitives live in external community packages), so it is marked `❌`. +- **Structured output** — every library now ships a built-in output-validation step, so the column is uniformly `✅`. Claude Agent SDK (`output_format` / `structured_output`, validated and re-prompted in the CLI it shells out to) and deepagents (`response_format` on `create_deep_agent`) added theirs in 2026. +- **Skills** — Pydantic AI and smolagents have no Markdown/frontmatter skills primitive in core, so they are marked `❌`. OpenAI Agents SDK (sandbox `Skills`) and AWS Strands (top-level `Skill` / `AgentSkills`) added one in 2026. +- **Built-in FS tools** — A `✅` means the project ships a model-facing filesystem toolkit with read/write/edit or search-style primitives. Google ADK (`ReadFile` / `WriteFile` / `EditFile`) and Microsoft Agent Framework (`FileAccessProvider` — read/write/delete/list/search) both ship one; both are gated experimental, but the mark reflects shipped capability regardless of maturity or LOC scope. OpenAI Agents SDK is marked `⚠️` because it ships only hosted `apply_patch` (create/update/delete) plus shell — not a full read/write/search toolkit (read and search come through bash, which doesn't count). Pydantic AI, smolagents, and AWS Strands ship no comparable toolkit in core, so they are marked `❌`. Generic shell or code-exec tools do not count as full filesystem tools. +- **OTel tracing** — AWS Strands, Microsoft Agent Framework, Pydantic AI, and Google ADK emit OpenTelemetry spans from their own code (`✅`). smolagents, OpenAI Agents SDK, and Agno are marked `⚠️` because they emit no spans from their own code: tracing comes from a separate external instrumentor (OpenInference, for smolagents and Agno) or a proprietary exporter with OTel reachable only via third-party processors (OpenAI). Claude Agent SDK is marked `⚠️` because the Python SDK itself ships no instrumentation beyond W3C traceparent propagation into the CLI subprocess, while the Claude Code CLI it shells out to has beta OTel support. deepagents leans on LangSmith rather than emitting OTel from its own code, so it is marked `❌`. ## What "Strict Framework-Only" Excludes @@ -59,4 +61,4 @@ Per-library exclusions: - **Pydantic AI** — `_a2a.py`, `ag_ui.py`, `ui/`, `durable_exec/`, `embeddings/`, `ext/` (A2A, UI, durable execution runtime, embedding models, ext). - **Google ADK** — `a2a/`, `apps/`, `cli/`, `cloud/`, `code_executors/`, `environment/`, `evaluation/`, `examples/`, `integrations/`, `optimization/`, `platform/`. - **OpenAI Agents SDK** — `realtime/`, `voice/`, `extensions/experimental`, `extensions/visualization.py`. -- **Agno** — `api/`, `client/`, `cloud/`, `db/`, `integrations/`, `knowledge/`, `learn/`, `os/`, `remote/`, `scheduler/`, `vectordb/`, `context/`, `culture/`, plus boundary cases `workflow/` and `eval/`. As shipped, Agno is 262,687 LOC. +- **Agno** — `api/`, `client/`, `cloud/`, `db/`, `integrations/`, `knowledge/`, `learn/`, `os/`, `remote/`, `scheduler/`, `vectordb/`, `context/`, `culture/`, plus boundary cases `workflow/` and `eval/`. As shipped, Agno is 265,195 LOC. diff --git a/tests/test_background_tools.py b/tests/test_background_tools.py index e377f42..7b3c2bf 100644 --- a/tests/test_background_tools.py +++ b/tests/test_background_tools.py @@ -775,7 +775,11 @@ async def foreground(_args): input_messages = json.loads(coalesced_chat.attributes["gen_ai.input.messages"]) notices = json.loads(coalesced_chat.attributes["thinharness.model.notices"]) contents = [part["content"] for message in input_messages for part in message["parts"]] - assert contents == [session.tool_outputs[0][0].output, session.tool_outputs[0][1].output] + assert contents == [ + session.tool_outputs[0][0].output, + session.tool_outputs[0][1].output, + f'\n{notices[0]["content"]}\n', + ] assert notices[0]["kind"] == "background_completion" assert "Tool: background" in notices[0]["content"] diff --git a/tests/test_streaming.py b/tests/test_streaming.py index bdc63e1..51e2860 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -157,6 +157,16 @@ async def test_stream_payloads_are_high_level_without_raw_provider_payloads(tmp_ assert "ok" in completed.output +async def test_stream_options_keep_model_text_visible(tmp_path: Path) -> None: + session = ScriptedSession(start_turn=ModelTurn(text="visible", raw={"id": "done"})) + harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=ScriptedModel([session])) + + events = await _collect_events(harness, "go", stream_options=StreamOptions()) + + message = next(event for event in events if isinstance(event, ModelMessageEvent)) + assert message.text == "visible" + + async def test_stream_limit_warning_events(tmp_path: Path) -> None: session = ScriptedSession(start_turn=ModelTurn(text="done", raw={"id": "done"})) harness = Harness(HarnessConfig(root=tmp_path, builtin_tools=[], max_model_requests=1), model=ScriptedModel([session])) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 808ca90..790f8a3 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -23,6 +23,7 @@ Harness, HarnessConfig, HarnessError, + ModelMessageEvent, SubAgentConfig, ToolResult, ToolSpec, @@ -30,8 +31,9 @@ build_child_harness, create_subagent_tool, ) -from thinharness.providers import ModelNotice, ModelToolCall, ModelTurn -from thinharness.tracing import ModelTraceSnapshot, _SpanAdapter, annotate_model_request, create_local_tracing_options, serialize_attribute_value +from thinharness.projections import model_request_delta_from_prompt, model_request_delta_from_tool_outputs +from thinharness.providers import ModelNotice, ModelToolCall, ModelTurn, ToolOutput +from thinharness.tracing import _SpanAdapter, annotate_model_request, create_local_tracing_options, serialize_attribute_value class Person(BaseModel): @@ -72,12 +74,12 @@ def test_harness_tracing_records_agent_model_and_tool_spans(tmp_path: Path) -> N assert root.attributes["gen_ai.operation.name"] == "invoke_agent" assert root.attributes["gen_ai.conversation.id"] == "conv-1" assert root.attributes["gen_ai.completion"] == "done" - assert root.attributes["langfuse.trace.input"] == "read hello" + assert root.attributes["gen_ai.prompt"] == "read hello" assert "gen_ai.system_instructions" in root.attributes assert first_chat.attributes["gen_ai.provider.name"] == "OpenAI" assert first_chat.attributes["gen_ai.request.model"] == "test-model" assert json.loads(first_chat.attributes["gen_ai.input.messages"])[0]["parts"][0]["content"] == "read hello" - assert json.loads(first_chat.attributes["langfuse.observation.input"]) == {"prompt": "read hello"} + assert json.loads(first_chat.attributes["gen_ai.prompt"]) == {"prompt": "read hello"} assert "gen_ai.system_instructions" not in first_chat.attributes first_output = json.loads(first_chat.attributes["gen_ai.output.messages"]) assert first_output[0]["parts"][0]["type"] == "tool_call" @@ -89,26 +91,49 @@ def test_harness_tracing_records_agent_model_and_tool_spans(tmp_path: Path) -> N assert "hello" in tool.attributes["gen_ai.tool.call.result"] assert second_chat.attributes["gen_ai.completion"] == "done" assert json.loads(second_chat.attributes["gen_ai.output.messages"])[0]["parts"][0]["content"] == "done" - assert json.loads(second_chat.attributes["langfuse.observation.output"]) == {"text": "done"} + assert all(key.startswith(("gen_ai.", "thinharness.")) for key in root.attributes) + assert all(key.startswith(("gen_ai.", "thinharness.")) for key in first_chat.attributes) + assert all(key.startswith("gen_ai.") for key in tool.attributes) + assert all(key.startswith(("gen_ai.", "thinharness.")) for key in second_chat.attributes) def test_serialize_attribute_value_keeps_none_absent() -> None: assert serialize_attribute_value(None) is None -def test_model_request_snapshot_keeps_tool_output_separate_from_notices() -> None: +def test_model_request_delta_includes_rendered_tool_output_notices() -> None: span = FakeSpan("chat", {}) - snapshot = ModelTraceSnapshot( + delta = model_request_delta_from_tool_outputs( kind="tool_outputs", - tool_outputs=[{"call_id": "call_1", "output": '{"ok":true,"content":"real output"}'}], - ).with_notices([ModelNotice(kind="limit_warning", content="notice text", limit_kind="model_requests", remaining=1)]) + outputs=[ToolOutput(call_id="call_1", output='{"ok":true,"content":"real output"}')], + notices=[ModelNotice(kind="limit_warning", content="notice text", limit_kind="model_requests", remaining=1)], + structured_output=None, + ) - annotate_model_request(_SpanAdapter(span), snapshot, capture_messages=True) + annotate_model_request(_SpanAdapter(span), delta, capture_messages=True) input_messages = json.loads(span.attributes["gen_ai.input.messages"]) notices = json.loads(span.attributes["thinharness.model.notices"]) assert input_messages[0]["parts"][0]["content"] == '{"ok":true,"content":"real output"}' - assert "notice text" not in span.attributes["gen_ai.input.messages"] + assert input_messages[1]["parts"][0]["content"] == '\nnotice text\n' + assert "notice text" in span.attributes["gen_ai.prompt"] assert notices[0]["content"] == "notice text" +def test_model_request_delta_includes_rendered_prompt_notices() -> None: + span = FakeSpan("chat", {}) + delta = model_request_delta_from_prompt( + kind="start", + prompt="hello", + notices=[ModelNotice(kind="limit_warning", content="final turn", limit_kind="model_requests", remaining=1)], + structured_output=None, + ) + + annotate_model_request(_SpanAdapter(span), delta, capture_messages=True) + + input_messages = json.loads(span.attributes["gen_ai.input.messages"]) + assert input_messages[0]["parts"][0]["content"] == 'hello\n\n\nfinal turn\n' + assert json.loads(span.attributes["gen_ai.prompt"]) == { + "prompt": 'hello\n\n\nfinal turn\n' + } + def test_local_tracing_writes_full_jsonl_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("THINHARNESS_DISABLE_LOCAL_TRACING", raising=False) (tmp_path / "hello.txt").write_text("hello", encoding="utf-8") @@ -138,7 +163,7 @@ def test_local_tracing_writes_full_jsonl_trace(tmp_path: Path, monkeypatch: pyte root = next(record for record in records if record["name"] == "invoke_agent thinharness") first_chat = next(record for record in records if record["name"] == "chat test-model" and "gen_ai.output.messages" in record["attributes"]) tool = next(record for record in records if record["name"] == "execute_tool read") - assert root["attributes"]["langfuse.trace.input"] == "read hello" + assert root["attributes"]["gen_ai.prompt"] == "read hello" assert json.loads(first_chat["attributes"]["gen_ai.input.messages"])[0]["parts"][0]["content"] == "read hello" assert json.loads(first_chat["attributes"]["gen_ai.output.messages"])[0]["parts"][0]["type"] == "tool_call" assert tool["attributes"]["gen_ai.tool.call.arguments"] == '{"path":"hello.txt"}' @@ -170,8 +195,8 @@ def test_local_tracing_nests_subagent_spans(tmp_path: Path, monkeypatch: pytest. subagent_tool = next(record for record in records if record["name"] == "execute_tool subagent") child_agent = next(record for record in records if record["name"] == "invoke_agent subagent.default") assert child_agent["parent_id"] == subagent_tool["span_id"] - assert child_agent["attributes"]["langfuse.observation.input"] == "help" - assert "child done" in child_agent["attributes"]["langfuse.observation.output"] + assert child_agent["attributes"]["gen_ai.prompt"] == "help" + assert child_agent["attributes"]["gen_ai.completion"] == "child done" def test_local_tracing_does_not_change_remote_capture_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("THINHARNESS_DISABLE_LOCAL_TRACING", raising=False) @@ -187,10 +212,6 @@ def test_local_tracing_does_not_change_remote_capture_policy(tmp_path: Path, mon harness.run_sync("read hello") forbidden = { - "langfuse.observation.input", - "langfuse.observation.output", - "langfuse.trace.input", - "langfuse.trace.output", "gen_ai.input.messages", "gen_ai.output.messages", "gen_ai.system_instructions", @@ -225,10 +246,6 @@ def test_capture_messages_false_omits_content_attributes(tmp_path: Path) -> None harness.run_sync("read hello") forbidden = { - "langfuse.observation.input", - "langfuse.observation.output", - "langfuse.trace.input", - "langfuse.trace.output", "gen_ai.input.messages", "gen_ai.output.messages", "gen_ai.system_instructions", @@ -255,6 +272,36 @@ def test_tool_tracing_marks_normalized_failures(tmp_path: Path) -> None: assert tool.status is not None assert tool.attributes["error.type"] == "ToolExecutionError" + +async def test_assistant_text_and_tool_calls_project_to_trace_and_stream(tmp_path: Path) -> None: + turn = ModelTurn( + text="thinking", + tool_calls=[ModelToolCall(id="call_1", name="echo", arguments='{"value":"ok"}')], + raw={"id": "start"}, + ) + session = ScriptedSession(start_turn=turn, continue_turn=ModelTurn(text="done", raw={"id": "done"})) + tracer = FakeTracer() + harness = Harness( + HarnessConfig(root=tmp_path, builtin_tools=[]), + model=ScriptedModel([session]), + tools=[echo_tool()], + tracing=[TracingOptions(tracer=tracer, capture_messages=True)], + ) + + events = [] + async for event in harness.stream("go"): + events.append(event) + + message = next(event for event in events if isinstance(event, ModelMessageEvent) and event.text == "thinking") + assert [(call.id, call.name) for call in message.tool_calls] == [("call_1", "echo")] + first_chat = next(span for span in tracer.spans if span.name == "chat scripted-model") + parts = json.loads(first_chat.attributes["gen_ai.output.messages"])[0]["parts"] + assert parts == [ + {"type": "text", "content": "thinking"}, + {"type": "tool_call", "id": "call_1", "name": "echo", "arguments": '{"value":"ok"}'}, + ] + + def test_subagent_tracing_nests_child_under_parent_tool_span(tmp_path: Path) -> None: parent_call = ModelTurn( tool_calls=[ModelToolCall(id="call_1", name="subagent", arguments='{"task":"help"}')], @@ -288,10 +335,8 @@ def test_subagent_tracing_nests_child_under_parent_tool_span(tmp_path: Path) -> assert child_chat.parent is child_agent assert final_chat.parent is root assert child_agent.attributes["gen_ai.agent.name"] == "subagent.default" - assert child_agent.attributes["langfuse.observation.input"] == "help" - assert child_agent.attributes["langfuse.observation.output"] - assert "langfuse.trace.input" not in child_agent.attributes - assert "langfuse.trace.output" not in child_agent.attributes + assert child_agent.attributes["gen_ai.prompt"] == "help" + assert child_agent.attributes["gen_ai.completion"] == "child done" assert subagent_tool.attributes["subagent.name"] == "default" assert subagent_tool.attributes["subagent.tool_mode"] == "inherited" assert subagent_tool.attributes["subagent.tools"] == ["echo"] @@ -341,8 +386,8 @@ def test_concurrent_subagent_fanout_keeps_each_child_under_own_tool_span(tmp_pat assert all(span.parent is root for span in subagent_tools) assert {id(span.parent) for span in child_agents} == {id(span) for span in subagent_tools} assert {id(span.parent) for span in child_model_spans} == {id(span) for span in child_agents} - assert {span.attributes["langfuse.observation.input"] for span in child_agents} == {"first", "second"} - assert all(span.attributes["langfuse.observation.input"] != "delegate" for span in child_agents) + assert {span.attributes["gen_ai.prompt"] for span in child_agents} == {"first", "second"} + assert all(span.attributes["gen_ai.prompt"] != "delegate" for span in child_agents) def test_trace_request_kinds_for_resume_and_output_retries(tmp_path: Path) -> None: retry_session = ScriptedSession( @@ -385,12 +430,12 @@ def test_trace_request_kinds_for_resume_and_output_retries(tmp_path: Path) -> No assert "resume" in kinds retry_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "output_retry_tool") assert json.loads(retry_chat.attributes["gen_ai.input.messages"])[0]["parts"][0]["content"].startswith("The previous response failed") - assert "Final request" not in retry_chat.attributes["gen_ai.input.messages"] + assert "Final request" in retry_chat.attributes["gen_ai.input.messages"] assert "Final request" in retry_chat.attributes["thinharness.model.notices"] correction_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "correction") - assert json.loads(correction_chat.attributes["langfuse.observation.input"])["correction"].startswith("The previous response failed") + assert json.loads(correction_chat.attributes["gen_ai.prompt"])["correction"].startswith("The previous response failed") resume_chat = next(span for span in chats if span.attributes.get("thinharness.model.request.kind") == "resume") - assert json.loads(resume_chat.attributes["langfuse.observation.input"]) == {"prompt": "follow-up"} + assert json.loads(resume_chat.attributes["gen_ai.prompt"]) == {"prompt": "follow-up"} def test_provider_error_keeps_trace_input_without_output(tmp_path: Path) -> None: tracer = FakeTracer() @@ -404,9 +449,9 @@ def test_provider_error_keeps_trace_input_without_output(tmp_path: Path) -> None harness.run_sync("fail please") root = tracer.spans[0] - assert root.attributes["langfuse.trace.input"] == "fail please" + assert root.attributes["gen_ai.prompt"] == "fail please" assert "gen_ai.system_instructions" in root.attributes - assert "langfuse.trace.output" not in root.attributes + assert "gen_ai.completion" not in root.attributes assert root.status is not None def test_unknown_named_subagent_trace_marks_failed_without_child_tool_mode(tmp_path: Path) -> None: diff --git a/thinharness/events.py b/thinharness/events.py index b8c21c1..54ba3e3 100644 --- a/thinharness/events.py +++ b/thinharness/events.py @@ -39,7 +39,6 @@ class StreamToolCall: class StreamOptions: """Visibility controls for Harness.stream().""" - include_model_text: bool = True include_subagents: bool = True diff --git a/thinharness/projections.py b/thinharness/projections.py new file mode 100644 index 0000000..42ab534 --- /dev/null +++ b/thinharness/projections.py @@ -0,0 +1,133 @@ +"""Shared projections from neutral model turns and transcript entries.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +from .events import StreamToolCall +from .providers import ( + AssistantEntry, + ModelNotice, + ModelTurn, + ToolOutput, + ToolResultEntry, + TranscriptEntry, + UserEntry, + append_notices_to_text, + render_model_notices, +) +from .types import Json + +ModelRequestKind = Literal["start", "resume", "tool_outputs", "approval_resume", "correction", "output_retry_tool", "background_completion"] + + +@dataclass(frozen=True) +class ModelRequestDelta: + """Exact model-visible input entries for one provider request.""" + + kind: ModelRequestKind + entries: list[TranscriptEntry] + notices: list[ModelNotice] = field(default_factory=list) + structured_output: str | None = None + + +def model_request_delta_from_prompt( + *, + kind: Literal["start", "resume", "correction", "background_completion"], + prompt: str, + notices: list[ModelNotice], + structured_output: str | None, +) -> ModelRequestDelta: + """Build a request delta for a user-text provider continuation.""" + return ModelRequestDelta( + kind=kind, + entries=[UserEntry(content=append_notices_to_text(prompt, notices))], + notices=list(notices), + structured_output=structured_output, + ) + + +def model_request_delta_from_tool_outputs( + *, + kind: Literal["tool_outputs", "approval_resume", "output_retry_tool", "background_completion"], + outputs: list[ToolOutput], + notices: list[ModelNotice], + structured_output: str | None, +) -> ModelRequestDelta: + """Build a request delta for a tool-output provider continuation.""" + entries: list[TranscriptEntry] = [ + ToolResultEntry(call_id=output.call_id, output=output.output) + for output in outputs + ] + if notice_text := render_model_notices(notices): + entries.append(UserEntry(content=notice_text, notice=True)) + return ModelRequestDelta( + kind=kind, + entries=entries, + notices=list(notices), + structured_output=structured_output, + ) + + +def trace_input_messages_from_entries(entries: list[TranscriptEntry]) -> list[Json]: + """Project neutral transcript entries into OTel-style input messages.""" + messages: list[Json] = [] + for entry in entries: + if isinstance(entry, UserEntry): + messages.append({"role": "user", "parts": [{"type": "text", "content": entry.content}]}) + elif isinstance(entry, ToolResultEntry): + messages.append({ + "role": "tool", + "parts": [{ + "type": "tool_result", + "id": entry.call_id, + "content": entry.output, + }], + }) + else: + messages.extend(trace_output_messages_from_assistant(entry)) + return messages + + +def trace_output_messages_from_assistant(entry_or_turn: AssistantEntry | ModelTurn) -> list[Json]: + """Project a neutral assistant entry or model turn into OTel-style output messages.""" + parts: list[Json] = [] + if entry_or_turn.text: + parts.append({"type": "text", "content": entry_or_turn.text}) + for call in entry_or_turn.tool_calls: + parts.append({ + "type": "tool_call", + "id": call.id, + "name": call.name, + "arguments": call.arguments, + }) + return [{"role": "assistant", "parts": parts}] + + +def model_request_input_from_delta(delta: ModelRequestDelta) -> Json | None: + """Return the trace display payload for one model-visible request delta.""" + if len(delta.entries) == 1 and isinstance(delta.entries[0], UserEntry): + content = delta.entries[0].content + if delta.kind in {"start", "resume"}: + return {"prompt": content} + if delta.kind == "correction": + return {"correction": content} + if delta.kind == "background_completion": + return {"background_completion": content} + + tool_outputs = [ + {"call_id": entry.call_id, "output": entry.output} + for entry in delta.entries + if isinstance(entry, ToolResultEntry) + ] + if tool_outputs and len(tool_outputs) == len(delta.entries): + return {"tool_outputs": tool_outputs} + if delta.entries: + return {"messages": trace_input_messages_from_entries(delta.entries)} + return None + + +def stream_tool_calls_from_assistant(entry_or_turn: AssistantEntry | ModelTurn) -> tuple[StreamToolCall, ...]: + """Project assistant tool calls into public stream-event summaries.""" + return tuple(StreamToolCall(id=call.id, name=call.name) for call in entry_or_turn.tool_calls) diff --git a/thinharness/runtime.py b/thinharness/runtime.py index 28e12e7..988fd8b 100644 --- a/thinharness/runtime.py +++ b/thinharness/runtime.py @@ -5,7 +5,7 @@ import json from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal, Protocol +from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from .approvals import build_approval_envelope from .events import ( @@ -17,14 +17,18 @@ RunCompletedEvent, RunStreamContext, StreamEmitter, - StreamToolCall, ) from .hooks import LimitReachedContext, RunEndContext from .output import OutputTurnDecision, resolve_turn_output +from .projections import ( + ModelRequestKind, + model_request_delta_from_prompt, + model_request_delta_from_tool_outputs, + stream_tool_calls_from_assistant, +) from .providers import ModelNotice, ModelSession, ModelTurn, StructuredOutputRequest, ToolOutput from .tool_execution import BackgroundToolCompletion, background_completion_message from .tracing import ( - ModelTraceSnapshot, RunTracer, _trace_output_mode, annotate_agent_result, @@ -75,7 +79,8 @@ async def start(self, prompt: str) -> tuple[ModelTurn, OutputTurnDecision]: structured_output=self._structured_output, notices=notices, ), - trace_snapshot=ModelTraceSnapshot(kind="start", prompt=prompt, structured_output=self._output_mode), + request_kind="start", + prompt=prompt, ) async def resume(self, prompt: str) -> tuple[ModelTurn, OutputTurnDecision]: @@ -89,7 +94,8 @@ async def resume(self, prompt: str) -> tuple[ModelTurn, OutputTurnDecision]: structured_output=self._structured_output, notices=notices, ), - trace_snapshot=ModelTraceSnapshot(kind="resume", prompt=prompt, structured_output=self._output_mode), + request_kind="resume", + prompt=prompt, ) async def send_tool_outputs( @@ -110,11 +116,8 @@ async def send_tool_outputs( structured_output=self._structured_output, notices=[*notices, *(extra_notices or [])], ), - trace_snapshot=ModelTraceSnapshot( - kind=kind, - tool_outputs=[{"call_id": item.call_id, "output": item.output} for item in outputs], - structured_output=self._output_mode, - ), + request_kind=kind, + tool_outputs=outputs, output_retry=output_retry, extra_notices=extra_notices, ) @@ -136,7 +139,8 @@ async def send_user_message( structured_output=self._structured_output, notices=notices, ), - trace_snapshot=ModelTraceSnapshot(kind=kind, prompt=message, structured_output=self._output_mode), + request_kind=kind, + prompt=message, output_retry=output_retry, ) @@ -144,14 +148,19 @@ async def _run_model_request( self, request: ModelRequest, *, - trace_snapshot: ModelTraceSnapshot, + request_kind: ModelRequestKind, + prompt: str | None = None, + tool_outputs: list[ToolOutput] | None = None, output_retry: bool = False, extra_notices: list[ModelNotice] | None = None, ) -> tuple[ModelTurn, OutputTurnDecision]: """Delegate to RunContext model advancement.""" return await self._run_ctx.advance_model( request, - trace_snapshot=trace_snapshot, + request_kind=request_kind, + prompt=prompt, + tool_outputs=tool_outputs, + structured_output=self._output_mode, output_retry=output_retry, extra_notices=extra_notices, ) @@ -372,7 +381,10 @@ async def advance_model( self, request: ModelRequest, *, - trace_snapshot: ModelTraceSnapshot, + request_kind: ModelRequestKind, + structured_output: str | None, + prompt: str | None = None, + tool_outputs: list[ToolOutput] | None = None, output_retry: bool = False, extra_notices: list[ModelNotice] | None = None, ) -> tuple[ModelTurn, OutputTurnDecision]: @@ -388,7 +400,7 @@ async def advance_model( all_notices = [*notices, *(extra_notices or [])] self.emit(ModelRequestStartedEvent( **self.stream_base(), - request_kind=trace_snapshot.kind, + request_kind=request_kind, model=self.harness.model_ref, provider=getattr(self.harness.model.provider, "name", None), )) @@ -405,11 +417,27 @@ async def advance_model( if output_retry: self.usage.output_retries += 1 with self.tracer.model(self.harness.model) as model_span: - snapshot = trace_snapshot.with_notices(all_notices) + if tool_outputs is None: + assert prompt is not None + assert request_kind in {"start", "resume", "correction", "background_completion"} + delta = model_request_delta_from_prompt( + kind=cast(Literal["start", "resume", "correction", "background_completion"], request_kind), + prompt=prompt, + notices=all_notices, + structured_output=structured_output, + ) + else: + assert request_kind in {"tool_outputs", "approval_resume", "output_retry_tool", "background_completion"} + delta = model_request_delta_from_tool_outputs( + kind=cast(Literal["tool_outputs", "approval_resume", "output_retry_tool", "background_completion"], request_kind), + outputs=tool_outputs, + notices=all_notices, + structured_output=structured_output, + ) model_span.for_each( lambda span, option: annotate_model_request( span, - snapshot, + delta, capture_messages=option.capture_messages, ) ) @@ -434,11 +462,10 @@ async def advance_model( "thinharness.output.mode": finalized_mode, "gen_ai.output.finalized": True, }) - options = self.stream.options if self.stream is not None else None self.emit(ModelMessageEvent( **self.stream_base(), - text=turn.text if options is None or options.include_model_text else "", - tool_calls=tuple(StreamToolCall(id=call.id, name=call.name) for call in turn.tool_calls), + text=turn.text, + tool_calls=stream_tool_calls_from_assistant(turn), finalized_output_mode=turn.finalized_output_mode, )) return turn, decision diff --git a/thinharness/tracing.py b/thinharness/tracing.py index 94dbf42..12ac4cc 100644 --- a/thinharness/tracing.py +++ b/thinharness/tracing.py @@ -1,11 +1,10 @@ """OpenTelemetry-compatible tracing helpers. -Model input messages are constructed from provider-neutral ModelTraceSnapshot -objects, never from provider payloads, because providers.py may already have -appended harness notices to those payloads. For top-level runs, -langfuse.trace.input stores the raw caller prompt while the first model span -stores the effective prompt after hooks. OTel GenAI message shapes follow the -semantic convention as retrieved on 2026-05-19: +Model input messages are constructed from provider-neutral request deltas, +never from provider payloads. For top-level runs, the agent span stores the raw +caller prompt while the first model span stores the effective prompt after +hooks. OTel GenAI message shapes follow the semantic convention as retrieved +on 2026-05-19: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/. """ @@ -20,13 +19,13 @@ import uuid from collections.abc import Callable, Iterator from contextlib import ExitStack, contextmanager -from dataclasses import asdict, dataclass, replace +from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Literal +from typing import Any from pydantic import BaseModel, ConfigDict -from .providers import ModelNotice +from .projections import ModelRequestDelta, model_request_input_from_delta, trace_input_messages_from_entries, trace_output_messages_from_assistant from .tools.base import Json try: @@ -73,22 +72,6 @@ class LocalTracing: trace_dir: Path -@dataclass(frozen=True) -class ModelTraceSnapshot: - """Canonical input for one model span.""" - - kind: Literal["start", "resume", "tool_outputs", "approval_resume", "correction", "output_retry_tool", "background_completion"] - prompt: str | None = None - tool_outputs: list[Json] | None = None - notices: list[Json] | None = None - structured_output: str | None = None - - def with_notices(self, notices: list[ModelNotice]) -> ModelTraceSnapshot: - """Return a copy with model-facing notices attached.""" - serialized = [asdict(notice) for notice in notices] - return replace(self, notices=serialized or None) - - def create_local_tracing(trace_dir: str | Path | None = None, *, project_root: str | Path | None = None) -> LocalTracing: """Create a plaintext JSONL tracer rooted in the local filesystem.""" resolved = Path(trace_dir or "~/.thinharness/traces").expanduser().resolve() @@ -474,18 +457,18 @@ def to_record(self) -> Json: } -def annotate_model_request(span: _SpanAdapter, snapshot: ModelTraceSnapshot, *, capture_messages: bool) -> None: - """Write opt-in request content for portable OTel and Langfuse display.""" +def annotate_model_request(span: _SpanAdapter, delta: ModelRequestDelta, *, capture_messages: bool) -> None: + """Write opt-in request content using OTel GenAI attributes.""" if not capture_messages: return - input_payload = _model_request_input(snapshot) + input_payload = model_request_input_from_delta(delta) + notices = [asdict(notice) for notice in delta.notices] span.set_attributes({ - "gen_ai.input.messages": serialize_attribute_value(_otel_input_messages(snapshot)), + "gen_ai.input.messages": serialize_attribute_value(trace_input_messages_from_entries(delta.entries)), "gen_ai.prompt": serialize_attribute_value(input_payload), - "langfuse.observation.input": serialize_attribute_value(input_payload), - "thinharness.model.request.kind": snapshot.kind, - "thinharness.output.mode_requested": snapshot.structured_output, - "thinharness.model.notices": serialize_attribute_value(snapshot.notices), + "thinharness.model.request.kind": delta.kind, + "thinharness.output.mode_requested": delta.structured_output, + "thinharness.model.notices": serialize_attribute_value(notices or None), }) @@ -499,12 +482,9 @@ def annotate_model_span(span: _SpanAdapter, turn: Any, *, capture_messages: bool "gen_ai.response.finish_reasons": _finish_reasons(raw), **_usage_attributes(raw), } - output_messages = _otel_output_messages(turn) + output_messages = trace_output_messages_from_assistant(turn) if capture_messages and output_messages and output_messages[0]["parts"]: attributes.update({ - "langfuse.observation.output": serialize_attribute_value( - {"text": text} if text else {"messages": output_messages} - ), "gen_ai.output.messages": serialize_attribute_value(output_messages), }) if text: @@ -525,11 +505,11 @@ def annotate_agent_start( return if top_level: span.set_attributes({ - "langfuse.trace.input": prompt, + "gen_ai.prompt": prompt, "gen_ai.system_instructions": serialize_attribute_value([{"type": "text", "content": instructions}]), }) else: - span.set_attribute("langfuse.observation.input", prompt) + span.set_attribute("gen_ai.prompt", prompt) def annotate_agent_result( @@ -543,15 +523,12 @@ def annotate_agent_result( """Write opt-in agent trace or observation output attributes.""" if not capture_messages: return - output_payload = output_schema.dump(result.output) if output_schema is not None and result.output is not None else None - output = {"text": result.text, "output": output_payload, "stop_reason": result.stop_reason} if top_level: span.set_attributes({ - "langfuse.trace.output": serialize_attribute_value(output), "gen_ai.completion": result.text, }) else: - span.set_attribute("langfuse.observation.output", serialize_attribute_value(output)) + span.set_attribute("gen_ai.completion", result.text) def _trace_output_mode(output_schema: Any | None) -> str | None: @@ -571,70 +548,6 @@ def serialize_attribute_value(value: Any) -> str | None: return str(value) -def _model_request_input(snapshot: ModelTraceSnapshot) -> Json | None: - """Return the backend-compatible logical request payload.""" - if snapshot.kind in {"start", "resume"}: - return {"prompt": snapshot.prompt} - if snapshot.kind in {"tool_outputs", "approval_resume", "output_retry_tool"} or ( - snapshot.kind == "background_completion" and snapshot.tool_outputs is not None - ): - return {"tool_outputs": snapshot.tool_outputs or []} - if snapshot.kind == "background_completion": - return {"background_completion": snapshot.prompt} - if snapshot.kind == "correction": - return {"correction": snapshot.prompt} - return None - - -def _otel_input_messages(snapshot: ModelTraceSnapshot) -> list[Json] | None: - """Return OTel-shaped logical input messages.""" - if snapshot.kind == "background_completion": - if snapshot.prompt is not None: - return [{"role": "user", "parts": [{"type": "text", "content": snapshot.prompt}]}] - return [ - { - "role": "tool", - "parts": [{ - "type": "tool_result", - "id": output.get("call_id"), - "content": output.get("output"), - }], - } - for output in snapshot.tool_outputs or [] - ] - if snapshot.kind in {"start", "resume", "correction"} and snapshot.prompt is not None: - return [{"role": "user", "parts": [{"type": "text", "content": snapshot.prompt}]}] - if snapshot.kind in {"tool_outputs", "approval_resume", "output_retry_tool"}: - return [ - { - "role": "tool", - "parts": [{ - "type": "tool_result", - "id": output.get("call_id"), - "content": output.get("output"), - }], - } - for output in snapshot.tool_outputs or [] - ] - return None - - -def _otel_output_messages(turn: Any) -> list[Json]: - """Return OTel-shaped assistant output messages.""" - parts: list[Json] = [] - text = getattr(turn, "text", "") or "" - if text: - parts.append({"type": "text", "content": text}) - for call in getattr(turn, "tool_calls", []) or []: - parts.append({ - "type": "tool_call", - "id": getattr(call, "id", None), - "name": getattr(call, "name", None), - "arguments": getattr(call, "arguments", None), - }) - return [{"role": "assistant", "parts": parts}] - - def _usage_attributes(raw: Json) -> Json: """Extract common token usage attributes from provider responses.""" usage = raw.get("usage") or {} From 54262c54f814351e99e23b71aba137e3b04bce95 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 23 Jun 2026 00:31:26 -0400 Subject: [PATCH 3/5] feat(providers): preserve native reasoning across same-provider resume Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_016zFdrr1esQDRzYJEEhtkxG --- tests/test_approvals.py | 2 +- tests/test_providers.py | 4 +- tests/test_reasoning_fidelity.py | 509 +++++++++++++++++++++++++++++++ tests/test_resume.py | 18 +- thinharness/projections.py | 4 + thinharness/providers.py | 214 ++++++++++++- 6 files changed, 726 insertions(+), 25 deletions(-) create mode 100644 tests/test_reasoning_fidelity.py diff --git a/tests/test_approvals.py b/tests/test_approvals.py index 47797ce..e7e7048 100644 --- a/tests/test_approvals.py +++ b/tests/test_approvals.py @@ -598,7 +598,7 @@ async def test_openai_approval_pause_round_trips_provider_state(tmp_path: Path) assert result.text == "done" assert called == [{"path": "hello.txt"}] assert paused.resume_state["provider_state"]["kind"] == "transcript" - assert paused.resume_state["provider_state"]["version"] == 2 + assert paused.resume_state["provider_state"]["version"] == 3 assert [entry["role"] for entry in paused.resume_state["provider_state"]["entries"]] == ["user", "assistant"] assert "previous_response_id" not in client.payloads[1] assert [item["type"] for item in client.payloads[1]["input"]] == ["message", "function_call", "function_call_output"] diff --git a/tests/test_providers.py b/tests/test_providers.py index 3b7cd52..bb04f3a 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -105,7 +105,7 @@ async def test_openai_appends_notices_to_string_and_tool_inputs() -> None: await session.continue_with_user_message("fix this", instructions="system", tools=[], notices=[notice]) resumed = model.resume_session({ "kind": "transcript", - "version": 2, + "version": 3, "origin_provider": "openai", "origin_model": "gpt-test", "entries": [{"role": "user", "content": "prior", "notice": False}], @@ -388,7 +388,7 @@ async def test_openai_notice_payload_live() -> None: @pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY is not set") async def test_anthropic_notice_payload_live() -> None: provider = AnthropicProvider() - model = AnthropicMessagesModel(os.getenv("THINHARNESS_LIVE_ANTHROPIC_MODEL", "claude-3-5-haiku-latest"), provider=provider) + model = AnthropicMessagesModel(os.getenv("THINHARNESS_LIVE_ANTHROPIC_MODEL", "claude-haiku-4-5"), provider=provider) session = model.new_session() sentinel_notice = ModelNotice( kind="limit_warning", diff --git a/tests/test_reasoning_fidelity.py b/tests/test_reasoning_fidelity.py new file mode 100644 index 0000000..7fadb6d --- /dev/null +++ b/tests/test_reasoning_fidelity.py @@ -0,0 +1,509 @@ +"""Same-provider reasoning fidelity in the neutral transcript (plan-31).""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path + +import pytest +from fakes import FakeAnthropicProvider, FakeOpenRouterProvider, echo_tool + +from thinharness import ( + AnthropicMessagesModel, + AnthropicProvider, + Harness, + HarnessConfig, + HarnessError, + OpenAIProvider, + OpenAIResponsesModel, + OpenRouterModel, + OpenRouterProvider, + ToolSpec, +) +from thinharness.projections import trace_input_messages_from_entries, trace_output_messages_from_assistant +from thinharness.providers import AssistantEntry, ModelSettings, ModelToolCall, ReasoningPart, UserEntry, _openai_supports_encrypted_reasoning + +REASONING_OPENAI_MODEL = "gpt-5-mini" +THINKING_SETTINGS = ModelSettings(extra_body={"thinking": {"type": "enabled", "budget_tokens": 1024}}) + + +# --- reasoning-emitting fakes (real-provider-backed, per plan §Tests) ----------------------------- + + +class ReasoningOpenAIProvider(OpenAIProvider): + """Responses fake whose first turn carries a native reasoning item + tool call.""" + + def __init__(self) -> None: + super().__init__(api_key="fake") + self.payloads: list = [] + self.calls = 0 + + async def create_response(self, payload): + self.calls += 1 + self.payloads.append(copy.deepcopy(payload)) + if self.calls == 1: + return { + "id": "resp_1", + "output": [ + {"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "thinking about it"}], "encrypted_content": "enc-blob-1"}, + {"type": "function_call", "call_id": "call_1", "name": "echo", "arguments": '{"value":"hi"}'}, + ], + } + return {"id": "resp_2", "output_text": "done"} + + +class TerminalOpenAIProvider(OpenAIProvider): + """Responses fake that records payloads and terminates with text.""" + + def __init__(self) -> None: + super().__init__(api_key="fake") + self.payloads: list = [] + + async def create_response(self, payload): + self.payloads.append(copy.deepcopy(payload)) + return {"id": f"resp_{len(self.payloads)}", "output_text": "done"} + + +class ReasoningTextOpenAIProvider(OpenAIProvider): + """Responses fake whose first turn carries reasoning + assistant text + a tool call.""" + + def __init__(self) -> None: + super().__init__(api_key="fake") + self.payloads: list = [] + self.calls = 0 + + async def create_response(self, payload): + self.calls += 1 + self.payloads.append(copy.deepcopy(payload)) + if self.calls == 1: + return { + "id": "resp_1", + "output": [ + {"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "considering"}], "encrypted_content": "enc-blob-1"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "let me call echo"}]}, + {"type": "function_call", "call_id": "call_1", "name": "echo", "arguments": '{"value":"hi"}'}, + ], + } + return {"id": "resp_2", "output_text": "done"} + + +class ReasoningAnthropicProvider(AnthropicProvider): + """Messages fake whose first turn carries configurable reasoning blocks + a tool call.""" + + def __init__(self, reasoning_blocks: list) -> None: + super().__init__(api_key="key") + self.reasoning_blocks = reasoning_blocks + self.payloads: list = [] + + async def create_message(self, payload): + self.payloads.append(copy.deepcopy(payload)) + last = payload["messages"][-1] + if isinstance(last["content"], str): + return { + "content": [ + *copy.deepcopy(self.reasoning_blocks), + {"type": "tool_use", "id": "toolu_1", "name": "echo", "input": {"value": last["content"]}}, + ], + "stop_reason": "tool_use", + } + return {"content": [{"type": "text", "text": "done"}], "stop_reason": "end_turn"} + + +class ReasoningOpenRouterProvider(OpenRouterProvider): + """OpenRouter fake whose first turn carries reasoning_details + a tool call.""" + + def __init__(self) -> None: + super().__init__(api_key="key") + self.payloads: list = [] + + async def create_chat_completion(self, payload): + self.payloads.append(copy.deepcopy(payload)) + last = payload["messages"][-1] + if last["role"] == "user": + return { + "choices": [{ + "message": { + "role": "assistant", + "reasoning_details": [{"type": "reasoning.encrypted", "data": "or-enc-1", "id": "rd_1"}], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "echo", "arguments": json.dumps({"value": last["content"]})}}], + } + }] + } + return {"choices": [{"message": {"role": "assistant", "content": "done"}}]} + + +def _harness(tmp_path: Path, model, **config) -> Harness: + return Harness(HarnessConfig(root=tmp_path, builtin_tools=[], **config), model=model, tools=[echo_tool()]) + + +async def _capture_state(tmp_path: Path, model) -> dict: + """Run one capturing turn and return its JSON-round-tripped resume state.""" + result = await _harness(tmp_path, model).run("first") + return json.loads(json.dumps(result.resume_state)) + + +def _assistant_reasoning(state: dict) -> list[dict]: + """Return the reasoning list of the first assistant entry carrying reasoning.""" + for entry in state["entries"]: + if entry["role"] == "assistant" and entry["reasoning"]: + return entry["reasoning"] + raise AssertionError("no assistant entry carried reasoning") + + +# --- 1. capture -------------------------------------------------------------------------------- + + +async def test_openai_capture_populates_reasoning_and_include(tmp_path: Path) -> None: + provider = ReasoningOpenAIProvider() + result = await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=provider)).run("first") + + assert provider.payloads[0]["include"] == ["reasoning.encrypted_content"] + reasoning = _assistant_reasoning(result.resume_state) + assert reasoning == [{"text": "thinking about it", "signature": "enc-blob-1", "id": "rs_1", "provider_name": "openai"}] + + +async def test_anthropic_capture_populates_reasoning(tmp_path: Path) -> None: + provider = ReasoningAnthropicProvider([{"type": "thinking", "thinking": "let me think", "signature": "sig-1"}]) + result = await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider)).run("first") + + assert _assistant_reasoning(result.resume_state) == [{"text": "let me think", "signature": "sig-1", "provider_name": "anthropic"}] + + +async def test_openrouter_capture_keeps_raw_reasoning_details(tmp_path: Path) -> None: + provider = ReasoningOpenRouterProvider() + result = await _harness(tmp_path, OpenRouterModel("openai/test", provider=provider)).run("first") + + assert _assistant_reasoning(result.resume_state) == [{ + "text": "", + "signature": "or-enc-1", + "id": "rd_1", + "provider_name": "openrouter", + "provider_details": {"type": "reasoning.encrypted", "data": "or-enc-1", "id": "rd_1"}, + }] + + +# --- 2. same-provider native re-emit ----------------------------------------------------------- + + +async def test_openai_same_provider_reemits_reasoning_item(tmp_path: Path) -> None: + state = await _capture_state(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=ReasoningOpenAIProvider())) + + provider = TerminalOpenAIProvider() + await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=provider)).run("follow-up", resume_from=state) + + items = provider.payloads[0]["input"] + # Trailing "message" pair = the source run's "done" assistant turn + the new follow-up prompt. + assert [item["type"] for item in items] == ["message", "reasoning", "function_call", "function_call_output", "message", "message"] + assert items[1] == {"type": "reasoning", "id": "rs_1", "encrypted_content": "enc-blob-1", "summary": []} + assert "previous_response_id" not in provider.payloads[0] + + +async def test_anthropic_same_provider_reemits_thinking_first(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "thinking", "thinking": "let me think", "signature": "sig-1"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + + provider = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider, settings=THINKING_SETTINGS)).run("follow-up", resume_from=state) + + content = provider.payloads[0]["messages"][1]["content"] + assert content[0] == {"type": "thinking", "thinking": "let me think", "signature": "sig-1"} + assert content[1]["type"] == "tool_use" + + +async def test_openrouter_same_provider_reattaches_reasoning_details(tmp_path: Path) -> None: + state = await _capture_state(tmp_path, OpenRouterModel("openai/test", provider=ReasoningOpenRouterProvider())) + + provider = FakeOpenRouterProvider() + await _harness(tmp_path, OpenRouterModel("openai/test", provider=provider)).run("follow-up", resume_from=state) + + assistant = provider.payloads[0]["messages"][2] + assert assistant["reasoning_details"] == [{"type": "reasoning.encrypted", "data": "or-enc-1", "id": "rd_1"}] + assert assistant["tool_calls"][0]["id"] == "call_1" + + +# --- 2b. cross-model same-provider degrades (resuming model can't accept native reasoning) ----- + + +async def test_openai_cross_model_falls_back_to_text(tmp_path: Path) -> None: + state = await _capture_state(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=ReasoningOpenAIProvider())) + + provider = TerminalOpenAIProvider() + await _harness(tmp_path, OpenAIResponsesModel("gpt-4.1-mini", provider=provider)).run("follow-up", resume_from=state) + + items = provider.payloads[0]["input"] + assert not any(item["type"] == "reasoning" for item in items) + fallback = next(item for item in items if item["type"] == "message" and item["role"] == "assistant") + assert fallback["content"][0]["text"] == "\nthinking about it\n" + assert "enc-blob" not in json.dumps(items) + + +async def test_openai_reasoning_text_toolcall_render_in_order(tmp_path: Path) -> None: + state = await _capture_state(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=ReasoningTextOpenAIProvider())) + + provider = TerminalOpenAIProvider() + await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=provider)).run("follow-up", resume_from=state) + + items = provider.payloads[0]["input"] + types = [item["type"] for item in items] + start = types.index("reasoning") + assert types[start:start + 3] == ["reasoning", "message", "function_call"] + assert items[start + 1]["content"][0]["text"] == "let me call echo" + + +# --- 3. cross-provider text fallback ----------------------------------------------------------- + + +async def test_cross_provider_fallback_to_openai_text(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "thinking", "thinking": "let me think", "signature": "sig-1"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + + provider = TerminalOpenAIProvider() + await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=provider)).run("follow-up", resume_from=state) + + items = provider.payloads[0]["input"] + assert not any(item["type"] == "reasoning" for item in items) + fallback = next(item for item in items if item["type"] == "message" and item["role"] == "assistant") + assert fallback["content"][0]["text"] == "\nlet me think\n" + assert "enc-blob" not in json.dumps(items) and "sig-1" not in json.dumps(items) + + +async def test_cross_provider_fallback_to_openrouter_text(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "thinking", "thinking": "let me think", "signature": "sig-1"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + + provider = FakeOpenRouterProvider() + await _harness(tmp_path, OpenRouterModel("openai/test", provider=provider)).run("follow-up", resume_from=state) + + assistant = provider.payloads[0]["messages"][2] + assert assistant["content"].startswith("\nlet me think\n") + assert "reasoning_details" not in assistant + assert "sig-1" not in json.dumps(provider.payloads[0]["messages"]) + + +# --- 4. anthropic thinking-disabled fallback --------------------------------------------------- + + +async def test_anthropic_same_provider_falls_back_when_thinking_disabled(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "thinking", "thinking": "let me think", "signature": "sig-1"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + + provider = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider)).run("follow-up", resume_from=state) + + content = provider.payloads[0]["messages"][1]["content"] + assert content[0] == {"type": "text", "text": "\nlet me think\n"} + assert content[1]["type"] == "tool_use" + assert not any(block["type"] == "thinking" for block in content) + + +# --- 5. redacted_thinking ---------------------------------------------------------------------- + + +async def test_redacted_thinking_round_trips_natively(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "redacted_thinking", "data": "redacted-blob"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + assert _assistant_reasoning(state) == [{"text": "", "signature": "redacted-blob", "id": "redacted_thinking", "provider_name": "anthropic"}] + + provider = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider, settings=THINKING_SETTINGS)).run("follow-up", resume_from=state) + content = provider.payloads[0]["messages"][1]["content"] + assert content[0] == {"type": "redacted_thinking", "data": "redacted-blob"} + assert content[1]["type"] == "tool_use" + + +async def test_redacted_thinking_dropped_when_thinking_disabled(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([{"type": "redacted_thinking", "data": "redacted-blob"}]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + + provider = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider)).run("follow-up", resume_from=state) + # redacted_thinking has no text, so a disabled-thinking resume drops it entirely (no native block, no fallback). + content = provider.payloads[0]["messages"][1]["content"] + assert [block["type"] for block in content] == ["tool_use"] + + +# --- 6. multi-part turn ------------------------------------------------------------------------ + + +async def test_multi_part_reasoning_renders_in_order(tmp_path: Path) -> None: + source = ReasoningAnthropicProvider([ + {"type": "thinking", "thinking": "step one", "signature": "sig-A"}, + {"type": "redacted_thinking", "data": "redacted-blob"}, + ]) + state = await _capture_state(tmp_path, AnthropicMessagesModel("claude-test", provider=source)) + assert len(_assistant_reasoning(state)) == 2 + + provider = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=provider, settings=THINKING_SETTINGS)).run("follow-up", resume_from=state) + content = provider.payloads[0]["messages"][1]["content"] + assert [block["type"] for block in content] == ["thinking", "redacted_thinking", "tool_use"] + + +# --- 7. round-trip serialization --------------------------------------------------------------- + + +async def test_reasoning_state_round_trips(tmp_path: Path) -> None: + state = (await _harness(tmp_path, OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=ReasoningOpenAIProvider())).run("first")).resume_state + + assert state["version"] == 3 + assert json.loads(json.dumps(state)) == state + assert _assistant_reasoning(state)[0]["signature"] == "enc-blob-1" + + +# --- 8. version rejection ---------------------------------------------------------------------- + + +@pytest.mark.parametrize("version", [1, 2]) +def test_old_transcript_versions_are_rejected(version: int) -> None: + state = {"kind": "transcript", "version": version, "origin_provider": "openai", "origin_model": "gpt-test", "entries": []} + with pytest.raises(HarnessError, match=f"resume_from version {version} is not supported"): + OpenAIResponsesModel("gpt-test", provider=OpenAIProvider(api_key="fake")).resume_session(state) + + +# --- 9. in-run guard --------------------------------------------------------------------------- + + +def test_openai_include_is_the_only_in_run_payload_delta() -> None: + reasoning = OpenAIResponsesModel(REASONING_OPENAI_MODEL, provider=OpenAIProvider(api_key="fake")).build_payload(input_payload="x", tools=[]) + plain = OpenAIResponsesModel("gpt-4.1-mini", provider=OpenAIProvider(api_key="fake")).build_payload(input_payload="x", tools=[]) + + assert set(reasoning) == {"model", "input", "tools", "include"} + assert set(plain) == {"model", "input", "tools"} + assert reasoning["include"] == ["reasoning.encrypted_content"] + assert {k: reasoning[k] for k in ("input", "tools")} == {k: plain[k] for k in ("input", "tools")} + + +async def test_anthropic_and_openrouter_in_run_payloads_carry_no_reasoning_keys(tmp_path: Path) -> None: + anthropic = FakeAnthropicProvider() + await _harness(tmp_path, AnthropicMessagesModel("claude-test", provider=anthropic)).run("first") + assert all(set(payload) == {"model", "max_tokens", "system", "messages", "tools"} for payload in anthropic.payloads) + + openrouter = FakeOpenRouterProvider() + await _harness(tmp_path, OpenRouterModel("openai/test", provider=openrouter)).run("first") + assert all(set(payload) == {"model", "messages", "tools"} for payload in openrouter.payloads) + + +# --- OTel projection (spec `thinking` part) ---------------------------------------------------- + + +def test_reasoning_projects_to_otel_thinking_part() -> None: + entry = AssistantEntry( + text="answer", + tool_calls=[ModelToolCall(id="call_1", name="echo", arguments="{}")], + reasoning=[ + ReasoningPart(text="let me think", signature="sig-secret", provider_name="anthropic"), + ReasoningPart(text="", signature="enc-secret", id="redacted_thinking", provider_name="anthropic"), + ], + ) + + parts = trace_output_messages_from_assistant(entry)[0]["parts"] + # thinking first, then text, then tool_call; the text-less (redacted) part is skipped. + assert [part["type"] for part in parts] == ["thinking", "text", "tool_call"] + assert parts[0] == {"type": "thinking", "content": "let me think"} + # opaque signatures/blobs never reach traces + assert "sig-secret" not in json.dumps(parts) and "enc-secret" not in json.dumps(parts) + + # input-history projection delegates to the same function, so it carries thinking too + input_messages = trace_input_messages_from_entries([UserEntry(content="hi"), entry]) + assert input_messages[1]["parts"][0] == {"type": "thinking", "content": "let me think"} + + +# --- supporting: model-name reasoning detection ------------------------------------------------ + + +@pytest.mark.parametrize( + ("model_name", "expected"), + [ + ("gpt-5-mini", True), + ("o3-mini", True), + ("o1", True), + ("gpt-5.1", True), + ("gpt-4.1-mini", False), + ("gpt-4o", False), + ("gpt-5-chat", False), + ("gpt-5.3-chat-latest", False), + ], +) +def test_openai_reasoning_detection(model_name: str, expected: bool) -> None: + assert _openai_supports_encrypted_reasoning(model_name) is expected + + +# --- guarded live suite (mirrors the plan §Tests smoke verification) ---------------------------- +# Gated behind real API keys; model names are env-overridable and default to reasoning-capable +# models. Each case captures native reasoning on a one-tool round-trip and resumes on the same +# provider/model, asserting the re-emitted native reasoning is accepted (the run completes). + + +def _multiply_tool() -> ToolSpec: + """A small arithmetic tool that nudges a reasoning model to actually reason.""" + return ToolSpec( + "multiply", + "Multiply two integers and return the product.", + {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"], "additionalProperties": False}, + lambda args: str(int(args["a"]) * int(args["b"])), + ) + + +def _has_signed_reasoning(state: dict) -> bool: + return any( + part.get("signature") + for entry in state["entries"] + if entry["role"] == "assistant" + for part in entry["reasoning"] + ) + + +async def _run_reasoning_resume_live(tmp_path: Path, make_model) -> None: + """Capture native reasoning, then resume on the same provider/model and assert acceptance.""" + first = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=make_model(), tools=[_multiply_tool()]).run( + "Use the multiply tool to compute 21 times 19, then state the product." + ) + state = json.loads(json.dumps(first.resume_state)) + assert _has_signed_reasoning(state), "no signed native reasoning captured in resume_state" + + second = await Harness(HarnessConfig(root=tmp_path, builtin_tools=[]), model=make_model(), tools=[_multiply_tool()]).run( + "Add 100 to that product.", resume_from=state + ) + assert second.text + + +@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OPENAI_API_KEY is not set") +async def test_openai_reasoning_resume_live(tmp_path: Path) -> None: + provider = OpenAIProvider() + name = os.getenv("THINHARNESS_LIVE_OPENAI_REASONING_MODEL", "gpt-5-mini") + settings = ModelSettings(extra_body={"reasoning": {"effort": "low"}}) + try: + await _run_reasoning_resume_live(tmp_path, lambda: OpenAIResponsesModel(name, provider=provider, settings=settings)) + finally: + await provider.aclose() + + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY is not set") +async def test_anthropic_reasoning_resume_live(tmp_path: Path) -> None: + provider = AnthropicProvider() + name = os.getenv("THINHARNESS_LIVE_ANTHROPIC_REASONING_MODEL", "claude-sonnet-4-5") + settings = ModelSettings(extra_body={"thinking": {"type": "enabled", "budget_tokens": 1024}}) + try: + await _run_reasoning_resume_live(tmp_path, lambda: AnthropicMessagesModel(name, provider=provider, settings=settings, max_tokens=2048)) + finally: + await provider.aclose() + + +@pytest.mark.skipif(not os.getenv("OPENROUTER_API_KEY"), reason="OPENROUTER_API_KEY is not set") +@pytest.mark.parametrize( + ("env_var", "default"), + [ + ("THINHARNESS_LIVE_OPENROUTER_REASONING_MODEL", "openai/gpt-5-mini"), + ("THINHARNESS_LIVE_OPENROUTER_REASONING_TEXT_MODEL", "anthropic/claude-sonnet-4.5"), + ], +) +async def test_openrouter_reasoning_resume_live(tmp_path: Path, env_var: str, default: str) -> None: + provider = OpenRouterProvider() + name = os.getenv(env_var, default) + settings = ModelSettings(extra_body={"reasoning": {"effort": "low"}}) + try: + await _run_reasoning_resume_live(tmp_path, lambda: OpenRouterModel(name, provider=provider, settings=settings)) + finally: + await provider.aclose() diff --git a/tests/test_resume.py b/tests/test_resume.py index 8fc9827..1b214ec 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -63,7 +63,7 @@ async def test_openai_resume_full_replays_transcript_for_followup(tmp_path: Path second = await harness.run("follow-up", resume_from=state) assert first.resume_state["kind"] == "transcript" - assert first.resume_state["version"] == 2 + assert first.resume_state["version"] == 3 assert first.resume_state["origin_provider"] == "openai" assert first.resume_state["origin_model"] == "gpt-test" assert [entry["role"] for entry in first.resume_state["entries"]] == ["user", "assistant", "tool", "assistant"] @@ -274,7 +274,7 @@ def harness() -> Harness: with pytest.raises(HarnessError, match="resume_from must be a dict"): harness().run_sync("follow-up", resume_from="resp_abc") # type: ignore[arg-type] - base_state = {"kind": "transcript", "version": 2, "origin_provider": "anthropic", "origin_model": "claude-test"} + base_state = {"kind": "transcript", "version": 3, "origin_provider": "anthropic", "origin_model": "claude-test"} with pytest.raises(HarnessError, match="resume_from kind None is not supported"): harness().run_sync("follow-up", resume_from={"version": 2, "origin_provider": "anthropic", "origin_model": "claude-test", "entries": []}) with pytest.raises(HarnessError, match="missing required field: 'entries'"): @@ -284,9 +284,10 @@ def harness() -> Harness: with pytest.raises(HarnessError, match="entry 'user' has wrong keys"): harness().run_sync("follow-up", resume_from={**base_state, "entries": [{"role": "user", "content": "hi"}]}) with pytest.raises(HarnessError, match="assistant tool call has wrong type"): + bad_assistant = {"role": "assistant", "text": "", "tool_calls": [{"id": 1, "name": "x", "arguments": "{}"}], "reasoning": []} harness().run_sync( "follow-up", - resume_from={**base_state, "entries": [{"role": "assistant", "text": "", "tool_calls": [{"id": 1, "name": "x", "arguments": "{}"}]}]}, + resume_from={**base_state, "entries": [bad_assistant]}, ) with pytest.raises(HarnessError, match="JSON-serializable"): harness().run_sync( @@ -303,10 +304,15 @@ def test_anthropic_resume_rejects_non_json_tool_arguments() -> None: with pytest.raises(HarnessError, match="resume_from assistant tool call arguments must be JSON"): model.resume_session({ "kind": "transcript", - "version": 2, + "version": 3, "origin_provider": "openrouter", "origin_model": "openai/test", - "entries": [{"role": "assistant", "text": "", "tool_calls": [{"id": "call_1", "name": "echo", "arguments": "{bad"}]}], + "entries": [{ + "role": "assistant", + "text": "", + "tool_calls": [{"id": "call_1", "name": "echo", "arguments": "{bad"}], + "reasoning": [], + }], }) @@ -568,7 +574,7 @@ def test_adapter_validation_does_not_mutate_state_on_failure() -> None: with pytest.raises(HarnessError): model.resume_session({ "kind": "transcript", - "version": 2, + "version": 3, "origin_provider": "anthropic", "origin_model": "claude-test", "entries": [{"role": "user", "content": "missing notice flag"}], diff --git a/thinharness/projections.py b/thinharness/projections.py index 42ab534..d497c23 100644 --- a/thinharness/projections.py +++ b/thinharness/projections.py @@ -93,6 +93,10 @@ def trace_input_messages_from_entries(entries: list[TranscriptEntry]) -> list[Js def trace_output_messages_from_assistant(entry_or_turn: AssistantEntry | ModelTurn) -> list[Json]: """Project a neutral assistant entry or model turn into OTel-style output messages.""" parts: list[Json] = [] + for reasoning in entry_or_turn.reasoning: + # OTel GenAI `thinking` part carries only text; opaque signatures/blobs are never traced. + if reasoning.text: + parts.append({"type": "thinking", "content": reasoning.text}) if entry_or_turn.text: parts.append({"type": "text", "content": entry_or_turn.text}) for call in entry_or_turn.tool_calls: diff --git a/thinharness/providers.py b/thinharness/providers.py index 8865be9..6dddf97 100644 --- a/thinharness/providers.py +++ b/thinharness/providers.py @@ -28,6 +28,22 @@ class ModelToolCall: arguments: str +@dataclass +class ReasoningPart: + """Provider-neutral carrier for one native reasoning block. + + The opaque blob is re-emitted natively only when ``provider_name`` matches the + resuming provider; otherwise ``text`` is replayed as a leading ```` block. + """ + + text: str = "" # plain reasoning text — always kept; cross-provider fallback + signature: str | None = None # opaque blob: Anthropic signature / redacted data, + # OpenAI encrypted_content, OpenRouter signature|data + id: str | None = None # provider reasoning-item id (OpenAI rs_…; "redacted_thinking" marker) + provider_name: str | None = None # origin provider prefix; native re-emit only when this matches + provider_details: Json | None = None # spillover: OpenAI summary raw_content; OpenRouter raw reasoning_details entry + + @dataclass class ModelTurn: """A normalized model response turn.""" @@ -35,6 +51,7 @@ class ModelTurn: text: str = "" tool_calls: list[ModelToolCall] = field(default_factory=list) raw: Json = field(default_factory=dict) + reasoning: list[ReasoningPart] = field(default_factory=list) finalized_output_mode: str | None = None @@ -44,6 +61,7 @@ class AssistantEntry: text: str tool_calls: list[ModelToolCall] + reasoning: list[ReasoningPart] = field(default_factory=list) @dataclass @@ -210,10 +228,12 @@ def __init__(self, message: str, *, status_code: int | None = None) -> None: _TRANSCRIPT_RESUME_KEYS = frozenset({"kind", "version", "origin_provider", "origin_model", "entries"}) _TRANSCRIPT_ENTRY_KEYS = { - "assistant": frozenset({"role", "text", "tool_calls"}), + "assistant": frozenset({"role", "text", "tool_calls", "reasoning"}), "user": frozenset({"role", "content", "notice"}), "tool": frozenset({"role", "call_id", "output"}), } +_REASONING_PART_KEYS = frozenset({"text", "signature", "id", "provider_name", "provider_details"}) +_TRANSCRIPT_VERSION = 3 def _validate_resume_state(state: dict[str, Any]) -> list[TranscriptEntry]: @@ -226,8 +246,8 @@ def _validate_resume_state(state: dict[str, Any]) -> list[TranscriptEntry]: raise HarnessError("resume_from must be JSON-serializable") from exc if state.get("kind") != "transcript": raise HarnessError(f"resume_from kind {state.get('kind')!r} is not supported; regenerate resume_state") - if state.get("version") != 2: - raise HarnessError(f"resume_from version {state.get('version')!r} is not supported") + if state.get("version") != _TRANSCRIPT_VERSION: + raise HarnessError(f"resume_from version {state.get('version')!r} is not supported; regenerate resume_state") unknown = set(state) - _TRANSCRIPT_RESUME_KEYS if unknown: raise HarnessError(f"resume_from has unknown keys: {sorted(unknown)!r}") @@ -243,7 +263,7 @@ def _transcript_state(*, model: Model, entries: list[TranscriptEntry]) -> dict[s """Return the neutral transcript resume envelope.""" return { "kind": "transcript", - "version": 2, + "version": _TRANSCRIPT_VERSION, "origin_provider": provider_prefix(model.provider.name), "origin_model": model.model, "entries": [_transcript_entry_to_dict(entry) for entry in entries], @@ -262,9 +282,23 @@ def _transcript_entry_to_dict(entry: TranscriptEntry) -> Json: {"id": call.id, "name": call.name, "arguments": call.arguments} for call in entry.tool_calls ], + "reasoning": [_reasoning_part_to_dict(part) for part in entry.reasoning], } +def _reasoning_part_to_dict(part: ReasoningPart) -> Json: + data: Json = {"text": part.text} + if part.signature is not None: + data["signature"] = part.signature + if part.id is not None: + data["id"] = part.id + if part.provider_name is not None: + data["provider_name"] = part.provider_name + if part.provider_details is not None: + data["provider_details"] = part.provider_details + return data + + def _transcript_entry_from_dict(value: Any) -> TranscriptEntry: if not isinstance(value, dict): raise HarnessError("resume_from entries must be dicts") @@ -281,11 +315,32 @@ def _transcript_entry_from_dict(value: Any) -> TranscriptEntry: if not isinstance(value["call_id"], str) or not isinstance(value["output"], str): raise HarnessError("resume_from tool entry has wrong type") return ToolResultEntry(call_id=value["call_id"], output=value["output"]) - if not isinstance(value["text"], str) or not isinstance(value["tool_calls"], list): + if not isinstance(value["text"], str) or not isinstance(value["tool_calls"], list) or not isinstance(value["reasoning"], list): raise HarnessError("resume_from assistant entry has wrong type") return AssistantEntry( text=value["text"], tool_calls=[_model_tool_call_from_dict(call) for call in value["tool_calls"]], + reasoning=[_reasoning_part_from_dict(part) for part in value["reasoning"]], + ) + + +def _reasoning_part_from_dict(value: Any) -> ReasoningPart: + if not isinstance(value, dict): + raise HarnessError("resume_from reasoning part must be a dict") + unknown = set(value) - _REASONING_PART_KEYS + if unknown: + raise HarnessError(f"resume_from reasoning part has unknown keys: {sorted(unknown)!r}") + if not isinstance(value.get("text"), str): + raise HarnessError("resume_from reasoning part text must be a string") + for key in ("signature", "id", "provider_name"): + if key in value and not isinstance(value[key], str): + raise HarnessError(f"resume_from reasoning part {key!r} must be a string") + return ReasoningPart( + text=value["text"], + signature=value.get("signature"), + id=value.get("id"), + provider_name=value.get("provider_name"), + provider_details=value.get("provider_details"), ) @@ -304,7 +359,13 @@ def _append_tool_results(transcript: list[TranscriptEntry], outputs: list[ToolOu def _append_assistant_turn(transcript: list[TranscriptEntry], turn: ModelTurn) -> None: - transcript.append(AssistantEntry(text=turn.text, tool_calls=copy.deepcopy(turn.tool_calls))) + transcript.append( + AssistantEntry( + text=turn.text, + tool_calls=copy.deepcopy(turn.tool_calls), + reasoning=copy.deepcopy(turn.reasoning), + ) + ) def _validate_anthropic_tool_arguments(entries: list[TranscriptEntry]) -> None: @@ -462,6 +523,22 @@ async def create_chat_completion(self, payload: Json) -> Json: # ============================================================================= +def _openai_supports_encrypted_reasoning(model_name: str) -> bool: + """Whether an OpenAI model returns encrypted reasoning content. + + Mirrors pydantic-ai's profile detection (``profiles/openai.py``): only reasoning + models accept ``include=["reasoning.encrypted_content"]``; non-reasoning models 400. + Like pydantic-ai, this enumerates known families explicitly, so a future reasoning + family must be added here (until then it degrades to text on resume rather than 400). + """ + is_gpt_5_1_plus = model_name.startswith(("gpt-5.1", "gpt-5.2", "gpt-5.3", "gpt-5.4", "gpt-5.5")) + is_gpt_5 = model_name.startswith("gpt-5") and not is_gpt_5_1_plus + is_o_series = model_name.startswith("o") + is_gpt_5_3_chat = model_name.startswith("gpt-5.3-chat") + thinking_always_enabled = is_o_series or (is_gpt_5 and "-chat" not in model_name) + return (thinking_always_enabled or is_gpt_5_1_plus) and not is_gpt_5_3_chat + + class OpenAIResponsesModel: """Responses-like model implemented with OpenAI Responses.""" @@ -501,6 +578,8 @@ def build_payload( ) -> Json: """Build a Responses API payload.""" payload: Json = {"model": self.model, "input": input_payload, "tools": tools} + if _openai_supports_encrypted_reasoning(self.model): + payload["include"] = ["reasoning.encrypted_content"] if instructions: payload["instructions"] = instructions if metadata: @@ -639,14 +718,19 @@ async def _complete(self, payload: Json) -> ModelTurn: """Send a Responses API payload and normalize the response.""" response = await self.model.provider.create_response(payload) self.previous_response_id = response.get("id") or self.previous_response_id - turn = ModelTurn(text=_extract_responses_text(response), tool_calls=_extract_responses_tool_calls(response), raw=response) + turn = ModelTurn( + text=_extract_responses_text(response), + tool_calls=_extract_responses_tool_calls(response), + reasoning=_extract_responses_reasoning(response), + raw=response, + ) _append_assistant_turn(self.transcript, turn) return turn def _prepend_replay(self, input_payload: str | list[Json]) -> str | list[Json]: if self._pending_replay is None: return input_payload - replay = _render_openai_transcript(self._pending_replay) + replay = _render_openai_transcript(self._pending_replay, encrypted_reasoning_ok=_openai_supports_encrypted_reasoning(self.model.model)) self._pending_replay = None if isinstance(input_payload, str): return [*replay, _openai_user_item(input_payload)] @@ -806,7 +890,12 @@ async def _complete(self, *, tools: list[Json], metadata: Json | None = None) -> payload.update(self.model.settings.extra_body) response = await self.model.provider.create_message(payload) self.messages.append({"role": "assistant", "content": response.get("content", [])}) - turn = ModelTurn(text=_extract_anthropic_text(response), tool_calls=_extract_anthropic_tool_calls(response), raw=response) + turn = ModelTurn( + text=_extract_anthropic_text(response), + tool_calls=_extract_anthropic_tool_calls(response), + reasoning=_extract_anthropic_reasoning(response), + raw=response, + ) _append_assistant_turn(self.transcript, turn) return turn @@ -814,7 +903,10 @@ def _apply_resume(self, instructions: str | None) -> None: if self._resume_entries is None: return self.system = instructions or "" - self.messages = _render_anthropic_transcript(self._resume_entries) + self.messages = _render_anthropic_transcript( + self._resume_entries, + thinking_enabled=_anthropic_thinking_enabled(self.model.settings), + ) self._resume_entries = None @@ -964,7 +1056,12 @@ async def _complete( response = await self.model.provider.create_chat_completion(payload) message = ((response.get("choices") or [{}])[0].get("message") or {}) self.messages.append(message) - turn = ModelTurn(text=str(message.get("content") or ""), tool_calls=_extract_chat_tool_calls(message), raw=response) + turn = ModelTurn( + text=str(message.get("content") or ""), + tool_calls=_extract_chat_tool_calls(message), + reasoning=_extract_openrouter_reasoning(message), + raw=response, + ) _append_assistant_turn(self.transcript, turn) return turn @@ -1057,7 +1154,18 @@ def append_notices_to_text(text: str, notices: list[ModelNotice] | None) -> str: return text if not notice_text else f"{text}\n\n{notice_text}" -def _render_anthropic_transcript(entries: list[TranscriptEntry]) -> list[Json]: +def _thinking_fallback(text: str) -> str: + """Render reasoning text as a degraded cross-provider thinking block.""" + return f"\n{text}\n" + + +def _anthropic_thinking_enabled(settings: ModelSettings) -> bool: + """Return whether the resuming Anthropic request enables extended thinking.""" + thinking = settings.extra_body.get("thinking") + return isinstance(thinking, dict) and thinking.get("type") == "enabled" + + +def _render_anthropic_transcript(entries: list[TranscriptEntry], *, thinking_enabled: bool = False) -> list[Json]: """Render neutral transcript entries as Anthropic Messages history.""" messages: list[Json] = [] index = 0 @@ -1072,6 +1180,14 @@ def _render_anthropic_transcript(entries: list[TranscriptEntry]) -> list[Json]: continue if isinstance(entry, AssistantEntry): content: list[Json] = [] + for part in entry.reasoning: + if thinking_enabled and part.provider_name == "anthropic" and part.signature: + if part.id == "redacted_thinking": + content.append({"type": "redacted_thinking", "data": part.signature}) + else: + content.append({"type": "thinking", "thinking": part.text, "signature": part.signature}) + elif part.text: + content.append({"type": "text", "text": _thinking_fallback(part.text)}) if entry.text: content.append({"type": "text", "text": entry.text}) content.extend({ @@ -1108,8 +1224,21 @@ def _render_openrouter_transcript(entries: list[TranscriptEntry]) -> list[Json]: messages.append({"role": "tool", "tool_call_id": entry.call_id, "content": entry.output}) else: message: Json = {"role": "assistant"} - if entry.text: - message["content"] = entry.text + reasoning_details = [ + part.provider_details + for part in entry.reasoning + if part.provider_name == "openrouter" and part.provider_details is not None + ] + fallback_blocks = [ + _thinking_fallback(part.text) + for part in entry.reasoning + if not (part.provider_name == "openrouter" and part.provider_details is not None) and part.text + ] + if reasoning_details: + message["reasoning_details"] = reasoning_details + text = "\n\n".join([*fallback_blocks, *([entry.text] if entry.text else [])]) + if text: + message["content"] = text if entry.tool_calls: message["tool_calls"] = [ { @@ -1119,7 +1248,7 @@ def _render_openrouter_transcript(entries: list[TranscriptEntry]) -> list[Json]: } for call in entry.tool_calls ] - if not entry.text and not entry.tool_calls: + if not text and not entry.tool_calls: message["content"] = "" messages.append(message) return messages @@ -1130,7 +1259,7 @@ def _openai_user_item(text: str) -> Json: return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} -def _render_openai_transcript(entries: list[TranscriptEntry]) -> list[Json]: +def _render_openai_transcript(entries: list[TranscriptEntry], *, encrypted_reasoning_ok: bool = False) -> list[Json]: """Render neutral transcript entries as Responses API input items.""" items: list[Json] = [] for entry in entries: @@ -1139,6 +1268,11 @@ def _render_openai_transcript(entries: list[TranscriptEntry]) -> list[Json]: elif isinstance(entry, ToolResultEntry): items.append({"type": "function_call_output", "call_id": entry.call_id, "output": entry.output}) else: + for part in entry.reasoning: + if encrypted_reasoning_ok and part.provider_name == "openai" and part.signature and part.id: + items.append({"type": "reasoning", "id": part.id, "encrypted_content": part.signature, "summary": []}) + elif part.text: + items.append({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": _thinking_fallback(part.text)}]}) if entry.text: items.append({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": entry.text}]}) items.extend({ @@ -1229,3 +1363,51 @@ def _extract_chat_tool_calls(message: Json) -> list[ModelToolCall]: function = call.get("function") or {} calls.append(ModelToolCall(id=str(call.get("id")), name=str(function.get("name")), arguments=function.get("arguments") or "{}")) return calls + + +def _extract_responses_reasoning(response: Json) -> list[ReasoningPart]: + """Extract native reasoning items from a Responses API response.""" + parts: list[ReasoningPart] = [] + for item in response.get("output", []) or []: + if not isinstance(item, dict) or item.get("type") != "reasoning": + continue + summary = item.get("summary") or [] + text = "".join(str(chunk.get("text", "")) for chunk in summary if isinstance(chunk, dict)) + content = item.get("content") + parts.append(ReasoningPart( + text=text, + signature=item.get("encrypted_content"), + id=item.get("id"), + provider_name="openai", + provider_details={"raw_content": content} if content is not None else None, + )) + return parts + + +def _extract_anthropic_reasoning(response: Json) -> list[ReasoningPart]: + """Extract native thinking blocks from an Anthropic response.""" + parts: list[ReasoningPart] = [] + for block in response.get("content", []) or []: + if not isinstance(block, dict): + continue + if block.get("type") == "thinking": + parts.append(ReasoningPart(text=str(block.get("thinking", "")), signature=block.get("signature"), provider_name="anthropic")) + elif block.get("type") == "redacted_thinking": + parts.append(ReasoningPart(text="", signature=block.get("data"), id="redacted_thinking", provider_name="anthropic")) + return parts + + +def _extract_openrouter_reasoning(message: Json) -> list[ReasoningPart]: + """Extract native reasoning_details from an OpenRouter chat message.""" + parts: list[ReasoningPart] = [] + for entry in message.get("reasoning_details") or []: + if not isinstance(entry, dict): + continue + parts.append(ReasoningPart( + text=str(entry.get("text") or ""), + signature=entry.get("signature") or entry.get("data"), + id=entry.get("id"), + provider_name="openrouter", + provider_details=entry, + )) + return parts From ec73af765b408d01970ef1ccd8e38857379ac835 Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 23 Jun 2026 00:32:28 -0400 Subject: [PATCH 4/5] docs: document same-provider reasoning fidelity and resume state version 3 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_016zFdrr1esQDRzYJEEhtkxG --- docs/behavior.md | 7 ++++--- docs/docs.md | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/behavior.md b/docs/behavior.md index f0a0274..3fefadd 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -73,10 +73,11 @@ Built-in provider resume state is a self-contained, provider-agnostic transcript - RESUME-1: `resume_state` is a provider-agnostic transcript; resume across built-in providers and across built-in models is supported. - RESUME-2: `resume_state` is self-contained and does not depend on provider continuation tokens such as OpenAI `previous_response_id`; an OpenAI run that never received a response id is still resumable. -- RESUME-3: Resume state version 2 preserves reasoning as visible transcript text only and does not preserve provider-specific reasoning chains. -- RESUME-4: Built-in provider resume state uses `version` 2; version 1 state and old provider-native `kind` values are rejected with a regenerate error. +- RESUME-3: Resuming on the originating provider preserves native reasoning (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`); resuming on a different provider degrades each reasoning part to a leading ``-tagged text block and drops the opaque blob. Native re-emit additionally requires the resuming run to be able to accept the block: OpenAI re-emits the native reasoning item only when the resuming model is reasoning-capable, and Anthropic only when extended thinking is enabled; otherwise both use the text fallback. So a reasoning-model capture resumed on a non-reasoning model of the same provider degrades to text. +- RESUME-4: Built-in provider resume state uses `version` 3; version 1 and version 2 state and old provider-native `kind` values are rejected with a regenerate error. - RESUME-5: On resume, the live system prompt from the resuming harness config is re-injected; captured system prompts are not stored or restored. -- RESUME-6: A session seeded via `OpenAIResponsesSession.start(previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. +- RESUME-6: A session seeded via `OpenAIResponsesSession.start(previous_response_id=...)` captures only new transcript entries, so externally seeded prior turns are not present when later resumed from `resume_state`. This is unrelated to reasoning fidelity and is not changed by RESUME-3/RESUME-7. +- RESUME-7: For reasoning-capable OpenAI Responses models the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; non-reasoning models are unaffected. Captured `resume_state` therefore contains encrypted reasoning blobs (OpenAI/OpenRouter) and signed thinking (Anthropic) and should be treated as sensitive, consistent with the local-trace sensitivity note. ## Model Observability Projections diff --git a/docs/docs.md b/docs/docs.md index 98583fe..6dfe3b4 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -572,9 +572,9 @@ Budgets span the pause. The paused batch counts against `usage.tool_calls` exact Built-in provider resume details: -- `resume_state["kind"] == "transcript"` and `version == 2`. +- `resume_state["kind"] == "transcript"` and `version == 3`. - The transcript is provider-agnostic and no longer depends on OpenAI server-side response retention. -- Provider-specific reasoning chains are not preserved; visible reasoning text, ordinary assistant text, tool calls, user messages, tool results, and harness notices are replayed. +- Provider-specific reasoning chains are preserved on same-provider resume (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`) and degraded to a leading ``-tagged text block on cross-provider resume. Anthropic native re-emit also requires extended thinking to be enabled in the resuming run. For reasoning-capable OpenAI models the harness adds `include=["reasoning.encrypted_content"]`, so `resume_state` can contain encrypted reasoning blobs — treat it as sensitive. - Cross-provider resume is supported by the built-in renderers, but real providers may reject foreign-format tool-call ids or malformed tool-call argument JSON. - `OpenAIResponsesSession.start(previous_response_id=...)` remains available as a low-level escape hatch, but later resume state captures only the new prompt onward, not the externally seeded prior turns. From b2722d134709c2f2750bcc80e507b8e02ac4f4da Mon Sep 17 00:00:00 2001 From: Ryan Brown Date: Tue, 23 Jun 2026 00:50:42 -0400 Subject: [PATCH 5/5] docs: update LOC table, add LongMemEval example page, and document reasoning fidelity Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01PFBTUTAZRRH4RtRkLnJWDv --- .plans/31-reasoning-fidelity.md | 147 ++++++++++++++++++++++++++++++++ README.md | 114 +++++++++---------------- docs/site/about/index.html | 49 +++++------ docs/site/assets/site.css | 24 ++++++ docs/site/examples/index.html | 38 +++++++-- docs/site/explainer/index.html | 36 +++++++- docs/site/index.html | 10 +-- examples/longmemeval.md | 33 +++++++ scripts/build_site.py | 47 +++++----- scripts/build_transcripts.py | 86 +++++++++++++++++-- 10 files changed, 438 insertions(+), 146 deletions(-) create mode 100644 .plans/31-reasoning-fidelity.md create mode 100644 examples/longmemeval.md diff --git a/.plans/31-reasoning-fidelity.md b/.plans/31-reasoning-fidelity.md new file mode 100644 index 0000000..9ff36a8 --- /dev/null +++ b/.plans/31-reasoning-fidelity.md @@ -0,0 +1,147 @@ +# Same-provider reasoning fidelity in the neutral transcript — plan v1 + +Capture native model reasoning into the neutral `Transcript` and re-emit it natively when a run is resumed on the **same provider**, so reasoning models do not lose their chain-of-thought across a resume boundary. Cross-provider resume degrades reasoning to text (unchanged intent of RESUME-3). + +This executes the item plan-29 (`.plans/29-unified-transcript-log.md`) explicitly deferred under "Out of scope": *"Same-provider reasoning fidelity — a `provider_extras` field stamped with `origin_provider`, re-emitted only on same-provider resume (Anthropic thinking `signature`, OpenAI `encrypted_content`)… v1 stores reasoning as text."* + +## Goal + +Plan-29 made `resume_state` a neutral transcript of `UserEntry`/`AssistantEntry`/`ToolResultEntry`. `AssistantEntry` keeps only `text` + `tool_calls`; `_append_assistant_turn` discards `turn.raw` (`providers.py:306-307`), which is the only place native reasoning lives. Consequences today: + +- **OpenAI Responses (sharpest regression).** Before plan-29, OpenAI `dump_state` serialized just `previous_response_id`; on resume OpenAI rebuilt the full prior conversation server-side **including reasoning items**. Plan-29 switched to self-contained client-side full replay with **no** `previous_response_id` and a transcript that carries no reasoning items → reasoning is dropped on every resume of a reasoning model. The retention-independence win was real; the casualty was the server-held reasoning. +- **OpenRouter.** `reasoning_details` arrive on the raw assistant message and are kept in-run via `self.messages`, but never reach the transcript → lost on resume. +- **Anthropic.** Only relevant once extended thinking is enabled (the harness does not enable it by default; see §6). When enabled, thinking blocks + signatures are kept in-run via `self.messages` but lost on resume. + +After this change a run captured and resumed on the same provider preserves native reasoning; a run resumed on a different provider keeps the reasoning **text** (a leading ``-tagged block) and drops the opaque blob. + +## What this is and is not + +**Is:** an additive `reasoning` field on `AssistantEntry`, populated from each turn's raw response and rendered back natively only when the resuming provider matches the originating provider. + +**Is not:** a change to the in-run request/response normalization (`ModelTurn.text`/`tool_calls` are unchanged), and **not** a cross-provider reasoning translator. The neutral part is an opaque carrier gated by provenance, exactly as pydantic-ai's `ThinkingPart` does. + +## Live API verification (done — gates the residual risk) + +The plan-29 residual risk was "real-provider divergence from fakes." The happy path (one tool round-trip, a multiply tool, low reasoning effort) was verified against all three live APIs on 2026-06-22: + +- **OpenAI** (`gpt-5-mini`): with `store` defaulted true (harness in-run mode) **plus** `include=["reasoning.encrypted_content"]`, the response's `reasoning` item carries `encrypted_content`. In-run `previous_response_id` chaining still works alongside `include`. A reasoning blob captured under `store=true` **replays statelessly** — full `input` `[reasoning, function_call, function_call_output]`, `store=false`, **no** `previous_response_id` → **200**, terse cached-reasoning answer. Replaying with `store` defaulted true (no `previous_response_id`) also → 200. Dropping the reasoning item also → 200 (model re-reasons). +- **Anthropic** (`claude-sonnet-4-6`, `thinking={type:enabled,budget_tokens:1024}`): T1 returns `[thinking{thinking,signature}, text, tool_use]`. Reconstructing the assistant message as `[thinking(signature), text, tool_use]` + a user `tool_result` → **200**. (Missing thinking block also accepted here, but native preservation is the goal.) +- **OpenRouter** (`reasoning={effort:"low"}`): `openai/gpt-5-mini` returns `reasoning_details=[{type:"reasoning.encrypted", data:"…"}]`; `anthropic/claude-sonnet-4.5` returns `[{type:"reasoning.text", text, format:"anthropic-claude-v1", index, signature}]`. Echoing `reasoning_details` verbatim on the assistant message + a `role:tool` result → **200** for both. + +Smoke scripts are reproducible from `.env` keys; see §Tests for the cases to keep as a guarded live suite. **Not yet verified** (residual risks below): redacted_thinking, multiple reasoning items per turn, interleaved-thinking constraints, long conversations, and signature acceptance across a model-version change. + +## Design + +### 1. Neutral `ReasoningPart` + `AssistantEntry.reasoning` (`providers.py`, leaf types) + +```python +@dataclass +class ReasoningPart: + text: str = "" # plain reasoning text — always kept; cross-provider fallback + signature: str | None = None # opaque blob: Anthropic signature / redacted data, + # OpenAI encrypted_content, OpenRouter signature|data + id: str | None = None # provider reasoning-item id (OpenAI rs_…; "redacted_thinking" marker) + provider_name: str | None = None # origin provider prefix; native re-emit only when this matches + provider_details: Json | None = None # spillover: OpenAI summary raw_content; OpenRouter raw reasoning_details entry + +@dataclass +class AssistantEntry: + text: str + tool_calls: list[ModelToolCall] + reasoning: list[ReasoningPart] = field(default_factory=list) # additive, defaulted +``` + +A list because a turn can carry several reasoning parts (Anthropic thinking + redacted_thinking; OpenAI multiple summaries sharing one id; OpenRouter multiple `reasoning_details` entries). + +`ModelTurn` gains `reasoning: list[ReasoningPart] = field(default_factory=list)`, populated in each `_complete` from `turn.raw`; `_append_assistant_turn` copies `turn.reasoning` (deep-copied) into the `AssistantEntry`. + +**Provenance is per-part, not per-envelope.** A transcript can accumulate entries from multiple providers after cross-provider resumes, so the gate keys on `ReasoningPart.provider_name`, not the envelope's `origin_provider`. This mirrors pydantic-ai's `provider_name == self.system` gate (`models/anthropic.py:1348`, `models/openai.py:3044`). + +### 2. Serialization (load-bearing) + +- Add `ReasoningPart` to/from dict; `signature`/`id`/`provider_name`/`provider_details` are `None`-omitted-or-present per the existing explicit style. +- Extend the assistant branch of `_transcript_entry_to_dict`/`_transcript_entry_from_dict` (`providers.py:258-289`) to include `"reasoning": [ … ]`. +- **`_TRANSCRIPT_ENTRY_KEYS["assistant"]` (`providers.py:213`) must gain `"reasoning"`.** The validator does strict `set(value) != _TRANSCRIPT_ENTRY_KEYS[role]` (`:274`), so an assistant entry without the key would fail — hence the version bump (§4) and: `dump_state` always emits `"reasoning"` (empty list when none), keeping the key-set exact. +- Round-trips through `json.loads(json.dumps(...))` like every other entry, so it is also persisted incrementally by the transcript-delta tracing path (`tracing.py:467`). + +### 3. Per-provider capture (IN) and native re-emit (OUT) + +The gate, applied in every `_render_*_transcript` when rendering an `AssistantEntry.reasoning[i]`: + +``` +native_ok = part.provider_name == and and +if native_ok: emit native reasoning block +elif part.text: emit a leading "\n{part.text}\n" block (text fallback) +else: drop +``` + +The text fallback is emitted as a leading content block (Anthropic `text` / OpenRouter `content` / OpenAI `output_text` message), **before** the assistant text and tool calls, matching pydantic-ai's `thinking_tags` degradation. + +**OpenAI Responses.** +- *Capture:* add `include=["reasoning.encrypted_content"]` to `build_payload` (`providers.py:493-513`) so every response's `reasoning` items carry `encrypted_content` even under in-run `store=true` chaining (verified). In `_complete`, extract items where `type == "reasoning"`: `ReasoningPart(text=joined summary text or "", signature=encrypted_content, id=rs_id, provider_name="openai", provider_details={"raw_content": [...]} if present)`. +- *Re-emit:* in `_render_openai_transcript` (`providers.py:1133`), for a matching part emit `{"type":"reasoning","id":part.id,"encrypted_content":part.signature,"summary":[]}` **before** the assistant `output_text` message and the `function_call` items. The resume path already sends no `previous_response_id`; default `store` is accepted for the replay (verified), so **no `store` toggle is required** (optionally set `store=false` for ZDR — call-out, not a requirement). + +**Anthropic Messages.** +- *Capture:* in `_complete` (`providers.py:793`), extract from `response["content"]`: `thinking` → `ReasoningPart(text=thinking, signature=signature, provider_name="anthropic")`; `redacted_thinking` → `ReasoningPart(text="", signature=data, id="redacted_thinking", provider_name="anthropic")`. +- *Re-emit:* in `_render_anthropic_transcript` (`providers.py:1060`), **prepend** reasoning blocks to the assistant `content` (before the optional text block and the `tool_use` blocks — Anthropic requires thinking first): `{"type":"thinking","thinking":part.text,"signature":part.signature}`, or `{"type":"redacted_thinking","data":part.signature}` when `id == "redacted_thinking"`. +- *Constraint:* Anthropic only accepts thinking blocks when the **resuming** request enables thinking. Gate the native emit additionally on "thinking enabled for this run" (derived from `self.model.settings.extra_body`); otherwise use the text fallback. Because the harness does not enable thinking by default (no `thinking` key is sent anywhere today), Anthropic native preservation is effectively inert until a caller turns thinking on — consistent with plan-29's note that extended thinking is out of scope. The in-run block-ordering constraints for live extended thinking remain out of scope (deferred in plan-29). + +**OpenRouter.** +- *Capture:* in `_complete` (`providers.py:944`), read `message.get("reasoning_details")`. Store each entry faithfully: `ReasoningPart(text=entry.get("text",""), signature=entry.get("signature") or entry.get("data"), id=entry.get("id"), provider_name="openrouter", provider_details=entry)` — keeping the full raw entry in `provider_details` so the self-describing OpenRouter shape (`type`/`format`/`index`) round-trips exactly. +- *Re-emit:* in `_render_openrouter_transcript` (`providers.py:1101`), reattach `message["reasoning_details"] = [part.provider_details for matching parts]` on the assistant message (verbatim from `provider_details`). OpenRouter accepted both encrypted and text forms echoed verbatim (verified). + +### 4. Version + validator coupling + +- **`version` bumps `2` → `3`.** The assistant entry shape changed (new required `reasoning` key in the exact-key-set check). Per plan-08/plan-29's contract a shape change bumps the version. Greenfield (no deployed persisted state); reject old `version: 2` and `version: 1` plus old `kind` values with the existing **`"resume_from"`-prefixed** `HarnessError` instructing regeneration (`providers.py:227-230`). +- The `_resume_approval_session` relabel still keys on the `"resume_from"` prefix (`core.py:724-728`, `test_approvals.py:733`); keep it. +- `_validate_anthropic_tool_arguments` (`providers.py:310`) is unaffected (reasoning carries no tool-arg JSON). + +### 5. Capability/boundary parity (unchanged) + +`resume_kind`, both capability gates (`core.py:460`, `:1135`), and the custom-`ResumableModel` boundary (custom models keep their opaque protocol) are untouched. The neutral-reasoning contract binds only the three built-in providers and their real-provider-backed fakes. + +## Behavior changes (update `docs/behavior.md` after review, before implementation) + +- **Update RESUME-3** — was "v1 does not preserve provider-specific reasoning chains across resume (reasoning replays as text)." Now: *same-provider* resume preserves native reasoning (Anthropic thinking signatures, OpenAI `encrypted_content`, OpenRouter `reasoning_details`); *cross-provider* resume degrades reasoning to text. +- **Add RESUME-7** — for OpenAI Responses, the harness requests `include=["reasoning.encrypted_content"]` so reasoning survives resume; this is captured into `resume_state` (which therefore contains encrypted reasoning blobs — treat as sensitive, consistent with the existing local-trace sensitivity note). +- **RESUME-6 is unchanged and still applies** — the `start(previous_response_id=…)` escape hatch still loses externally-seeded prior turns on resume. This plan does **not** fix RESUME-6; state that explicitly so "reasoning regression fixed" is not misread as "all resume regressions fixed." +- `version` is now `3` (extends RESUME-4's regenerate-on-old-state rule). + +## Implementation steps + +1. **`providers.py`** — add `ReasoningPart`; add `reasoning` to `AssistantEntry` and `ModelTurn`; `ReasoningPart` (de)serialization; extend assistant entry (de)serialization + `_TRANSCRIPT_ENTRY_KEYS["assistant"]`; bump validator to `version == 3` (reject 1/2 + old `kind`, keep `"resume_from"` prefix); per-provider capture in the three `_complete`s; `_append_assistant_turn` copies `turn.reasoning`; OpenAI `include` in `build_payload`; native re-emit + text fallback in the three `_render_*_transcript`s with the per-part provenance gate (Anthropic also gated on thinking-enabled). +2. **`docs/behavior.md`** — update RESUME-3, add RESUME-7, restate RESUME-6 scope, note `version: 3` (after review). +3. **`README.md` / `docs/docs.md`** — update the Resume section: line 303 ("Provider-specific reasoning chains are not preserved") becomes "preserved on same-provider resume, degraded to text cross-provider"; note the OpenAI `include`/sensitivity point. +4. **Tests** — unit (fakes) + the guarded live suite (§Tests). +5. **Run** `uv run pyright`, ruff, full pytest (project `CLAUDE.md`). + +## Tests + +Real-provider-backed fakes (`FakeClient`/`FakeAnthropicProvider`/`FakeOpenRouterProvider`) only; leave scripted/sequence fakes unchanged (per plan-29 §Tests). Fakes must be extended to emit reasoning in their raw responses (OpenAI `reasoning` item w/ `encrypted_content`; Anthropic `thinking` block w/ `signature`; OpenRouter `reasoning_details`). + +Unit cases: +1. **Capture** — each provider's `_complete` populates `AssistantEntry.reasoning` with the right `provider_name`, `signature`/`id`, and (OpenAI) `include` is present in the payload. +2. **Same-provider native re-emit** — resume on the same provider renders the native block (OpenAI `reasoning` item with `encrypted_content` ahead of the `function_call`, no `previous_response_id`; Anthropic `thinking{signature}` first in assistant content; OpenRouter `reasoning_details` reattached). +3. **Cross-provider text fallback** — capture on Anthropic-shaped fake, resume on OpenAI/OpenRouter-shaped fakes: assert a leading ``-tagged text block, no opaque blob, no foreign reasoning item. +4. **Anthropic thinking-disabled fallback** — same-provider resume but thinking not enabled in the resuming config → text fallback, not a `thinking` block. +5. **redacted_thinking** — Anthropic `redacted_thinking` round-trips to `{"type":"redacted_thinking","data":…}`. +6. **Multi-part turn** — ≥2 reasoning parts and ≥1 tool call render in the right order. +7. **Round-trip serialization** — `json.loads(json.dumps(dump_state()))` equals `dump_state()` for an assistant turn carrying reasoning; `version == 3`. +8. **Version rejection** — `version: 2` and `version: 1` state raise the `"resume_from"`-prefixed `HarnessError`. +9. **In-run guard** — Anthropic/OpenRouter in-run payloads stay byte-identical; OpenAI in-run payload changes **only** by the added `include` key (assert exactly that delta — the deliberate exception to plan-29's byte-identical guard). + +Live suite (guarded behind keys, mirrors the verified smoke tests; one tool round-trip each): OpenAI stateless reasoning replay; Anthropic reconstructed thinking(signature) acceptance; OpenRouter `reasoning_details` echo for one encrypted and one text model. Gate "same-provider reasoning preserved" on these passing. + +## Residual risks + +- **Verified only on the happy path.** redacted_thinking, multiple reasoning items per turn, interleaved-thinking ordering, long multi-turn conversations, and signature acceptance across a model-version change between capture and resume are **not** yet verified — add live cases or document as experimental. +- **`resume_state` now contains encrypted reasoning blobs** for OpenAI/OpenRouter (and signed thinking for Anthropic). Bigger payloads and sensitive content; documented in RESUME-7. +- **Anthropic native re-emit depends on thinking being enabled** in the resuming run; mismatched config silently uses the text fallback (intended, but worth a doc line). +- **OpenAI `include` changes every in-run payload** (the one accepted exception to plan-29's byte-identical guard). + +## Out of scope (deferred) + +- Cross-provider reasoning translation (kept as text by design). +- In-run extended-thinking block-ordering constraints (deferred in plan-29; revisit if the harness enables live thinking). +- RESUME-6 (`previous_response_id` escape-hatch seeding) — unrelated; not addressed here. +- OpenAI same-provider `previous_response_id` fast-path on resume (plan-29 chose uniform full replay). diff --git a/README.md b/README.md index 586b881..9dc0945 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ ThinHarness exists for the gap between building the agent loop yourself and adop It owns a focused set of agent-loop primitives that generalize well and are tedious to rebuild, leaving the rest of the application stack for you to own. -I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple but powerful, yet most frameworks don't include them out of the box, and the ones that do are among the largest and heaviest options (Claude Code -> Claude Agent SDK, LangChain -> deepagents, Agno). I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway. +I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple yet powerful, but you mostly get them by adopting a large framework with layers of abstraction. I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway.
@@ -49,39 +49,36 @@ I started building ThinHarness after running into this gap in practice. Filesyst LOC1 Tool
retries2 Subagents - Structured
output Skills FS
tools OTel
tracing - + ThinHarness - 8,241 - ✅ + 8,658 ✅ ✅ ✅ ✅ ✅ - + -  Claude Agent SDK3 +  Claude Agent SDK - 8,263 + 8,2633 ❌ ✅ - ❌ ✅ ✅ ⚠️ - + @@ -90,126 +87,118 @@ I started building ThinHarness after running into this gap in practice. Filesyst 9,840 ❌ ✅ - ✅ ❌ ❌ - ✅ + ⚠️ - + -  deepagents4 +  deepagents - 17,039 + 17,6644 ❌ ✅ - ❌ ✅ ✅ ❌ - +  AWS Strands - 28,157 + 32,526 ⚠️ ✅ ✅ ❌ - ❌ ✅ - +  Microsoft
Agent Framework - 40,514 + 41,331 ❌ ✅ ✅ ✅ - ❌ ✅ - +  Pydantic AI - 59,034 + 59,087 ✅ ❌ - ✅ ❌ ❌ ✅ - +  Google ADK - 64,890 + 65,799 ⚠️ ✅ ✅ ✅ - ❌ ✅ - +  OpenAI Agents SDK - 73,139 - ✅ - ✅ - ✅ - ❌ + 73,796 ❌ ✅ + ✅ + ⚠️ + ⚠️ - +  Agno - 111,539 + 113,477 ⚠️ ✅ ✅ ✅ - ✅ - ✅ + ⚠️ -

* Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

+

* Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, structured output, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.
2. Tool retries: a documented primitive (e.g. Pydantic AI's ModelRetry) that lets tools signal "model passed bad args — retry with this feedback," distinct from generic exception propagation.
3. Claude Agent SDK shells out to the Claude Code CLI binary, which is 200k+ LOC.
- 4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈111k LOC. + 4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈112k LOC.

@@ -226,13 +215,9 @@ ThinHarness has opinions. They are the reason it stays small. **No bash by default.** Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in `BashTool` for exploratory runs before the workflow is hardened with typed tools. -**Skills are tools, not auto-discovery.** Skills live in directories you point at explicitly. The agent calls `skill_read` and `skill_run` like any other tool. No interactive scan of the workspace, no global skill marketplace, no magic. SDK use is deliberate; the auto-discovery design is for interactive coding agents and doesn't belong here. - **Search is a top priority.** The `search` tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a `jsonl_search` variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, `where` filters, range filters, and snippets from large multiline fields. -**Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs reliability beyond a single agent loop — majority vote, ensembled extraction. Set `builtin_parallel_llm_model` to enable the default `parallel_llm` tool for plain-text batches; for validated structured output per call, instantiate `ParallelLlmTool` yourself with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. - -**Background tools are simple.** Some long-running tools can start in the background so the agent can keep working. There is no detached job queue, polling API, or job-control surface; the current run still owns the task, and the completion is sent back to the model when it finishes. +**Parallel LLM calls, built in.** Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Set `builtin_parallel_llm_model` to enable the default `parallel_llm` tool for plain-text batches; for validated structured output per call, instantiate `ParallelLlmTool` yourself with `output_type` (a Pydantic model). Each call is stateless, and large batches can write JSON to `output_file`. **No token streaming.** Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, background, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal. @@ -292,7 +277,7 @@ Streaming emits coarse run, model, tool, background, retry, limit, and subagent - **Subagents:** opt-in delegation through a built-in `subagent` tool and explicit `SubAgentConfig`. - **Parallel LLM:** opt-in `parallel_llm` fan-out for batches of independent one-shot prompts, plus `ParallelLlmTool(...).spec()` for renameable tools with explicit model, path, prompt, and retry settings. - **Skills:** explicit `skill_read` and `skill_run` tools for selected skill directories, with Python, shell, JavaScript, and Go script runners. -- **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models. +- **Resume:** clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers. - **MCP:** optional MCP client support with lazy tool discovery and collision checks. - **Parallel tool calls:** same-turn tool batches run concurrently when every called tool is parallel-safe. - **Background tools:** opt-in long-running tool calls return a start notice immediately, keep the agent loop moving, and deliver completion back to the model when ready. @@ -302,25 +287,6 @@ Streaming emits coarse run, model, tool, background, retry, limit, and subagent - **Limits and notices:** configured request, tool-call, output-retry, and tool-retry budgets bound each run; near-limit guidance can warn the model before request or tool-call budgets are exhausted. - **Tracing:** local plaintext JSONL traces plus OpenTelemetry-compatible spans for runs, provider calls, tools, and subagents. -## Resume - -Cleanly completed runs return `HarnessResult.resume_state`, a JSON-serializable transcript that can be passed back as `resume_from` with the next user message. - -```python -first = await harness.run("Summarize this repository.") -second = await harness.run("Now turn that into a checklist.", resume_from=first.resume_state) -``` - -For the built-in OpenAI, Anthropic, and OpenRouter adapters, resume state is self-contained and provider-agnostic. A run captured with one built-in provider or model can be replayed by another built-in provider or model, subject to provider wire-format acceptance for tool-call ids and argument JSON. The live harness config supplies the system prompt and tool schemas on resume; captured system prompts are not stored. - -Provider-specific reasoning chains are not preserved in version 2 resume state. OpenAI runs seeded manually with `previous_response_id` still work, but later resume state captures only the new transcript entries, not the externally seeded prior turns. - -## Tracing - -Local tracing is on by default. It writes full plaintext JSONL traces under `~/.thinharness/traces//`, including prompts, model outputs, tool arguments, and tool results, so treat that directory as sensitive local data. - -Set `local_tracing=False` or `THINHARNESS_DISABLE_LOCAL_TRACING=1` to disable local trace files. External tracing is generic OpenTelemetry: pass any tracer with `start_as_current_span(...)` or `start_span(...)` in `TracingOptions`, and each sink keeps its own capture policy. - ## Examples Three agents built on ThinHarness, from a self-contained demo to a benchmark run to one I use live. @@ -333,7 +299,7 @@ It isn't meant to be a state-of-the-art research agent; it's a worked example sh ### 2. LongMemEval-V2 Reproduction -I ran ThinHarness on a 127-question subset of a real long-term-memory retrieval benchmark, and did a local reproduction of the benchmark's optimized harness on the same subset. +I ran ThinHarness on a retrieval-heavy 127 question subset of a benchmark for long-term agent memory, and did a local reproduction of the benchmark's optimized harness on the same subset. - **Performance:** Matched-or-better accuracy (74.0% vs 72.4% on the 127 dynamic questions) with ~46% less token usage (62M vs. 116M). See [my fork](https://github.com/ryanbbrown/LongMemEval-V2) for more details. - **Simpler Setup:** ThinHarness only had its built-in filesystem tools (with `jsonl_search` doing the heavy lifting), while the benchmark harness was a full Codex instance with shell and a custom Python tool designed for the task. @@ -346,7 +312,7 @@ Approval happens over Telegram, and it can be simple accept/reject or involve mu ## Status -Pre-1.0. APIs may shift, but I don't expect dramatic changes. Forking is a real option, not just a theoretical one: the codebase is small enough that pulling upstream changes into your fork by hand stays cheap. Each major feature (MCP, subagents, jsonl_search, parallel_llm, background tools, skills) lives in its own file with no hidden dependencies. If you don't use one, that's even less code to worry about. If you want to delete it entirely, that's a one-shot 10-word prompt to a coding agent. +Pre-1.0. APIs may shift, but I don't expect dramatic changes. Forking is a real option, not just a theoretical one: the codebase is small enough that pulling upstream changes into your fork by hand stays cheap. Each major feature (MCP, subagents, jsonl_search, parallel_llm, skills) lives in its own file with no hidden dependencies. If you don't use one, that's even less code to worry about. If you want to delete it entirely, that's a one-shot 10-word prompt to a coding agent. ThinHarness was built with coding agents, but isn't vibe-coded. I have used it, iterated on it, and reviewed its design + behavior. The [docs site](https://ryanbbrown.com/thinharness/) includes a [codebase explainer](https://ryanbbrown.com/thinharness/explainer.html) that I iterated on to understand the library, and the [web research example](https://ryanbbrown.com/thinharness/examples.html) has the transcript from a non-trivial agent run to show that it works effectively. diff --git a/docs/site/about/index.html b/docs/site/about/index.html index dca0653..369d390 100644 --- a/docs/site/about/index.html +++ b/docs/site/about/index.html @@ -41,7 +41,6 @@ install use features - tracing status license @@ -49,7 +48,7 @@
// why this exists

Why this exists

-

Production agents rarely stop at framework configuration. Things like orchestration, permissions, user/session storage, and deployment become specific to the application and its users.

ThinHarness exists for the gap between building the agent loop yourself and adopting a large agent runtime where the loop comes bundled with assumptions you don’t need and can’t easily change.

It owns a focused set of agent-loop primitives that generalize well and are tedious to rebuild, leaving the rest of the application stack for you to own.

I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple but powerful, yet most frameworks don't include them out of the box, and the ones that do are among the largest and heaviest options (Claude Code -> Claude Agent SDK, LangChain -> deepagents, Agno). I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway.

+

Production agents rarely stop at framework configuration. Things like orchestration, permissions, user/session storage, and deployment become specific to the application and its users.

ThinHarness exists for the gap between building the agent loop yourself and adopting a large agent runtime where the loop comes bundled with assumptions you don’t need and can’t easily change.

It owns a focused set of agent-loop primitives that generalize well and are tedious to rebuild, leaving the rest of the application stack for you to own.

I started building ThinHarness after running into this gap in practice. Filesystem-enabled agents are simple yet powerful, but you mostly get them by adopting a large framework with layers of abstraction. I usually needed only a small slice of the functionality, but that slice came with coupled assumptions that didn't match my application. Making it fit meant writing enough wrappers, adapters, and fixes that I ended up owning framework-shaped code anyway.

@@ -69,7 +68,6 @@

How small, exactly

LOC1 Tool
retries2 Sub-
agents - Structured
output Skills FS
tools OTel
tracing @@ -78,58 +76,58 @@

How small, exactly

ThinHarness
- 8,493 - ●●●●●● + 8,658 + ●●●●● -
Claude Agent SDK3
- 8,263 - ○●○●●◑ +
Claude Agent SDK
+ 8,2633 + ○●●●◑
smolagents
9,840 - ○●●○○● + ○●○○◑ -
deepagents4
- 17,664 - ○●○●●○ +
deepagents
+ 17,6644 + ○●●●○
AWS Strands
32,526 - ◑●●○○● + ◑●●○●
Microsoft Agent Framework
41,331 - ○●●●○● + ○●●●●
Pydantic AI
59,087 - ●○●○○● + ●○○○●
Google ADK
65,799 - ◑●●●○● + ◑●●●●
OpenAI Agents SDK
73,796 - ●●●○○● + ○●●◑◑
Agno
113,477 - ◑●●●●● + ◑●●●◑ -

Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

+

Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, lifecycle hooks, multi-turn conversations, structured output, and human-in-the-loop. It intentionally does not compare framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors.

1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.

2. Tool retries: a documented primitive (e.g. Pydantic AI's ModelRetry) that lets tools signal "model passed bad args — retry with this feedback," distinct from generic exception propagation.

@@ -146,10 +144,8 @@

Opinions

purpose_built

Purpose-built agents, not universal agents

ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants. For business use cases, focused agent loops orchestrated by deterministic code are usually a better fit than sprawling multi-agent systems with broad authority.

no_bash

No bash by default

Purpose-built business agents usually don't need a shell. Bash is a broad security and reliability surface: it gives the model open-ended authority instead of typed, bounded actions. ThinHarness keeps bash out of the default and built-in tool sets, but exposes an opt-in BashTool for exploratory runs before the workflow is hardened with typed tools.

-
skills

Skills are tools, not auto-discovery

Skills live in directories you point at explicitly. The agent calls skill_read and skill_run like any other tool. No interactive scan of the workspace, no global skill marketplace, no magic. SDK use is deliberate; the auto-discovery design is for interactive coding agents and doesn't belong here.

search

Search is a top priority

The search tool exposes ripgrep as compact grouped path/line results, tuned for document and business-workflow agents rather than code navigation. There's also a jsonl_search variant, because JSONL is the right shape when you're replacing RAG with agent-driven search over structured data: ripgrep row prefiltering, jq-style field projection, where filters, range filters, and snippets from large multiline fields.

-
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs reliability beyond a single agent loop — majority vote, ensembled extraction. Set builtin_parallel_llm_model to enable the default parallel_llm tool for plain-text batches; for validated structured output per call, instantiate ParallelLlmTool yourself with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

-
background_tools

Background tools are simple

Some long-running tools can start in the background so the agent can keep working. There is no detached job queue, polling API, or job-control surface; the current run still owns the task, and the completion is sent back to the model when it finishes.

+
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability. Set builtin_parallel_llm_model to enable the default parallel_llm tool for plain-text batches; for validated structured output per call, instantiate ParallelLlmTool yourself with output_type (a Pydantic model). Each call is stateless, and large batches can write JSON to output_file.

no_token_streaming

No token streaming

Streaming is for workflow progress, not live chatbot text. ThinHarness emits run, model-turn, tool, retry, limit, background, and subagent events, but it does not stream provider token deltas. Token streaming would add provider-specific plumbing, event merging, cancellation edge cases, and more surface area to keep stable. For workflow-style agents, step-level updates are usually the useful signal.

providers

Three providers, no matrix

ThinHarness ships small provider classes for OpenAI, Anthropic, and OpenRouter. If your gateway speaks one of those protocols, you swap a base URL and move on. If not, the provider classes are small enough to fork or replace, and ignoring the bundled ones costs you nothing.

no_compaction

No compaction

Compaction is a workaround for context windows filling up across long, accumulating runs — useful for interactive coding sessions that sprawl over hours. For SDK-based business agents, the right answer to "context is getting big" is almost always better task decomposition: shorter runs, separate harness instances, narrower subagents.

@@ -193,7 +189,7 @@

Features

Subagents

Opt-in delegation through a built-in subagent tool and explicit SubAgentConfig.

Parallel LLM

Opt-in parallel_llm fan-out for batches of independent one-shot prompts, plus ParallelLlmTool(...).spec() for renameable tools with explicit model, path, prompt, and retry settings.

Skills

Explicit skill_read and skill_run tools for selected skill directories, with Python, shell, JavaScript, and Go script runners.

-
Resume

Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models.

+
Resume

Clean new-turn continuation through self-contained transcript state that can replay across built-in providers and models, preserving native reasoning on same-provider resume and degrading it to text across providers.

MCP

Optional MCP client support with lazy tool discovery and collision checks.

Parallel tool calls

Same-turn tool batches run concurrently when every called tool is parallel-safe.

Background tools

Opt-in long-running tool calls return a start notice immediately, keep the agent loop moving, and deliver completion back to the model when ready.

@@ -205,13 +201,6 @@

Features

-
-
// tracing
-

Tracing

-

Local tracing is on by default. It writes full plaintext JSONL traces under ~/.thinharness/traces/<encoded-project-root>/, including prompts, model outputs, tool arguments, and tool results, so treat that directory as sensitive local data.

-

Set local_tracing=False or THINHARNESS_DISABLE_LOCAL_TRACING=1 to disable local trace files. External tracing is generic OpenTelemetry: pass any tracer with start_as_current_span(...) or start_span(...) in TracingOptions, and each sink keeps its own capture policy.

-
-
// status

Status

diff --git a/docs/site/assets/site.css b/docs/site/assets/site.css index 7d7f3c8..c62ed3d 100644 --- a/docs/site/assets/site.css +++ b/docs/site/assets/site.css @@ -1812,3 +1812,27 @@ body.page-transcripts { background: #fff; } } + +/* ---- examples page: LongMemEval / transcript toggle ---- */ +.ex-toggle{display:flex;justify-content:center;gap:8px;width:min(1180px,calc(100vw - 48px));margin:18px auto 6px;} +.ex-toggle button{font-family:var(--mono);font-size:13px;padding:8px 16px;border:1px solid var(--line);border-radius:8px;background:var(--panel);color:var(--ink-soft);cursor:pointer;transition:color .15s,background .15s,border-color .15s;} +.ex-toggle button:hover{color:var(--green-deep);border-color:var(--green-line);} +.ex-toggle button.is-active{color:var(--green-deep);background:var(--green-wash);border-color:var(--green-line);font-weight:600;} +.ex-panel[hidden]{display:none !important;} +.md-render{width:min(860px,calc(100vw - 48px));margin:0 auto;padding:14px 0 64px;color:var(--ink);font-family:var(--sans);line-height:1.62;} +.md-render .md-eyebrow{font-family:var(--mono);font-size:13px;color:var(--green);letter-spacing:.04em;font-weight:500;margin:6px 0 6px;} +.md-render h1{font-size:30px;margin:0 0 8px;letter-spacing:-.02em;font-weight:700;} +.md-render h2{font-size:20px;margin:30px 0 10px;color:var(--green-deep);border-bottom:1px solid var(--line);padding-bottom:6px;} +.md-render h3{font-size:16px;margin:20px 0 8px;} +.md-render p{margin:10px 0;color:var(--ink-soft);} +.md-render ul{margin:10px 0;padding-left:22px;color:var(--ink-soft);} +.md-render li{margin:7px 0;} +.md-render a{color:var(--green);text-decoration:underline;text-underline-offset:2px;} +.md-render code{font-family:var(--mono);font-size:.86em;background:var(--green-wash);padding:1px 5px;border-radius:4px;color:var(--green-deep);} +.md-render strong{color:var(--ink);} +.md-table-wrap{overflow-x:auto;margin:14px 0;border:1px solid var(--line);border-radius:8px;max-width:620px;} +.md-render table{width:100%;border-collapse:collapse;font-size:13px;} +.md-render th,.md-render td{padding:8px 12px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top;} +.md-render td:first-child,.md-render th:first-child{font-family:var(--mono);font-size:12px;color:var(--ink);white-space:nowrap;} +.md-render thead th{background:var(--green-wash);color:var(--green-deep);font-weight:600;} +.md-render tbody tr:last-child td{border-bottom:none;} diff --git a/docs/site/examples/index.html b/docs/site/examples/index.html index 4f9ca92..3a207a4 100644 --- a/docs/site/examples/index.html +++ b/docs/site/examples/index.html @@ -3,7 +3,7 @@ - Web Research Example — ThinHarness + Examples — ThinHarness @@ -22,11 +22,30 @@
-
// example run
-

Web Research Example

-
A complete DeepSeek + Exa run: plan, search, source notes, citation critique, final report.
+
// examples
+

Examples

+
Real ThinHarness runs. Toggle between the LongMemEval-V2 benchmark write-up and the Web Research transcript.
-
+
+ + +
+
+
// benchmark reproduction

LongMemEval-V2 - ThinHarness Benchmark

+

Context

+

I used a fork of the LongMemEval-V2 benchmark to test whether ThinHarness is a strong general-purpose agent harness on a nontrivial non-coding task: information retrieval over long trajectory haystacks.

+

I scoped the comparison to the 127 dynamic questions in the small tier: dynamic-environment and dynamic-environment-abs across the web and enterprise domains. After reviewing the benchmark categories, this subset looked like the best test for my purposes: given a long history of interaction traces, can the memory layer efficiently and accurately find the state change, UI behavior, or environment fact needed to answer the question?

+

Run Details

+

To get a local baseline for more detailed metrics than the leaderboard provides, I ran AgentRunbook-C with the resumable wrapper in evaluation/scripts/run_agentrunbook_c_dynamic.sh. That script plans the 127-question dynamic set, runs one question per output directory, and can be re-run to fill only missing questions. Running one question at a time is slower but was helpful when encountering intermittent issues running on my local machine. The matching ThinHarness wrapper is evaluation/scripts/run_thinharness_dynamic.sh.

+

This is a best-faith reproduction rather than an exact reproduction of the paper. The paper setup expects a local Qwen/Qwen3.5-9B reader deployment; I used qwen/qwen3.5-9b through OpenRouter. The paper uses Codex v0.117.0 for Codex and AgentRunbook-C; my rerun used the local Codex CLI available, codex-cli 0.141.0. Both the AgentRunbook-C rerun and ThinHarness run used gpt-5.4-mini with xhigh reasoning for the query-time memory agent and gpt-5.2 for the evaluator.

+

For the ThinHarness run, I used only its generic built-in filesystem tools (read, search, jsonl_search, list, and glob). I did restructure the memory files into a JSONL-friendly corpus so its generic jsonl_search tool could work well; that seems acceptable here because AgentRunbook-C also creates a custom trajectory structure rather than using the vanilla Codex raw layout. I tried to keep the query-time system prompt as close as practical to AgentRunbook-C, changing the tool instructions only where ThinHarness needed to know how to use its built-in tools.

+

Results

+

Across all 127 dynamic questions in the small tier:

+
MetricAgentRunbook-C rerunThinHarness
Dynamic score72.4% (92/127)74.0% (94/127)
Non-abstention86.0% (74/86)84.9% (73/86)
Abstention43.9% (18/41)51.2% (21/41)
Memory query time151.9s avg, 129.1s median99.7s avg, 87.9s median
Memory-agent tokens / usage114.77M input, 1.32M output, 116.09M total60.14M input, 2.10M output, 62.24M total
Dynamic-subset LAFS3.769.73 (+5.96)
+

The 72.4% accuracy for AgentRunbook-C matches the paper, but I would not treat this single consolidated run as a statistically signficant claim that ThinHarness has higher accuracy than AgentRunbook-C--I saw meaningful variance on a portion of the questions when doing targeted reruns. The result does make me reasonably confident that ThinHarness at least matches AgentRunbook-C's performance on this slice, and the published leaderboard reference for vanilla Codex is materially lower than both.

+

The memory query time is the harness-measured time around memory.query(...): it includes the query-time memory retrieval agent, but not the downstream reader, scorer, or prior runtime input generation. The timing comparison isn't perfect (local Codex CLI vs. OpenAI API), but the 46.4% lower token usage indicates that ~34% time savings is probably in the right ballpark. Note that the paper only provides a single aggregate query time figure across all questions, 108.3s, which is far lower than the 151.9s above but includes all questions in the small tier (some of which may have been faster).

+
+
- 8,493 + 8,658 README-stated framework LOC, intentionally small enough to inspect, adapt, and fork.
@@ -899,7 +899,7 @@

Implementation Deep Dive

Resume state Provider-agnostic transcript state copied into HarnessResult.resume_state while building the final result. - Built-in providers emit kind="transcript", version=2, origin diagnostics, and neutral user/assistant/tool entries. Callers can store and pass it back, but should not edit or construct it. + Built-in providers emit kind="transcript", version=3, origin diagnostics, and neutral user/assistant/tool entries. Callers can store and pass it back, but should not edit or construct it. Approval pause state @@ -914,6 +914,38 @@

Implementation Deep Dive

exhaustion, output validation failure, unexpected model behavior, and tool-mode final_result exits intentionally produce no checkpoint.

+

+ Resume also carries model reasoning. Each built-in session keeps the provider's native reasoning parts in the + neutral transcript, so resuming on the same provider replays them verbatim — Anthropic signed thinking + blocks, OpenAI encrypted_content, OpenRouter reasoning_details. An opaque blob cannot + be replayed to a different provider, so cross-provider resume degrades every reasoning part to a leading + <thinking>-tagged text block and drops the blob. Native re-emit also requires the resuming + request to be able to accept the block: +

+ + + + + + + + + + + + + + + + + + + +
ProviderNative reasoning in resume stateRe-emits natively only when
OpenAI Responsesencrypted_content, captured via include=["reasoning.encrypted_content"] on reasoning-capable modelsthe resuming model is reasoning-capable; otherwise the text fallback is used
Anthropic Messagessigned thinking / redacted_thinking blocksextended thinking is enabled on the resuming run; otherwise the text fallback is used
OpenRouter chat completionsreasoning_detailsresuming on OpenRouter — no additional capability gate
+

+ Because resume_state can therefore hold encrypted reasoning blobs and signed thinking, treat it as + sensitive, like the local traces it mirrors. +

diff --git a/docs/site/index.html b/docs/site/index.html index a386f36..42af279 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -36,7 +36,7 @@

A minimal, opinionated agent harness.
install.sh
-
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 23 files · 8,493 LOC
+
copy$ uv add thinharness
# or: pip install thinharness
# requires python 3.11+

✓ resolved · 23 files · 8,658 LOC

@@ -46,10 +46,10 @@

A minimal, opinionated agent harness.
purpose_built

Purpose-built agents

ThinHarness is for bounded agent loops inside software you control, not open-ended interactive assistants.

no_bash

No bash by default

Bash stays out of the default tools, with an opt-in BashTool only for prototyping before typed tools.

-
skills

Skills are tools, not auto-discovery

Skills live in directories you point at explicitly. The agent calls them like any other tool. No magic scan, no marketplace.

-
search

Search is a top priority

Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus JSONL search with field projection, range filters, and multiline snippets.

-
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs reliability beyond a single loop — majority vote, ensembled extraction.

-
background_tools

Background tools are simple

Long-running tools can start in the background, but the current run still owns the task and receives completion.

+
search

Search is a top priority

Ripgrep exposed as compact grouped results, tuned for documents and business workflows — plus a custom JSONL search tool for structured corpuses.

+
parallel_llm

Parallel LLM calls, built in

Fan out from inside the harness when a workflow needs efficient parallel processing or majority vote for reliability.

+
no_compaction

No compaction

Compaction makes sense for sprawling coding sessions. For business agents the fix is smarter task decomposition and context management

+
no_deployment

No deployment layer

Serving, auth, durable jobs, and session storage stay yours. ThinHarness owns the agent loop, not the production stack around it.

diff --git a/examples/longmemeval.md b/examples/longmemeval.md new file mode 100644 index 0000000..15f607c --- /dev/null +++ b/examples/longmemeval.md @@ -0,0 +1,33 @@ +# LongMemEval-V2 - ThinHarness Benchmark + +## Context + +I used a [fork](https://github.com/ryanbbrown/LongMemEval-V2) of the LongMemEval-V2 benchmark to test whether ThinHarness is a strong general-purpose agent harness on a nontrivial non-coding task: information retrieval over long trajectory haystacks. + +I scoped the comparison to the 127 dynamic questions in the small tier: `dynamic-environment` and `dynamic-environment-abs` across the web and enterprise domains. After reviewing the benchmark categories, this subset looked like the best test for my purposes: given a long history of interaction traces, can the memory layer efficiently and accurately find the state change, UI behavior, or environment fact needed to answer the question? + +## Run Details + +To get a local baseline for more detailed metrics than the leaderboard provides, I ran AgentRunbook-C with the resumable wrapper in `evaluation/scripts/run_agentrunbook_c_dynamic.sh`. That script plans the 127-question dynamic set, runs one question per output directory, and can be re-run to fill only missing questions. Running one question at a time is slower but was helpful when encountering intermittent issues running on my local machine. The matching ThinHarness wrapper is `evaluation/scripts/run_thinharness_dynamic.sh`. + +This is a best-faith reproduction rather than an exact reproduction of the paper. The paper setup expects a local `Qwen/Qwen3.5-9B` reader deployment; I used `qwen/qwen3.5-9b` through OpenRouter. The paper uses Codex v0.117.0 for Codex and AgentRunbook-C; my rerun used the local Codex CLI available, `codex-cli 0.141.0`. Both the AgentRunbook-C rerun and ThinHarness run used `gpt-5.4-mini` with `xhigh` reasoning for the query-time memory agent and `gpt-5.2` for the evaluator. + +For the ThinHarness run, I used only its generic built-in filesystem tools (`read`, `search`, `jsonl_search`, `list`, and `glob`). I did restructure the memory files into a JSONL-friendly corpus so its generic `jsonl_search` tool could work well; that seems acceptable here because AgentRunbook-C also creates a custom trajectory structure rather than using the vanilla Codex raw layout. I tried to keep the query-time system prompt as close as practical to AgentRunbook-C, changing the tool instructions only where ThinHarness needed to know how to use its built-in tools. + +## Results + +Across all 127 dynamic questions in the small tier: + +| Metric | AgentRunbook-C rerun | ThinHarness | +| --- | --- | --- | +| Dynamic score | 72.4% (92/127) | 74.0% (94/127) | +| Non-abstention | 86.0% (74/86) | 84.9% (73/86) | +| Abstention | 43.9% (18/41) | 51.2% (21/41) | +| Memory query time | 151.9s avg, 129.1s median | 99.7s avg, 87.9s median | +| Memory-agent tokens / usage | 114.77M input, 1.32M output, 116.09M total | 60.14M input, 2.10M output, 62.24M total | +| Dynamic-subset LAFS | 3.76 | 9.73 (+5.96) | + +The 72.4% accuracy for AgentRunbook-C matches the paper, but I would not treat this single consolidated run as a statistically signficant claim that ThinHarness has higher accuracy than AgentRunbook-C--I saw meaningful variance on a portion of the questions when doing targeted reruns. The result does make me reasonably confident that ThinHarness at least matches AgentRunbook-C's performance on this slice, and the published leaderboard reference for vanilla Codex is materially lower than both. + +The memory query time is the harness-measured time around `memory.query(...)`: it includes the query-time memory retrieval agent, but not the downstream reader, scorer, or prior runtime input generation. The timing comparison isn't perfect (local Codex CLI vs. OpenAI API), but the 46.4% lower token usage indicates that ~34% time savings is probably in the right ballpark. Note that the paper only provides a single aggregate query time figure across all questions, 108.3s, which is far lower than the 151.9s above but includes all questions in the small tier (some of which may have been faster). + diff --git a/scripts/build_site.py b/scripts/build_site.py index 21e70c2..bbf6f4c 100644 --- a/scripts/build_site.py +++ b/scripts/build_site.py @@ -46,10 +46,8 @@ def slug_for_opinion(title: str) -> str: known_tags = { "Purpose-built agents, not universal agents": "purpose_built", "No bash by default": "no_bash", - "Skills are tools, not auto-discovery": "skills", "Search is a top priority": "search", "Parallel LLM calls, built in": "parallel_llm", - "Background tools are simple": "background_tools", "Three providers, no matrix": "providers", "No compaction": "no_compaction", "No deployment layer": "no_deployment", @@ -98,22 +96,31 @@ def __init__(self) -> None: self._row: list[dict[str, str]] | None = None self._cell: dict[str, str] | None = None self._text: list[str] = [] + self._sup: list[str] = [] + self._in_sup = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag == "tr": self._row = [] elif tag in {"td", "th"} and self._row is not None: - self._cell = {"text": "", "img": ""} + self._cell = {"text": "", "img": "", "sup": ""} self._text = [] + self._sup = [] + self._in_sup = False elif tag == "img" and self._cell is not None: attrs_dict = dict(attrs) self._cell["img"] = attrs_dict.get("src") or "" + elif tag == "sup" and self._cell is not None: + self._in_sup = True elif tag == "br" and self._cell is not None: self._text.append(" ") def handle_endtag(self, tag: str) -> None: - if tag in {"td", "th"} and self._cell is not None and self._row is not None: + if tag == "sup" and self._cell is not None: + self._in_sup = False + elif tag in {"td", "th"} and self._cell is not None and self._row is not None: self._cell["text"] = normalize_table_text("".join(self._text)) + self._cell["sup"] = normalize_table_text("".join(self._sup)) self._row.append(self._cell) self._cell = None elif tag == "tr" and self._row is not None: @@ -122,7 +129,11 @@ def handle_endtag(self, tag: str) -> None: self._row = None def handle_data(self, data: str) -> None: - if self._cell is not None: + if self._cell is None: + return + if self._in_sup: + self._sup.append(data) + else: self._text.append(data) @@ -149,12 +160,8 @@ def mark(value: str) -> str: def library_cell(cell: dict[str, str], asset_prefix: str = "assets/") -> str: name = cell["text"] - superscript = "" - match = re.search(r"(\d+)$", name) - if match and name not in {"3"}: - name = name[: -len(match.group(1))].strip() - superscript = f"{match.group(1)}" - display_name = html.escape(name).replace("Claude Agent SDK", "Claude Agent SDK").replace("OpenAI Agents SDK", "OpenAI Agents SDK") + superscript = f"{html.escape(cell['sup'])}" if cell.get("sup") else "" + display_name = html.escape(name) if name == "ThinHarness": return f'
ThinHarness
' if name == "Agno": @@ -169,12 +176,14 @@ def comparison_table(markdown: str, asset_prefix: str = "assets/") -> str: body_rows = [] for row in rows[1:]: library = row[0] + loc = row[1] + loc_sup = f'{html.escape(loc["sup"])}' if loc.get("sup") else "" css = ' class="me"' if library["text"] == "ThinHarness" else "" marks = "".join(f"{mark(cell['text'])}" for cell in row[2:]) body_rows.append( f""" {library_cell(library, asset_prefix)} - {html.escape(row[1]["text"])} + {html.escape(loc["text"])}{loc_sup} {marks} """ ) @@ -204,7 +213,6 @@ def render_about(markdown: str) -> str: use = section(markdown, "Use") use_code = code_highlight(fenced_code(use, "python")) use_paragraph = paragraphs(use.split("```", 2)[2].strip())[0] - tracing = paragraphs(section(markdown, "Tracing")) opinion_items = "\n".join( f'
{tag}

{html.escape(title)}

{body}

' @@ -219,7 +227,7 @@ def render_about(markdown: str) -> str: ) table_summary = ( "Table focuses on harness-level features that differentiate the libraries. All listed also support MCP, " - "lifecycle hooks, multi-turn conversations, and human-in-the-loop. It intentionally does not compare " + "lifecycle hooks, multi-turn conversations, structured output, and human-in-the-loop. It intentionally does not compare " "framework/platform features like vector DB integrations, hosted deployment, memory/session stores, or broad SaaS connectors." ) retry_footnote = ( @@ -274,7 +282,6 @@ def render_about(markdown: str) -> str: install use features - tracing status license @@ -302,7 +309,6 @@ def render_about(markdown: str) -> str: LOC1 Tool
retries2 Sub-
agents - Structured
output Skills FS
tools OTel
tracing @@ -318,7 +324,7 @@ def render_about(markdown: str) -> str:

1. LOC excludes anything that is not the core agent harness framework. See raw README source comments for exact commands.

2. {retry_footnote}

3. Claude Agent SDK shells out to the Claude Code CLI binary, which is 200k+ LOC.

-

4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈111k LOC.

+

4. deepagents is a thin wrapper over LangChain/LangGraph; effective import surface is ≈112k LOC.

See docs/table.md for per-cell rationale and how the LOC numbers are measured.

@@ -354,13 +360,6 @@ def render_about(markdown: str) -> str: -
-
// tracing
-

Tracing

-

{inline_markdown(tracing[0])}

-

{inline_markdown(tracing[1])}

-
-
// status

Status

diff --git a/scripts/build_transcripts.py b/scripts/build_transcripts.py index 3fea81c..547dbdf 100644 --- a/scripts/build_transcripts.py +++ b/scripts/build_transcripts.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import html import json import re from pathlib import Path @@ -8,6 +9,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] EXAMPLES_ROOT = REPO_ROOT / "examples" +LONGMEMEVAL_MD = EXAMPLES_ROOT / "longmemeval.md" DEFAULT_OUTPUT = REPO_ROOT / "docs" / "site" / "examples.html" LONG_PREVIEW_CHARS = 1200 WEB_RESEARCH_REPORT_META = { @@ -33,6 +35,72 @@ AGENT_META = {"web_research_report": WEB_RESEARCH_REPORT_META} +def md_inline(text: str) -> str: + """Render inline markdown (code, bold, links) to HTML, preserving links.""" + placeholders: list[str] = [] + + def hold(value: str) -> str: + placeholders.append(value) + return f"\0{len(placeholders) - 1}\0" + + text = re.sub(r"`([^`]+)`", lambda m: hold(f"{html.escape(m.group(1))}"), text) + text = re.sub( + r"\[([^\]]+)\]\(([^)]+)\)", + lambda m: hold(f'{html.escape(m.group(1))}'), + text, + ) + escaped = html.escape(text, quote=False).replace("\n", " ") + escaped = re.sub(r"\*\*([^*]+)\*\*", r"\1", escaped) + for index, value in enumerate(placeholders): + escaped = escaped.replace(f"\0{index}\0", value) + return escaped + + +def md_table(block: str) -> str: + """Render a GFM table block to an HTML table wrapped for horizontal scroll.""" + def cells(row: str) -> list[str]: + return [cell.strip() for cell in row.strip().strip("|").split("|")] + + def is_separator(row: str) -> bool: + return set(row.replace("|", "").replace("-", "").replace(":", "").strip()) <= {" ", ""} + + rows = [cells(line) for line in block.splitlines() if line.strip().startswith("|") and not is_separator(line)] + head, *body = rows + thead = "".join(f"{md_inline(cell)}" for cell in head) + tbody = "".join("" + "".join(f"{md_inline(cell)}" for cell in row) + "" for row in body) + return f'
{thead}{tbody}
' + + +def render_markdown(md: str) -> str: + """Render the LongMemEval excerpt (headings, paragraphs, one table) to HTML. + + Block-level only, matching the regex-based markdown approach used in build_site.py; the + leading h1 gets a site-style eyebrow so it reads like the rest of the docs pages. + """ + out: list[str] = [] + eyebrow_done = False + for block in re.split(r"\n[ \t]*\n", md.strip()): + block = block.strip() + if not block: + continue + if block.lstrip().startswith("|"): + out.append(md_table(block)) + elif block.startswith("### "): + out.append(f"

{md_inline(block[4:].strip())}

") + elif block.startswith("## "): + out.append(f"

{md_inline(block[3:].strip())}

") + elif block.startswith("# "): + title = md_inline(block[2:].strip()) + if eyebrow_done: + out.append(f"

{title}

") + else: + out.append(f'
// benchmark reproduction

{title}

') + eyebrow_done = True + else: + out.append(f"

{md_inline(block)}

") + return "\n".join(out) + + def parse_jsonish(value: Any) -> Any: if not isinstance(value, str) or not value.strip(): return value @@ -445,12 +513,18 @@ def load_agents() -> list[dict[str, Any]]: def render_html(agents: list[dict[str, Any]], *, template_path: Path | None = None) -> str: data = json.dumps({"agents": agents}, ensure_ascii=False) script_data = data.replace(")(.*?)()', re.S) - if pattern.search(template): - return pattern.sub(lambda match: f"{match.group(1)}{script_data}{match.group(3)}", template, count=1) - raise ValueError("examples template must contain )', re.S) + if not trace_pattern.search(template): + raise ValueError("examples template must contain