From 7775419e0cf978d207c75f267d333c32e8505e25 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 13:49:25 +0000 Subject: [PATCH 1/8] fix(intercept/responses): record all tool call types in responses interceptor Previously, the responses interceptor only recorded function_call and custom_tool_call types. This left tool calls like web_search_call, computer_call, shell_call, etc. unrecorded, meaning interceptions that did significant work via built-in tools showed no tool usage. Now all tool call types are recorded: - web_search_call - computer_call - local_shell_call - shell_call - apply_patch_call - code_interpreter_call - mcp_call - file_search_call - image_generation_call For these types, the tool name falls back to the type when no explicit name is set, and the call ID falls back to the item ID when call_id is empty (web_search_call, file_search_call, etc. use id instead of call_id). Adds a web_search blocking fixture and comprehensive tests covering all new tool types. Closes: https://github.com/coder/aibridge/issues/121 --- fixtures/fixtures.go | 3 + .../responses/blocking/web_search.txtar | 102 +++++++++++++ intercept/responses/base.go | 41 +++++- intercept/responses/base_test.go | 139 ++++++++++++++++++ 4 files changed, 280 insertions(+), 5 deletions(-) create mode 100644 fixtures/openai/responses/blocking/web_search.txtar diff --git a/fixtures/fixtures.go b/fixtures/fixtures.go index c731e0fb..c6b3a78c 100644 --- a/fixtures/fixtures.go +++ b/fixtures/fixtures.go @@ -108,6 +108,9 @@ var ( //go:embed openai/responses/blocking/wrong_response_format.txtar OaiResponsesBlockingWrongResponseFormat []byte + + //go:embed openai/responses/blocking/web_search.txtar + OaiResponsesBlockingWebSearch []byte ) var ( diff --git a/fixtures/openai/responses/blocking/web_search.txtar b/fixtures/openai/responses/blocking/web_search.txtar new file mode 100644 index 00000000..c4d42f26 --- /dev/null +++ b/fixtures/openai/responses/blocking/web_search.txtar @@ -0,0 +1,102 @@ +{ + "input": [ + { + "role": "user", + "content": "What is the current weather in San Francisco?" + } + ], + "model": "gpt-4.1", + "stream": false, + "tools": [ + { + "type": "web_search_preview" + } + ] +} + +-- non-streaming -- +{ + "id": "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + "object": "response", + "created_at": 1767875200, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1767875205, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + "type": "web_search_call", + "status": "completed" + }, + { + "id": "msg_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Based on my search, the current weather in San Francisco is partly cloudy with a temperature of around 62°F (17°C).", + "annotations": [ + { + "type": "url_citation", + "start_index": 28, + "end_index": 47, + "url": "https://weather.example.com/san-francisco", + "title": "San Francisco Weather" + } + ] + } + ] + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "web_search_preview" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 42, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 150, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 192 + }, + "user": null, + "metadata": {} +} diff --git a/intercept/responses/base.go b/intercept/responses/base.go index daccc300..667bc329 100644 --- a/intercept/responses/base.go +++ b/intercept/responses/base.go @@ -204,14 +204,45 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte } for _, item := range response.Output { - var args recorder.ToolArgs + var ( + args recorder.ToolArgs + toolName string + callID string + ) - // recording other function types to be considered: https://github.com/coder/aibridge/issues/121 + //nolint:goconst // tool type string values are clearer inline. switch item.Type { case string(constant.ValueOf[constant.FunctionCall]()): args = i.parseFunctionCallJSONArgs(ctx, item.Arguments) + toolName = item.Name + callID = item.CallID + case string(constant.ValueOf[constant.CustomToolCall]()): args = item.Input + toolName = item.Name + callID = item.CallID + + case "web_search_call", + "computer_call", + "file_search_call", + "code_interpreter_call", + "image_generation_call", + "mcp_call", + "local_shell_call", + "shell_call", + "apply_patch_call": + // For these tool types, use the type as the tool name + // when there is no explicit name, and prefer call_id + // but fall back to the item id. + toolName = item.Name + if toolName == "" { + toolName = item.Type + } + callID = item.CallID + if callID == "" { + callID = item.ID + } + default: continue } @@ -219,12 +250,12 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte if err := i.recorder.RecordToolUsage(ctx, &recorder.ToolUsageRecord{ InterceptionID: i.ID().String(), MsgID: response.ID, - ToolCallID: item.CallID, - Tool: item.Name, + ToolCallID: callID, + Tool: toolName, Args: args, Injected: false, }); err != nil { - i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", item.Name)) + i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", toolName)) } } } diff --git a/intercept/responses/base_test.go b/intercept/responses/base_test.go index ea5c87b5..7e1cc35e 100644 --- a/intercept/responses/base_test.go +++ b/intercept/responses/base_test.go @@ -191,6 +191,145 @@ func TestRecordToolUsage(t *testing.T) { }, }, }, + { + name: "web_search_call_with_no_name", + response: &oairesponses.Response{ + ID: "resp_ws", + Output: []oairesponses.ResponseOutputItemUnion{ + { + Type: "web_search_call", + ID: "ws_abc", + }, + }, + }, + expected: []*recorder.ToolUsageRecord{ + { + InterceptionID: id.String(), + MsgID: "resp_ws", + ToolCallID: "ws_abc", + Tool: "web_search_call", + Injected: false, + }, + }, + }, + { + name: "all_additional_tool_types", + response: &oairesponses.Response{ + ID: "resp_all", + Output: []oairesponses.ResponseOutputItemUnion{ + { + Type: "web_search_call", + ID: "ws_1", + }, + { + Type: "computer_call", + CallID: "call_comp", + }, + { + Type: "local_shell_call", + CallID: "call_lsh", + }, + { + Type: "shell_call", + CallID: "call_sh", + }, + { + Type: "apply_patch_call", + CallID: "call_ap", + }, + { + Type: "code_interpreter_call", + ID: "ci_1", + }, + { + Type: "mcp_call", + ID: "mcp_1", + Name: "my_mcp_tool", + }, + { + Type: "file_search_call", + ID: "fs_1", + }, + { + Type: "image_generation_call", + ID: "ig_1", + }, + { + Type: "message", + ID: "msg_skip", + }, + { + Type: "reasoning", + ID: "rs_skip", + }, + }, + }, + expected: []*recorder.ToolUsageRecord{ + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "ws_1", + Tool: "web_search_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "call_comp", + Tool: "computer_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "call_lsh", + Tool: "local_shell_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "call_sh", + Tool: "shell_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "call_ap", + Tool: "apply_patch_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "ci_1", + Tool: "code_interpreter_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "mcp_1", + Tool: "my_mcp_tool", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "fs_1", + Tool: "file_search_call", + Injected: false, + }, + { + InterceptionID: id.String(), + MsgID: "resp_all", + ToolCallID: "ig_1", + Tool: "image_generation_call", + Injected: false, + }, + }, + }, } for _, tc := range tests { From 63ea2fa0dfbab2253eac6af415ce5f1ce31bf815 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:15:02 +0000 Subject: [PATCH 2/8] refactor: split tool types into agentic/hosted groups, use SDK constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Agentic tools (computer_call, local_shell_call, shell_call, apply_patch_call) use call_id for correlation. - Hosted tools (web_search_call, file_search_call, code_interpreter_call, image_generation_call, mcp_call) use id only — referenced in OpenAI API docs. - Use constant.ValueOf[...] for all types with SDK constants; define local computerCall const for the missing one. - Remove the nolint:goconst directive. --- intercept/responses/base.go | 39 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/intercept/responses/base.go b/intercept/responses/base.go index 667bc329..6d65a91e 100644 --- a/intercept/responses/base.go +++ b/intercept/responses/base.go @@ -34,6 +34,10 @@ import ( const ( requestTimeout = time.Second * 600 + + // computerCall is defined locally because the OpenAI Go SDK does + // not export a typed constant for the "computer_call" output item. + computerCall = "computer_call" ) type responsesInterceptionBase struct { @@ -210,7 +214,6 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte callID string ) - //nolint:goconst // tool type string values are clearer inline. switch item.Type { case string(constant.ValueOf[constant.FunctionCall]()): args = i.parseFunctionCallJSONArgs(ctx, item.Arguments) @@ -222,26 +225,32 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte toolName = item.Name callID = item.CallID - case "web_search_call", - "computer_call", - "file_search_call", - "code_interpreter_call", - "image_generation_call", - "mcp_call", - "local_shell_call", - "shell_call", - "apply_patch_call": - // For these tool types, use the type as the tool name - // when there is no explicit name, and prefer call_id - // but fall back to the item id. + // Agentic tools: the client sends a corresponding *_output + // item correlated by call_id. + case computerCall, + string(constant.ValueOf[constant.LocalShellCall]()), + string(constant.ValueOf[constant.ShellCall]()), + string(constant.ValueOf[constant.ApplyPatchCall]()): toolName = item.Name if toolName == "" { toolName = item.Type } callID = item.CallID - if callID == "" { - callID = item.ID + + // Hosted tools: executed server-side, these output items + // carry only an id field — not call_id. The client never + // submits output for them. + // https://platform.openai.com/docs/api-reference/responses/create + case string(constant.ValueOf[constant.WebSearchCall]()), + string(constant.ValueOf[constant.FileSearchCall]()), + string(constant.ValueOf[constant.CodeInterpreterCall]()), + string(constant.ValueOf[constant.ImageGenerationCall]()), + string(constant.ValueOf[constant.McpCall]()): + toolName = item.Name + if toolName == "" { + toolName = item.Type } + callID = item.ID default: continue From 7cf769ffc813765366f4cf39c61194d105349e75 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:21:36 +0000 Subject: [PATCH 3/8] fix(fixtures): use gpt-5 and web_search tool type in fixture The web_search_preview tool type is deprecated in favor of web_search. --- fixtures/openai/responses/blocking/web_search.txtar | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fixtures/openai/responses/blocking/web_search.txtar b/fixtures/openai/responses/blocking/web_search.txtar index c4d42f26..25a762e7 100644 --- a/fixtures/openai/responses/blocking/web_search.txtar +++ b/fixtures/openai/responses/blocking/web_search.txtar @@ -5,11 +5,11 @@ "content": "What is the current weather in San Francisco?" } ], - "model": "gpt-4.1", + "model": "gpt-5", "stream": false, "tools": [ { - "type": "web_search_preview" + "type": "web_search" } ] } @@ -30,7 +30,7 @@ "instructions": null, "max_output_tokens": null, "max_tool_calls": null, - "model": "gpt-4.1-2025-04-14", + "model": "gpt-5-0806", "output": [ { "id": "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", @@ -80,7 +80,7 @@ "tool_choice": "auto", "tools": [ { - "type": "web_search_preview" + "type": "web_search" } ], "top_logprobs": 0, From 4514694a0f4e61219c34c9c8f613c25b44a9d935 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:22:31 +0000 Subject: [PATCH 4/8] fix(fixtures): use Cape Town in web search fixture --- fixtures/openai/responses/blocking/web_search.txtar | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fixtures/openai/responses/blocking/web_search.txtar b/fixtures/openai/responses/blocking/web_search.txtar index 25a762e7..bd974d1d 100644 --- a/fixtures/openai/responses/blocking/web_search.txtar +++ b/fixtures/openai/responses/blocking/web_search.txtar @@ -2,7 +2,7 @@ "input": [ { "role": "user", - "content": "What is the current weather in San Francisco?" + "content": "What is the current weather in Cape Town?" } ], "model": "gpt-5", @@ -45,14 +45,14 @@ "content": [ { "type": "output_text", - "text": "Based on my search, the current weather in San Francisco is partly cloudy with a temperature of around 62°F (17°C).", + "text": "Based on my search, the current weather in Cape Town is partly cloudy with a temperature of around 22°C (72°F).", "annotations": [ { "type": "url_citation", "start_index": 28, "end_index": 47, - "url": "https://weather.example.com/san-francisco", - "title": "San Francisco Weather" + "url": "https://weather.example.com/cape-town", + "title": "Cape Town Weather" } ] } From 4d87bcab8f7f874ee497520ed1d27c42b7597aaf Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:22:52 +0000 Subject: [PATCH 5/8] refactor: drop computerCall const, use string literal --- intercept/responses/base.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/intercept/responses/base.go b/intercept/responses/base.go index 6d65a91e..b81bf23a 100644 --- a/intercept/responses/base.go +++ b/intercept/responses/base.go @@ -34,10 +34,6 @@ import ( const ( requestTimeout = time.Second * 600 - - // computerCall is defined locally because the OpenAI Go SDK does - // not export a typed constant for the "computer_call" output item. - computerCall = "computer_call" ) type responsesInterceptionBase struct { @@ -227,7 +223,7 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte // Agentic tools: the client sends a corresponding *_output // item correlated by call_id. - case computerCall, + case "computer_call", string(constant.ValueOf[constant.LocalShellCall]()), string(constant.ValueOf[constant.ShellCall]()), string(constant.ValueOf[constant.ApplyPatchCall]()): From 7b2a0dfd3a5bf7156fbe9837c8656989e90a2b82 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:25:00 +0000 Subject: [PATCH 6/8] test: add integration test for web_search fixture Add blocking_web_search case to TestResponsesOutputMatchesUpstream that exercises the hosted-tool recording path (id, not call_id). Also fix the fixture's missing -- request -- txtar header. --- .../responses/blocking/web_search.txtar | 1 + internal/integrationtest/responses_test.go | 32 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/fixtures/openai/responses/blocking/web_search.txtar b/fixtures/openai/responses/blocking/web_search.txtar index bd974d1d..ff139219 100644 --- a/fixtures/openai/responses/blocking/web_search.txtar +++ b/fixtures/openai/responses/blocking/web_search.txtar @@ -1,3 +1,4 @@ +-- request -- { "input": [ { diff --git a/internal/integrationtest/responses_test.go b/internal/integrationtest/responses_test.go index 1a35f707..6e0c3f55 100644 --- a/internal/integrationtest/responses_test.go +++ b/internal/integrationtest/responses_test.go @@ -163,11 +163,33 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { "total_tokens": 172, }, }, - expectedClient: aibridge.ClientUnknown, - }, - { - name: "streaming_simple", - fixture: fixtures.OaiResponsesStreamingSimple, + expectedClient: aibridge.ClientUnknown, + }, + { + name: "blocking_web_search", + fixture: fixtures.OaiResponsesBlockingWebSearch, + expectModel: "gpt-5", + expectPromptRecorded: "What is the current weather in Cape Town?", + expectToolRecorded: &recorder.ToolUsageRecord{ + MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Tool: "web_search_call", + ToolCallID: "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Injected: false, + }, + expectTokenUsage: &recorder.TokenUsageRecord{ + MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Input: 42, + Output: 150, + ExtraTokenTypes: map[string]int64{ + "input_cached": 0, + "output_reasoning": 0, + "total_tokens": 192, + }, + }, + expectedClient: aibridge.ClientUnknown, + }, + { + name: "streaming_simple", fixture: fixtures.OaiResponsesStreamingSimple, streaming: true, expectModel: "gpt-4o-mini", expectPromptRecorded: "tell me a joke", From 240b0e6d1f37cac32beabe86b081bcf55f150cb8 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Fri, 17 Apr 2026 14:28:03 +0000 Subject: [PATCH 7/8] style: fix formatting in responses_test.go --- internal/integrationtest/responses_test.go | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/internal/integrationtest/responses_test.go b/internal/integrationtest/responses_test.go index 6e0c3f55..b432fbbd 100644 --- a/internal/integrationtest/responses_test.go +++ b/internal/integrationtest/responses_test.go @@ -163,33 +163,33 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { "total_tokens": 172, }, }, - expectedClient: aibridge.ClientUnknown, + expectedClient: aibridge.ClientUnknown, + }, + { + name: "blocking_web_search", + fixture: fixtures.OaiResponsesBlockingWebSearch, + expectModel: "gpt-5", + expectPromptRecorded: "What is the current weather in Cape Town?", + expectToolRecorded: &recorder.ToolUsageRecord{ + MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Tool: "web_search_call", + ToolCallID: "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Injected: false, }, - { - name: "blocking_web_search", - fixture: fixtures.OaiResponsesBlockingWebSearch, - expectModel: "gpt-5", - expectPromptRecorded: "What is the current weather in Cape Town?", - expectToolRecorded: &recorder.ToolUsageRecord{ - MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", - Tool: "web_search_call", - ToolCallID: "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", - Injected: false, - }, - expectTokenUsage: &recorder.TokenUsageRecord{ - MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", - Input: 42, - Output: 150, - ExtraTokenTypes: map[string]int64{ - "input_cached": 0, - "output_reasoning": 0, - "total_tokens": 192, - }, + expectTokenUsage: &recorder.TokenUsageRecord{ + MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Input: 42, + Output: 150, + ExtraTokenTypes: map[string]int64{ + "input_cached": 0, + "output_reasoning": 0, + "total_tokens": 192, }, - expectedClient: aibridge.ClientUnknown, }, - { - name: "streaming_simple", fixture: fixtures.OaiResponsesStreamingSimple, + expectedClient: aibridge.ClientUnknown, + }, + { + name: "streaming_simple", fixture: fixtures.OaiResponsesStreamingSimple, streaming: true, expectModel: "gpt-4o-mini", expectPromptRecorded: "tell me a joke", From 34c9a190da9af8703bebd440e3f8a558062b239b Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 21 Apr 2026 11:50:04 +0000 Subject: [PATCH 8/8] feat(recorder): add ItemID field to ToolUsageRecord Captures the output item's unique ID separately from ToolCallID. This avoids conflating two semantically different identifiers: - ItemID: the output item's unique id (always present) - ToolCallID: the call_id used to correlate agentic tool output (empty for hosted tools like web_search_call) --- intercept/responses/base.go | 7 ++++++- intercept/responses/base_test.go | 18 ++++++++++++------ intercept/responses/injected_tools.go | 1 + internal/integrationtest/responses_test.go | 12 ++++++++---- recorder/types.go | 1 + 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/intercept/responses/base.go b/intercept/responses/base.go index b81bf23a..76f142dd 100644 --- a/intercept/responses/base.go +++ b/intercept/responses/base.go @@ -207,6 +207,7 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte var ( args recorder.ToolArgs toolName string + itemID string callID string ) @@ -214,11 +215,13 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte case string(constant.ValueOf[constant.FunctionCall]()): args = i.parseFunctionCallJSONArgs(ctx, item.Arguments) toolName = item.Name + itemID = item.ID callID = item.CallID case string(constant.ValueOf[constant.CustomToolCall]()): args = item.Input toolName = item.Name + itemID = item.ID callID = item.CallID // Agentic tools: the client sends a corresponding *_output @@ -231,6 +234,7 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte if toolName == "" { toolName = item.Type } + itemID = item.ID callID = item.CallID // Hosted tools: executed server-side, these output items @@ -246,7 +250,7 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte if toolName == "" { toolName = item.Type } - callID = item.ID + itemID = item.ID default: continue @@ -255,6 +259,7 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte if err := i.recorder.RecordToolUsage(ctx, &recorder.ToolUsageRecord{ InterceptionID: i.ID().String(), MsgID: response.ID, + ItemID: itemID, ToolCallID: callID, Tool: toolName, Args: args, diff --git a/intercept/responses/base_test.go b/intercept/responses/base_test.go index 7e1cc35e..d787813c 100644 --- a/intercept/responses/base_test.go +++ b/intercept/responses/base_test.go @@ -113,6 +113,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_456", + ItemID: "", ToolCallID: "call_abc", Tool: "get_weather", Args: "", @@ -160,6 +161,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_789", + ItemID: "", ToolCallID: "call_1", Tool: "get_weather", Args: map[string]any{"location": "NYC"}, @@ -168,6 +170,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_789", + ItemID: "", ToolCallID: "call_2", Tool: "bad_json_args", Args: `{"bad": args`, @@ -176,6 +179,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_789", + ItemID: "", ToolCallID: "call_3", Tool: "search", Args: `{\"query\": \"test\"}`, @@ -184,6 +188,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_789", + ItemID: "", ToolCallID: "call_4", Tool: "calculate", Args: map[string]any{"a": float64(1), "b": float64(2)}, @@ -206,7 +211,8 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_ws", - ToolCallID: "ws_abc", + ItemID: "ws_abc", + ToolCallID: "", Tool: "web_search_call", Injected: false, }, @@ -268,7 +274,7 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_all", - ToolCallID: "ws_1", + ItemID: "ws_1", Tool: "web_search_call", Injected: false, }, @@ -303,28 +309,28 @@ func TestRecordToolUsage(t *testing.T) { { InterceptionID: id.String(), MsgID: "resp_all", - ToolCallID: "ci_1", + ItemID: "ci_1", Tool: "code_interpreter_call", Injected: false, }, { InterceptionID: id.String(), MsgID: "resp_all", - ToolCallID: "mcp_1", + ItemID: "mcp_1", Tool: "my_mcp_tool", Injected: false, }, { InterceptionID: id.String(), MsgID: "resp_all", - ToolCallID: "fs_1", + ItemID: "fs_1", Tool: "file_search_call", Injected: false, }, { InterceptionID: id.String(), MsgID: "resp_all", - ToolCallID: "ig_1", + ItemID: "ig_1", Tool: "image_generation_call", Injected: false, }, diff --git a/intercept/responses/injected_tools.go b/intercept/responses/injected_tools.go index dd44014b..b5872467 100644 --- a/intercept/responses/injected_tools.go +++ b/intercept/responses/injected_tools.go @@ -178,6 +178,7 @@ func (i *responsesInterceptionBase) invokeInjectedTool(ctx context.Context, resp _ = i.recorder.RecordToolUsage(ctx, &recorder.ToolUsageRecord{ InterceptionID: i.ID().String(), MsgID: responseID, + ItemID: fc.ID, ToolCallID: fc.CallID, ServerURL: &tool.ServerURL, Tool: tool.Name, diff --git a/internal/integrationtest/responses_test.go b/internal/integrationtest/responses_test.go index b432fbbd..d3a4983f 100644 --- a/internal/integrationtest/responses_test.go +++ b/internal/integrationtest/responses_test.go @@ -73,6 +73,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { expectToolRecorded: &recorder.ToolUsageRecord{ MsgID: "resp_0da6045a8b68fa5200695fa23dcc2c81a19c849f627abf8a31", Tool: "add", + ItemID: "fc_0da6045a8b68fa5200695fa23e198081a19bf68887d47ae93d", ToolCallID: "call_CJSaa2u51JG996575oVljuNq", Args: map[string]any{"a": float64(3), "b": float64(5)}, Injected: false, @@ -115,6 +116,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { expectToolRecorded: &recorder.ToolUsageRecord{ MsgID: "resp_09c614364030cdf000696942589da081a0af07f5859acb7308", Tool: "code_exec", + ItemID: "ctc_09c614364030cdf0006969425bf33481a09cc0f9522af2d980", ToolCallID: "call_haf8njtwrVZ1754Gm6fjAtuA", Args: "print(\"hello world\")", Injected: false, @@ -171,10 +173,10 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { expectModel: "gpt-5", expectPromptRecorded: "What is the current weather in Cape Town?", expectToolRecorded: &recorder.ToolUsageRecord{ - MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", - Tool: "web_search_call", - ToolCallID: "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", - Injected: false, + MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Tool: "web_search_call", + ItemID: "ws_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + Injected: false, }, expectTokenUsage: &recorder.TokenUsageRecord{ MsgID: "resp_a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", @@ -234,6 +236,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { expectToolRecorded: &recorder.ToolUsageRecord{ MsgID: "resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458", Tool: "add", + ItemID: "fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e", ToolCallID: "call_7VaiUXZYuuuwWwviCrckxq6t", Args: map[string]any{"a": float64(3), "b": float64(5)}, Injected: false, @@ -278,6 +281,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { expectToolRecorded: &recorder.ToolUsageRecord{ MsgID: "resp_0c26996bc41c2a0500696942e83634819fb71b2b8ff8a4a76c", Tool: "code_exec", + ItemID: "ctc_0c26996bc41c2a0500696942ee6db8819fa6e841317eecbfb2", ToolCallID: "call_2gSnF58IEhXLwlbnqbm5XKMd", Args: "print(\"hello world\")", Injected: false, diff --git a/recorder/types.go b/recorder/types.go index cd541eeb..22a923f1 100644 --- a/recorder/types.go +++ b/recorder/types.go @@ -75,6 +75,7 @@ type ToolUsageRecord struct { InterceptionID string MsgID string Tool string + ItemID string ToolCallID string ServerURL *string Args ToolArgs