From d431ebaf9ded6bcfc33b2f8d51f4f07dbfa6e4d5 Mon Sep 17 00:00:00 2001 From: Pascal Birchler Date: Mon, 20 Jul 2026 13:08:32 +0200 Subject: [PATCH 1/3] Remove unused `listToolsFromPage` function --- evals-cli/src/evaluator/browser.ts | 46 ------------------------------ evals-cli/src/evaluator/index.ts | 3 +- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/evals-cli/src/evaluator/browser.ts b/evals-cli/src/evaluator/browser.ts index 79df444..39a01ed 100644 --- a/evals-cli/src/evaluator/browser.ts +++ b/evals-cli/src/evaluator/browser.ts @@ -46,52 +46,6 @@ export async function launchBrowser(): Promise { }); } -/** - * Launches Chrome Canary, navigates to the given URL, and retrieves the list - * of tools exposed by the page via Puppeteer. - * - * Requires Chrome Canary 150+ with the `chrome://flags/#enable-webmcp-testing` - * flag enabled. The browser is always closed after the tools are retrieved, - * even if an error occurs. - */ -export async function listToolsFromPage(url: string): Promise { - const executablePath = await findChromePath(); - let browser: Browser | null = null; - - try { - console.log(`Launching Chrome Canary from: ${executablePath}`); - browser = await launchBrowser(); - - const page = await browser.newPage(); - - console.log(`Navigating to: ${url}`); - const response = await page.goto(url, { - waitUntil: "networkidle2", - timeout: 30000, - }); - - if (!response || !response.ok()) { - throw new Error( - `Failed to navigate to ${url}. HTTP status: ${response?.status() ?? "unknown"}`, - ); - } - - const rawTools = await getToolsFromBrowserPage(page); - if (rawTools.length === 0) { - throw new Error( - `WebMCP Tools are not available on ${url} (0 tools registered on page).\nDebug info: [URL="${url}", Executable="${executablePath}", Flags="${PUPPETEER_FLAGS.join(" ")}"]`, - ); - } - - console.log(`Found ${rawTools.length} tool(s) via Puppeteer/Native API.`); - return mapRawBrowserToolsToConfig(rawTools, []); - } finally { - if (browser) { - await browser.close(); - } - } -} - export class BrowserToolRegistry implements ToolRegistry { private currentTools: Tool[] = []; diff --git a/evals-cli/src/evaluator/index.ts b/evals-cli/src/evaluator/index.ts index 59bc772..e518dde 100644 --- a/evals-cli/src/evaluator/index.ts +++ b/evals-cli/src/evaluator/index.ts @@ -3,8 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { listToolsFromPage } from "./browser.js"; import { executeInBrowserEvals } from "./browserEvaluator.js"; import { executeLocalEvals } from "./localEvaluator.js"; -export { executeInBrowserEvals, executeLocalEvals, listToolsFromPage }; +export { executeInBrowserEvals, executeLocalEvals }; From 6e837eb0a0869ff8286c389b346ed6a5e7a1cd16 Mon Sep 17 00:00:00 2001 From: Pascal Birchler Date: Mon, 20 Jul 2026 14:35:53 +0200 Subject: [PATCH 2/3] Hook into `toolactivated` to gracefully handle pending form submission --- evals-cli/src/evaluator/browser.ts | 36 +++++++++++++++---- .../src/test/browserToolRegistry.test.ts | 10 ++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/evals-cli/src/evaluator/browser.ts b/evals-cli/src/evaluator/browser.ts index 39a01ed..828f6f9 100644 --- a/evals-cli/src/evaluator/browser.ts +++ b/evals-cli/src/evaluator/browser.ts @@ -73,12 +73,36 @@ export class BrowserToolRegistry implements ToolRegistry { const tools = await mc.getTools(); const tool = tools.find((item) => item.name === name); if (tool) { - const resStr = await mc.executeTool(tool, JSON.stringify(callArgs || {})); - try { - return { success: true, data: JSON.parse(resStr as string) }; - } catch { - return { success: true, data: resStr }; - } + return new Promise((resolve) => { + let timer: any = null; + + const onActivated = (e: any) => { + if (!e.toolName || e.toolName === name) { + timer = setTimeout(() => { + window.removeEventListener("toolactivated", onActivated); + resolve({ success: true, data: "pending form submission" }); + }, 1000); + } + }; + + window.addEventListener("toolactivated", onActivated); + + mc.executeTool(tool, JSON.stringify(callArgs || {})) + .then((resStr: any) => { + if (timer) clearTimeout(timer); + window.removeEventListener("toolactivated", onActivated); + try { + resolve({ success: true, data: JSON.parse(resStr as string) }); + } catch { + resolve({ success: true, data: resStr }); + } + }) + .catch((err: any) => { + if (timer) clearTimeout(timer); + window.removeEventListener("toolactivated", onActivated); + resolve({ success: false, error: err?.message || String(err) }); + }); + }); } } } diff --git a/evals-cli/src/test/browserToolRegistry.test.ts b/evals-cli/src/test/browserToolRegistry.test.ts index 148a121..35fed32 100644 --- a/evals-cli/src/test/browserToolRegistry.test.ts +++ b/evals-cli/src/test/browserToolRegistry.test.ts @@ -78,6 +78,16 @@ describe("BrowserToolRegistry", () => { assert.deepStrictEqual(page.evaluateCalls[0].args[1], { id: "btn-1" }); }); + it("should return 'pending form submission' when tool result data is 'pending form submission'", async () => { + const page = new MockBrowserPage(); + page.evaluateResult = { success: true, data: "pending form submission" }; + + const registry = new BrowserToolRegistry(page); + const result = await registry.executeTool("book_table", { guests: 2 }); + + assert.strictEqual(result, "pending form submission"); + }); + it("should return error if page execution reports success: false", async () => { const page = new MockBrowserPage(); page.evaluateResult = { success: false }; From dde02a1c776593eee63fc5b0a84e9174d7e285d0 Mon Sep 17 00:00:00 2001 From: Pascal Birchler Date: Mon, 20 Jul 2026 15:30:11 +0200 Subject: [PATCH 3/3] Improve handling of manual form submissions --- .../french-bistro/evals-toolautosubmit.json | 65 +++++++++++++++++++ evals-cli/examples/french-bistro/evals.json | 6 +- evals-cli/src/backends/vercel.ts | 20 +++++- evals-cli/src/commands/index.ts | 36 +++++++++- evals-cli/src/report/report.ts | 63 +++++++++++++++--- evals-cli/src/test/utils.test.ts | 27 ++++++++ evals-cli/src/types/evals.ts | 3 + evals-cli/src/types/tools.ts | 1 + evals-cli/src/utils.ts | 11 +++- 9 files changed, 216 insertions(+), 16 deletions(-) create mode 100644 evals-cli/examples/french-bistro/evals-toolautosubmit.json diff --git a/evals-cli/examples/french-bistro/evals-toolautosubmit.json b/evals-cli/examples/french-bistro/evals-toolautosubmit.json new file mode 100644 index 0000000..bdef454 --- /dev/null +++ b/evals-cli/examples/french-bistro/evals-toolautosubmit.json @@ -0,0 +1,65 @@ +[ + { + "name": "Book Bistro Table: Bob Smith Terrace Anniversary", + "messages": [ + { + "role": "user", + "type": "message", + "content": "Can I get a table at the bistro for 2 people on the Terrace tomorrow at 18:30? The name is Bob Smith, phone 555-123-4567. We are celebrating an anniversary." + } + ], + "expectedCall": [ + { + "functionName": "book_table_le_petit_bistro", + "arguments": { + "name": { + "$pattern": "Bob.*Smith" + }, + "phone": { + "$pattern": "555.*123.*4567" + }, + "time": "18:30", + "date": { + "$type": "string" + }, + "guests": "2", + "seating": "Terrace", + "requests": { + "$contains": "anniversary" + } + }, + "result": { + "$contains": "welcoming you" + } + } + ] + }, + { + "name": "Book Bistro Table: Alice Wong Private Booth", + "messages": [ + { + "role": "user", + "type": "message", + "content": "I need to book a Private Booth for 6 people tonight at 20:00. Name: Alice Wong. Phone: 555-987-6543." + } + ], + "expectedCall": [ + { + "functionName": "book_table_le_petit_bistro", + "arguments": { + "name": "Alice Wong", + "phone": "555-987-6543", + "time": "20:00", + "date": { + "$type": "string" + }, + "guests": "6", + "seating": "Private Booth" + }, + "result": { + "$contains": "welcoming you" + } + } + ] + } +] diff --git a/evals-cli/examples/french-bistro/evals.json b/evals-cli/examples/french-bistro/evals.json index 0127ed4..f147ec6 100644 --- a/evals-cli/examples/french-bistro/evals.json +++ b/evals-cli/examples/french-bistro/evals.json @@ -27,7 +27,8 @@ "requests": { "$contains": "anniversary" } - } + }, + "result": "pending form submission" } ] }, @@ -52,7 +53,8 @@ }, "guests": "6", "seating": "Private Booth" - } + }, + "result": "pending form submission" } ] } diff --git a/evals-cli/src/backends/vercel.ts b/evals-cli/src/backends/vercel.ts index 873b78d..36a564b 100644 --- a/evals-cli/src/backends/vercel.ts +++ b/evals-cli/src/backends/vercel.ts @@ -87,9 +87,16 @@ export class VercelBackend implements Backend { for (const step of aiResult.steps ?? []) { for (const call of (step.toolCalls ?? []) as any[]) { if (validToolNames.has(call.toolName)) { + const matchingResult: any = (step.toolResults ?? []).find( + (r: any) => r.toolCallId === call.toolCallId || r.toolName === call.toolName, + ); + const result = matchingResult + ? (matchingResult.result ?? matchingResult.output) + : undefined; toolCalls.push({ functionName: call.toolName, args: call.input || call.args || call.arguments || {}, + result, }); } } @@ -185,13 +192,22 @@ export class VercelBackend implements Backend { // Gather executed tool calls across all steps const executedCalls: ToolCall[] = []; - if (resultPayload.steps && resultPayload.steps.length > 0) { - for (const step of resultPayload.steps) { + const stepsToIterate = + resultPayload.steps && resultPayload.steps.length > 0 ? resultPayload.steps : stepsHistory; + if (stepsToIterate && stepsToIterate.length > 0) { + for (const step of stepsToIterate) { if (step.toolCalls && step.toolCalls.length > 0) { for (const call of step.toolCalls) { + const matchingResult: any = (step.toolResults ?? []).find( + (r: any) => r.toolCallId === call.toolCallId || r.toolName === call.toolName, + ); + const result = matchingResult + ? (matchingResult.result ?? matchingResult.output) + : undefined; executedCalls.push({ functionName: call.toolName, args: (call as any).input || (call as any).args || (call as any).arguments || {}, + result, }); } } diff --git a/evals-cli/src/commands/index.ts b/evals-cli/src/commands/index.ts index 8aadd15..a657594 100644 --- a/evals-cli/src/commands/index.ts +++ b/evals-cli/src/commands/index.ts @@ -162,11 +162,44 @@ export async function runWebCommand(options: CommandOptions, command?: Command): } } +import { matchesArgument } from "../matcher.js"; + +function getFailureDetail(res: any): string { + if (res.outcome === "pass") return "-"; + if (!res.response) return "No tool called"; + + const expected = res.test.expectedCall?.[0] as FunctionCall | undefined; + if (!expected) return "Unexpected tool call"; + + if (expected.functionName !== res.response.functionName) { + return `Function mismatch (expected "${expected.functionName}", got "${res.response.functionName}")`; + } + + if (expected.arguments != null && !matchesArgument(expected.arguments, res.response.args)) { + return "Arguments mismatch"; + } + + if (expected.result !== undefined && !matchesArgument(expected.result, res.response.result)) { + const expStr = + typeof expected.result === "object" + ? JSON.stringify(expected.result) + : String(expected.result); + const actStr = + typeof res.response.result === "object" + ? JSON.stringify(res.response.result) + : String(res.response.result ?? null); + const truncatedAct = actStr.length > 40 ? actStr.slice(0, 37) + "..." : actStr; + return `Result mismatch: expected "${expStr}", got "${truncatedAct}"`; + } + + return res.outcome === "error" ? "Execution error" : "Failed"; +} + function printConsoleSummary(finalResults: any): void { console.log("\n" + chalk.bold.underline("Evaluation Summary") + "\n"); const table = new Table({ - head: ["Step", "Status", "Expected Function", "Actual Function"], + head: ["Step", "Status", "Expected Function", "Actual Function", "Details"], style: { head: ["cyan"], border: ["grey"], @@ -181,6 +214,7 @@ function printConsoleSummary(finalResults: any): void { passed ? chalk.green("PASS") : chalk.red(res.outcome.toUpperCase()), (res.test.expectedCall?.[0] as FunctionCall)?.functionName || "-", res.response?.functionName || "-", + getFailureDetail(res), ]); } diff --git a/evals-cli/src/report/report.ts b/evals-cli/src/report/report.ts index 6de6523..340a38e 100644 --- a/evals-cli/src/report/report.ts +++ b/evals-cli/src/report/report.ts @@ -348,18 +348,24 @@ function renderRunIteration(run: TestRun, totalRuns: number): string { function renderStepDetails(stepEval: TestStep, totalSteps: number): string { const { stepIndex, result } = stepEval; + const expected = result.test.expectedCall?.[0] as FunctionCall | undefined; const functionNameOutcome = - (result.test.expectedCall?.[0] as FunctionCall)?.functionName === result.response?.functionName + expected?.functionName === result.response?.functionName ? "pass" : "fail"; + + const argsOutcome = + expected?.arguments == null || matchesArgument(expected?.arguments, result.response?.args) ? "pass" : "fail"; - const argsOutcome = matchesArgument( - (result.test.expectedCall?.[0] as FunctionCall)?.arguments, - result.response?.args, - ) - ? "pass" - : "fail"; + const hasExpectedResult = expected?.result !== undefined; + const hasActualResult = result.response?.result !== undefined; + const hasResultRow = hasExpectedResult || hasActualResult; + + const resultOutcome = + expected?.result === undefined || matchesArgument(expected?.result, result.response?.result) + ? "pass" + : "fail"; const statusColor = result.outcome === "pass" @@ -391,7 +397,7 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string { Function Name - ${escapeHtml((result.test.expectedCall?.[0] as FunctionCall)?.functionName || null)} + ${escapeHtml(expected?.functionName || null)} ${escapeHtml(result.response?.functionName || null)} @@ -403,7 +409,7 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string { Arguments
- 
${escapeHtml(JSON.stringify(sortObjectKeys((result.test.expectedCall?.[0] as FunctionCall)?.arguments) || null, null, 2))}
+ 
${escapeHtml(JSON.stringify(sortObjectKeys(expected?.arguments) || null, null, 2))}
@@ -417,6 +423,45 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string { + ${ + hasResultRow + ? ` + + Result + + ${ + hasExpectedResult + ? `
+ 
${escapeHtml(
+                          typeof expected?.result === "object"
+                            ? JSON.stringify(sortObjectKeys(expected.result), null, 2)
+                            : String(expected?.result ?? null),
+                        )}
+
` + : `Unconstrained` + } + + + ${ + hasActualResult + ? `
+ 
${escapeHtml(
+                          typeof result.response?.result === "object"
+                            ? JSON.stringify(sortObjectKeys(result.response.result), null, 2)
+                            : String(result.response?.result ?? null),
+                        )}
+
` + : `None` + } + + + + ${resultOutcome.toUpperCase()} + + + ` + : "" + } diff --git a/evals-cli/src/test/utils.test.ts b/evals-cli/src/test/utils.test.ts index bb5043d..24a06f7 100644 --- a/evals-cli/src/test/utils.test.ts +++ b/evals-cli/src/test/utils.test.ts @@ -624,6 +624,33 @@ describe("functionCallOutcome", () => { "pass", ); }); + + it("passes when expected.result matches actual.result", () => { + assert.strictEqual( + functionCallOutcome( + { functionName: "foo", result: "pending form submission" }, + { functionName: "foo", args: {}, result: "pending form submission" }, + ), + "pass", + ); + assert.strictEqual( + functionCallOutcome( + { functionName: "foo", result: { $contains: "confirmed" } }, + { functionName: "foo", args: {}, result: "Reservation confirmed for 2 people" }, + ), + "pass", + ); + }); + + it("fails when expected.result does not match actual.result", () => { + assert.strictEqual( + functionCallOutcome( + { functionName: "foo", result: "pending form submission" }, + { functionName: "foo", args: {}, result: "Reservation confirmed" }, + ), + "fail", + ); + }); }); describe("countExpectedCalls with optional", () => { diff --git a/evals-cli/src/types/evals.ts b/evals-cli/src/types/evals.ts index e64ac9f..7578f85 100644 --- a/evals-cli/src/types/evals.ts +++ b/evals-cli/src/types/evals.ts @@ -43,6 +43,9 @@ export type FunctionCall = { // Optional: when omitted (or explicitly null), the eval imposes no // constraint on the tool call's arguments — any actual args are accepted. arguments?: object | null; + // Optional: when specified, checks whether the actual tool execution result + // matches this expected value or constraint. + result?: unknown; // Optional: mock output returned to the model when it invokes this tool // during a local (non-browser) multi-step trajectory. When omitted, an // empty object `{}` is returned. Only consulted by `executeLocalEvals` — diff --git a/evals-cli/src/types/tools.ts b/evals-cli/src/types/tools.ts index 96eb7fa..a18bde7 100644 --- a/evals-cli/src/types/tools.ts +++ b/evals-cli/src/types/tools.ts @@ -6,6 +6,7 @@ export type ToolCall = { args: object; functionName: string; + result?: unknown; }; export type Tool = { diff --git a/evals-cli/src/utils.ts b/evals-cli/src/utils.ts index 2e22a24..09bfc97 100644 --- a/evals-cli/src/utils.ts +++ b/evals-cli/src/utils.ts @@ -49,9 +49,16 @@ export function functionCallOutcome( // An eval that omits `arguments` (or sets it to null) imposes no constraint // on the tool call's arguments. Treat any actual args — including the empty // object `{}` that most SDKs emit for no-arg tool calls — as a match. - if (expected.arguments == null) return "pass"; + if (expected.arguments != null) { + if (!matchesArgument(expected.arguments, actual.args)) return "fail"; + } + + // An eval that specifies `result` imposes a constraint on the tool execution result. + if (expected.result !== undefined) { + if (!matchesArgument(expected.result, actual.result)) return "fail"; + } - return matchesArgument(expected.arguments, actual.args) ? "pass" : "fail"; + return "pass"; } export interface TrajectoryResult {