Handle manual form submissions in evals-cli

evals-cli/examples/french-bistro/evals-toolautosubmit.json

@@ -0,0 +1,65 @@
+[
+  {
+    "name": "Book Bistro Table: Bob Smith Terrace Anniversary",
+    "messages": [
+      {
+        "role": "user",
+        "type": "message",
+        "content": "Can I get a table at the bistro for 2 people on the Terrace tomorrow at 18:30? The name is Bob Smith, phone 555-123-4567. We are celebrating an anniversary."
+      }
+    ],
+    "expectedCall": [
+      {
+        "functionName": "book_table_le_petit_bistro",
+        "arguments": {
+          "name": {
+            "$pattern": "Bob.*Smith"
+          },
+          "phone": {
+            "$pattern": "555.*123.*4567"
+          },
+          "time": "18:30",
+          "date": {
+            "$type": "string"
+          },
+          "guests": "2",
+          "seating": "Terrace",
+          "requests": {
+            "$contains": "anniversary"
+          }
+        },
+        "result": {
+          "$contains": "welcoming you"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Book Bistro Table: Alice Wong Private Booth",
+    "messages": [
+      {
+        "role": "user",
+        "type": "message",
+        "content": "I need to book a Private Booth for 6 people tonight at 20:00. Name: Alice Wong. Phone: 555-987-6543."
+      }
+    ],
+    "expectedCall": [
+      {
+        "functionName": "book_table_le_petit_bistro",
+        "arguments": {
+          "name": "Alice Wong",
+          "phone": "555-987-6543",
+          "time": "20:00",
+          "date": {
+            "$type": "string"
+          },
+          "guests": "6",
+          "seating": "Private Booth"
+        },
+        "result": {
+          "$contains": "welcoming you"
+        }
+      }
+    ]
+  }
+]

evals-cli/examples/french-bistro/evals.json

@@ -27,7 +27,8 @@
           "requests": {
             "$contains": "anniversary"
           }
-        }
+        },
+        "result": "pending form submission"
       }
     ]
   },
@@ -52,7 +53,8 @@
           },
           "guests": "6",
           "seating": "Private Booth"
-        }
+        },
+        "result": "pending form submission"
       }
     ]
   }

evals-cli/src/backends/vercel.ts

@@ -87,9 +87,16 @@ export class VercelBackend implements Backend {
     for (const step of aiResult.steps ?? []) {
       for (const call of (step.toolCalls ?? []) as any[]) {
         if (validToolNames.has(call.toolName)) {
+          const matchingResult: any = (step.toolResults ?? []).find(
+            (r: any) => r.toolCallId === call.toolCallId || r.toolName === call.toolName,
+          );
+          const result = matchingResult
+            ? (matchingResult.result ?? matchingResult.output)
+            : undefined;
           toolCalls.push({
             functionName: call.toolName,
             args: call.input || call.args || call.arguments || {},
+            result,
           });
         }
       }
@@ -185,13 +192,22 @@ export class VercelBackend implements Backend {
 
       // Gather executed tool calls across all steps
       const executedCalls: ToolCall[] = [];
-      if (resultPayload.steps && resultPayload.steps.length > 0) {
-        for (const step of resultPayload.steps) {
+      const stepsToIterate =
+        resultPayload.steps && resultPayload.steps.length > 0 ? resultPayload.steps : stepsHistory;
+      if (stepsToIterate && stepsToIterate.length > 0) {
+        for (const step of stepsToIterate) {
           if (step.toolCalls && step.toolCalls.length > 0) {
             for (const call of step.toolCalls) {
+              const matchingResult: any = (step.toolResults ?? []).find(
+                (r: any) => r.toolCallId === call.toolCallId || r.toolName === call.toolName,
+              );
+              const result = matchingResult
+                ? (matchingResult.result ?? matchingResult.output)
+                : undefined;
               executedCalls.push({
                 functionName: call.toolName,
                 args: (call as any).input || (call as any).args || (call as any).arguments || {},
+                result,
               });
             }
           }

evals-cli/src/commands/index.ts

@@ -162,11 +162,44 @@ export async function runWebCommand(options: CommandOptions, command?: Command):
   }
 }
 
+import { matchesArgument } from "../matcher.js";
+
+function getFailureDetail(res: any): string {
+  if (res.outcome === "pass") return "-";
+  if (!res.response) return "No tool called";
+
+  const expected = res.test.expectedCall?.[0] as FunctionCall | undefined;
+  if (!expected) return "Unexpected tool call";
+
+  if (expected.functionName !== res.response.functionName) {
+    return `Function mismatch (expected "${expected.functionName}", got "${res.response.functionName}")`;
+  }
+
+  if (expected.arguments != null && !matchesArgument(expected.arguments, res.response.args)) {
+    return "Arguments mismatch";
+  }
+
+  if (expected.result !== undefined && !matchesArgument(expected.result, res.response.result)) {
+    const expStr =
+      typeof expected.result === "object"
+        ? JSON.stringify(expected.result)
+        : String(expected.result);
+    const actStr =
+      typeof res.response.result === "object"
+        ? JSON.stringify(res.response.result)
+        : String(res.response.result ?? null);
+    const truncatedAct = actStr.length > 40 ? actStr.slice(0, 37) + "..." : actStr;
+    return `Result mismatch: expected "${expStr}", got "${truncatedAct}"`;
+  }
+
+  return res.outcome === "error" ? "Execution error" : "Failed";
+}
+
 function printConsoleSummary(finalResults: any): void {
   console.log("\n" + chalk.bold.underline("Evaluation Summary") + "\n");
 
   const table = new Table({
-    head: ["Step", "Status", "Expected Function", "Actual Function"],
+    head: ["Step", "Status", "Expected Function", "Actual Function", "Details"],
     style: {
       head: ["cyan"],
       border: ["grey"],
@@ -181,6 +214,7 @@ function printConsoleSummary(finalResults: any): void {
       passed ? chalk.green("PASS") : chalk.red(res.outcome.toUpperCase()),
       (res.test.expectedCall?.[0] as FunctionCall)?.functionName || "-",
       res.response?.functionName || "-",
+      getFailureDetail(res),
     ]);
   }
 

evals-cli/src/evaluator/browser.ts

@@ -46,52 +46,6 @@ export async function launchBrowser(): Promise<Browser> {
   });
 }
 
-/**
- * Launches Chrome Canary, navigates to the given URL, and retrieves the list
- * of tools exposed by the page via Puppeteer.
- *
- * Requires Chrome Canary 150+ with the `chrome://flags/#enable-webmcp-testing`
- * flag enabled. The browser is always closed after the tools are retrieved,
- * even if an error occurs.
- */
-export async function listToolsFromPage(url: string): Promise<Tool[]> {
-  const executablePath = await findChromePath();
-  let browser: Browser | null = null;
-
-  try {
-    console.log(`Launching Chrome Canary from: ${executablePath}`);
-    browser = await launchBrowser();
-
-    const page = await browser.newPage();
-
-    console.log(`Navigating to: ${url}`);
-    const response = await page.goto(url, {
-      waitUntil: "networkidle2",
-      timeout: 30000,
-    });
-
-    if (!response || !response.ok()) {
-      throw new Error(
-        `Failed to navigate to ${url}. HTTP status: ${response?.status() ?? "unknown"}`,
-      );
-    }
-
-    const rawTools = await getToolsFromBrowserPage(page);
-    if (rawTools.length === 0) {
-      throw new Error(
-        `WebMCP Tools are not available on ${url} (0 tools registered on page).\nDebug info: [URL="${url}", Executable="${executablePath}", Flags="${PUPPETEER_FLAGS.join(" ")}"]`,
-      );
-    }
-
-    console.log(`Found ${rawTools.length} tool(s) via Puppeteer/Native API.`);
-    return mapRawBrowserToolsToConfig(rawTools, []);
-  } finally {
-    if (browser) {
-      await browser.close();
-    }
-  }
-}
-
 export class BrowserToolRegistry implements ToolRegistry {
   private currentTools: Tool[] = [];
 
@@ -119,12 +73,36 @@ export class BrowserToolRegistry implements ToolRegistry {
               const tools = await mc.getTools();
               const tool = tools.find((item) => item.name === name);
               if (tool) {
-                const resStr = await mc.executeTool(tool, JSON.stringify(callArgs || {}));
-                try {
-                  return { success: true, data: JSON.parse(resStr as string) };
-                } catch {
-                  return { success: true, data: resStr };
-                }
+                return new Promise((resolve) => {
+                  let timer: any = null;
+
+                  const onActivated = (e: any) => {
+                    if (!e.toolName || e.toolName === name) {
+                      timer = setTimeout(() => {
+                        window.removeEventListener("toolactivated", onActivated);
+                        resolve({ success: true, data: "pending form submission" });
+                      }, 1000);
+                    }
+                  };
+
+                  window.addEventListener("toolactivated", onActivated);
+
+                  mc.executeTool(tool, JSON.stringify(callArgs || {}))
+                    .then((resStr: any) => {
+                      if (timer) clearTimeout(timer);
+                      window.removeEventListener("toolactivated", onActivated);
+                      try {
+                        resolve({ success: true, data: JSON.parse(resStr as string) });
+                      } catch {
+                        resolve({ success: true, data: resStr });
+                      }
+                    })
+                    .catch((err: any) => {
+                      if (timer) clearTimeout(timer);
+                      window.removeEventListener("toolactivated", onActivated);
+                      resolve({ success: false, error: err?.message || String(err) });
+                    });
+                });
               }
             }
           }

evals-cli/src/evaluator/index.ts

@@ -3,8 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { listToolsFromPage } from "./browser.js";
 import { executeInBrowserEvals } from "./browserEvaluator.js";
 import { executeLocalEvals } from "./localEvaluator.js";
 
-export { executeInBrowserEvals, executeLocalEvals, listToolsFromPage };
+export { executeInBrowserEvals, executeLocalEvals };

evals-cli/src/report/report.ts

@@ -348,18 +348,24 @@ function renderRunIteration(run: TestRun, totalRuns: number): string {
 
 function renderStepDetails(stepEval: TestStep, totalSteps: number): string {
   const { stepIndex, result } = stepEval;
+  const expected = result.test.expectedCall?.[0] as FunctionCall | undefined;
 
   const functionNameOutcome =
-    (result.test.expectedCall?.[0] as FunctionCall)?.functionName === result.response?.functionName
+    expected?.functionName === result.response?.functionName ? "pass" : "fail";
+
+  const argsOutcome =
+    expected?.arguments == null || matchesArgument(expected?.arguments, result.response?.args)
       ? "pass"
       : "fail";
 
-  const argsOutcome = matchesArgument(
-    (result.test.expectedCall?.[0] as FunctionCall)?.arguments,
-    result.response?.args,
-  )
-    ? "pass"
-    : "fail";
+  const hasExpectedResult = expected?.result !== undefined;
+  const hasActualResult = result.response?.result !== undefined;
+  const hasResultRow = hasExpectedResult || hasActualResult;
+
+  const resultOutcome =
+    expected?.result === undefined || matchesArgument(expected?.result, result.response?.result)
+      ? "pass"
+      : "fail";
 
   const statusColor =
     result.outcome === "pass"
@@ -391,7 +397,7 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string {
           <tbody class="divide-y divide-slate-100">
             <tr class="hover:bg-slate-50/30">
               <td class="px-4 py-3 font-semibold text-slate-700 whitespace-nowrap">Function Name</td>
-              <td class="px-4 py-3"><code class="px-1.5 py-0.5 bg-slate-100 text-slate-800 rounded font-mono text-xs">${escapeHtml((result.test.expectedCall?.[0] as FunctionCall)?.functionName || null)}</code></td>
+              <td class="px-4 py-3"><code class="px-1.5 py-0.5 bg-slate-100 text-slate-800 rounded font-mono text-xs">${escapeHtml(expected?.functionName || null)}</code></td>
               <td class="px-4 py-3"><code class="px-1.5 py-0.5 bg-slate-100 text-slate-800 rounded font-mono text-xs">${escapeHtml(result.response?.functionName || null)}</code></td>
               <td class="px-4 py-3">
                 <span class="${functionNameOutcome === "pass" ? "text-emerald-600" : "text-rose-600"} font-bold text-xs">
@@ -403,7 +409,7 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string {
               <td class="px-4 py-3 font-semibold text-slate-700 whitespace-nowrap align-top">Arguments</td>
               <td class="px-4 py-3">
                 <div class="bg-slate-800 rounded-md p-3 overflow-x-auto max-w-md">
-                  <pre class="text-xs text-slate-200 font-mono m-0 leading-relaxed">${escapeHtml(JSON.stringify(sortObjectKeys((result.test.expectedCall?.[0] as FunctionCall)?.arguments) || null, null, 2))}</pre>
+                  <pre class="text-xs text-slate-200 font-mono m-0 leading-relaxed">${escapeHtml(JSON.stringify(sortObjectKeys(expected?.arguments) || null, null, 2))}</pre>
                 </div>
               </td>
               <td class="px-4 py-3">
@@ -417,6 +423,45 @@ function renderStepDetails(stepEval: TestStep, totalSteps: number): string {
                 </span>
               </td>
             </tr>
+            ${
+              hasResultRow
+                ? `
+            <tr class="hover:bg-slate-50/30">
+              <td class="px-4 py-3 font-semibold text-slate-700 whitespace-nowrap align-top">Result</td>
+              <td class="px-4 py-3">
+                ${
+                  hasExpectedResult
+                    ? `<div class="bg-slate-800 rounded-md p-3 overflow-x-auto max-w-md">
+                        <pre class="text-xs text-slate-200 font-mono m-0 leading-relaxed">${escapeHtml(
+                          typeof expected?.result === "object"
+                            ? JSON.stringify(sortObjectKeys(expected.result), null, 2)
+                            : String(expected?.result ?? null),
+                        )}</pre>
+                      </div>`
+                    : `<span class="text-xs text-slate-400 italic">Unconstrained</span>`
+                }
+              </td>
+              <td class="px-4 py-3">
+                ${
+                  hasActualResult
+                    ? `<div class="bg-slate-800 rounded-md p-3 overflow-x-auto max-w-md">
+                        <pre class="text-xs text-slate-200 font-mono m-0 leading-relaxed">${escapeHtml(
+                          typeof result.response?.result === "object"
+                            ? JSON.stringify(sortObjectKeys(result.response.result), null, 2)
+                            : String(result.response?.result ?? null),
+                        )}</pre>
+                      </div>`
+                    : `<span class="text-xs text-slate-400 italic">None</span>`
+                }
+              </td>
+              <td class="px-4 py-3 align-top">
+                <span class="${resultOutcome === "pass" ? "text-emerald-600" : "text-rose-600"} font-bold text-xs mt-2 inline-block">
+                  ${resultOutcome.toUpperCase()}
+                </span>
+              </td>
+            </tr>`
+                : ""
+            }
           </tbody>
         </table>
       </div>