[draft PR] Evals report analyzer

evals-cli/.gitignore

@@ -3,3 +3,4 @@ dist/
 .DS_Store
 .env
 report*.html
+.agents/skills/*

evals-cli/package.json

@@ -39,6 +39,7 @@
     "cli-table3": "^0.6.5",
     "commander": "^13.1.0",
     "dotenv": "^17.2.3",
+    "modern-web-guidance": "^0.0.177",
     "ollama": "^0.6.3",
     "open": "^11.0.0",
     "ora": "^9.3.0",

evals-cli/skills-lock.json

@@ -0,0 +1,11 @@
+{
+  "version": 1,
+  "skills": {
+    "ai-sdk": {
+      "source": "vercel/ai",
+      "sourceType": "github",
+      "skillPath": "skills/use-ai-sdk/SKILL.md",
+      "computedHash": "da25c27a274ef2b71017c255f95a26350d2519765a75cb54ee1a37872b121dab"
+    }
+  }
+}

evals-cli/src/analyzer/index.ts

@@ -0,0 +1,164 @@
+/**
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { readFile } from "fs/promises";
+import { resolve, dirname, join, basename, extname } from "path";
+import { fileURLToPath } from "url";
+import { exec } from "child_process";
+import { promisify } from "util";
+import { generateText } from "ai";
+import { getModel } from "../evaluator/models.js";
+import { Config } from "../types/config.js";
+
+const execAsync = promisify(exec);
+
+export const DEFAULT_MODEL_EVAL_ANALYZER = "google:gemini-3-flash-preview";
+
+/**
+ * Loads the WebMCP best practices and API guides from the installed modern-web-guidance package.
+ */
+async function loadWebMcpContext(): Promise<string> {
+  try {
+    const packageJsonUrl = import.meta.resolve("modern-web-guidance/package.json");
+    const packageDir = dirname(fileURLToPath(packageJsonUrl));
+    const scriptPath = join(packageDir, "skills/modern-web-guidance/modern-web.mjs");
+
+    const { stdout } = await execAsync(
+      `node "${scriptPath}" retrieve webmcp,agentic-forms,agentic-javascript-tools`,
+    );
+    return stdout;
+  } catch (error: any) {
+    throw new Error(`Failed to retrieve modern-web-guidance WebMCP context: ${error.message}`);
+  }
+}
+
+/**
+ * Reads the report JSON from path, with a fallback for HTML report paths.
+ */
+async function readEvalReportJson(reportPath: string): Promise<any> {
+  const fullPath = resolve(process.cwd(), reportPath);
+  const ext = extname(fullPath).toLowerCase();
+
+  let targetJsonPath = fullPath;
+
+  if (ext === ".html") {
+    // Fallback: look for a file with the same timestamp prefix but .json extension
+    const base = basename(fullPath, ".html");
+    const dir = dirname(fullPath);
+    targetJsonPath = join(dir, `${base}.json`);
+  }
+
+  try {
+    const reportRaw = await readFile(targetJsonPath, "utf-8");
+    return JSON.parse(reportRaw);
+  } catch (error: any) {
+    if (ext === ".html") {
+      throw new Error(
+        `No corresponding JSON report found at "${targetJsonPath}". ` +
+          `Please rerun evaluations with JSON reporter enabled (e.g. '--reporter json' or '--reporter console html json').`,
+      );
+    }
+    throw new Error(`Failed to read JSON report from "${targetJsonPath}": ${error.message}`);
+  }
+}
+
+/**
+ * Extracts the title from the corresponding HTML report file if available.
+ */
+async function extractEvalReportTitle(reportPath: string): Promise<string> {
+  const ext = extname(reportPath).toLowerCase();
+  let htmlPath = reportPath;
+
+  if (ext === ".json") {
+    const base = basename(reportPath, ".json");
+    const dir = dirname(reportPath);
+    htmlPath = join(dir, `${base}.html`);
+  }
+
+  try {
+    const htmlContent = await readFile(htmlPath, "utf-8");
+    const titleMatch = htmlContent.match(/<title>(.*?)<\/title>/i);
+    if (titleMatch && titleMatch[1]) {
+      return titleMatch[1].trim();
+    }
+  } catch {
+    // Fallback if HTML file not readable
+  }
+
+  return basename(reportPath, ext);
+}
+
+/**
+ * Main function to execute the report analysis using the configured LLM.
+ */
+export async function analyzeEvalReport(reportPath: string, config: Config): Promise<string> {
+  // Load from the JSON report evals config, assertions (passed/failed), and trajectories.
+  // - reportData.config: evaluation configuration context
+  // - reportData.results.results[].outcome: passed and failed assertions
+  // - reportData.results.results[].trajectory: step-by-step state (available tools), agent action (tool calls inputs), and response (tool outputs)
+  const reportData = await readEvalReportJson(reportPath);
+  const webMcpDomainKnowledge = await loadWebMcpContext();
+  const reportTitle = await extractEvalReportTitle(reportPath);
+
+  // Default to a reasoning model for analysis if not explicitly specified
+  const analyzerConfig = {
+    ...config,
+    model: config.model || DEFAULT_MODEL_EVAL_ANALYZER,
+  };
+
+  const modelInstance = getModel(analyzerConfig);
+
+  const systemPrompt = `You are a WebMCP Evals Analyzer, a specialized developer tool built to analyze agentic browser evaluation reports.
+Your role is to inspect the evaluation outcomes, identify failed steps, and deduce high-quality hypotheses explaining why the model deviated or failed.
+
+Here is the WebMCP Specification and best practices reference for your context:
+=========================================
+${webMcpDomainKnowledge}
+=========================================
+
+Analyze the provided evaluation report JSON. For any failures, assess the trajectory using these three hypotheses:
+1. Model Logic Failure: Did the LLM fail to follow instructions or send invalid parameters?
+2. App/Tool API Failure: Is the application's tool description confusing, or did the tool return broken results?
+3. Test/Assertion Over-Rigidity: Did the model behave correctly/smartly (e.g., self-correcting), but the test assertion was too strict?
+
+CRITICAL BEHAVIOR RULES:
+- **Tone & Terminology**: You MUST maintain a strictly neutral, objective, and professional engineering tone. Avoid conversational adjectives, colloquialisms, or hyperbole. Describe actions and logic using precise technical observations.
+- **Minimal Output on Clean Success**: If the evaluation run was 100% successful (0 failures, 0 errors) and you do NOT suspect any false positives, you MUST keep the report extremely minimal. Briefly state that the run was successful, and do NOT write any detailed trajectory deep-dives or root cause analysis.
+- **Selective Deep Dives**: Only include deep-dives, trajectories, hypotheses, or recommendations if there are actual failures, OR if you believe a passing run is a FALSE POSITIVE (e.g., tests passed because assertions were too lenient/lax).
+
+Format your response in Markdown using the following structure:
+# WebMCP Evals Analysis — Eval report ${reportTitle}
+**Analysis Model / Backend:** ${analyzerConfig.model} (via ${analyzerConfig.backend} backend)
+**Evals Execution Context:** [Describe Target App URL, and Evals execution Model/Backend parsed from the report JSON]
+
+## 1. Summary
+Provide a natural-language paragraph summarizing the evaluation run outcomes.
+
+## 2. Failed Trajectories & Deep Dives
+[Describe failed cases and what went wrong. If the run succeeded with no false positives, state "No failures or false positives detected." and omit details.]
+
+## 3. Root Cause Hypotheses
+[Hypotheses explaining failures or false-positive assertions. Omit if no failures/false positives.]
+
+## 4. Actionable Fixes
+[Concrete checkbox items for fixes. Omit if no failures/false positives.]`;
+
+  // The LLM processes the JSON report to extract eval config, identify failed step assertions,
+  // and analyze trajectories to deduce root causes.
+  const prompt = `Please analyze the following WebMCP evaluation report JSON and provide a detailed markdown report:
+
+\`\`\`json
+${JSON.stringify(reportData, null, 2)}
+\`\`\`
+`;
+
+  const { text } = await generateText({
+    model: modelInstance,
+    system: systemPrompt,
+    prompt: prompt,
+  });
+
+  return text;
+}

evals-cli/src/bin/webmcp-evals.ts

@@ -8,7 +8,7 @@
 import { createRequire } from "node:module";
 import { Command } from "commander";
 import dotenv from "dotenv";
-import { runLocalCommand, runWebCommand } from "../commands/index.js";
+import { runLocalCommand, runWebCommand, runAnalyzeCommand } from "../commands/index.js";
 
 const require = createRequire(import.meta.url);
 const pkg = require("../../package.json");
@@ -51,4 +51,18 @@ program
   .option("--open", "Automatically open the HTML report in browser upon completion", false)
   .action(runWebCommand);
 
+// Command: analyze evaluation report using an LLM
+program
+  .command("analyze")
+  .description(
+    "Analyze an evaluation JSON report using an LLM to identify root causes and hypotheses for eval failures",
+  )
+  .argument("<report-path>", "Path to the JSON report file (e.g. .evals/report-*.json)")
+  .option(
+    "-m, --model <model>",
+    "Model identifier for the analyzer (defaults to google:gemini-3-pro-preview)",
+  )
+  .option("--open", "Automatically open the analysis markdown report upon completion", false)
+  .action(runAnalyzeCommand);
+
 program.parse(process.argv);

evals-cli/src/commands/index.ts

@@ -5,7 +5,7 @@
 
 import { Command } from "commander";
 import { readFile, writeFile, mkdir } from "fs/promises";
-import { resolve } from "path";
+import { resolve, basename, extname } from "path";
 import { SingleBar } from "cli-progress";
 import chalk from "chalk";
 import Table from "cli-table3";
@@ -17,6 +17,7 @@ import { Tool, ToolsSchema } from "../types/tools.js";
 import { executeLocalEvals, executeInBrowserEvals } from "../evaluator/index.js";
 import { renderReport, renderWebmcpReport } from "../report/report.js";
 import { createBackend } from "../backends/index.js";
+import { analyzeEvalReport, DEFAULT_MODEL_EVAL_ANALYZER } from "../analyzer/index.js";
 
 export interface CommandOptions {
   backend: string;
@@ -231,3 +232,60 @@ async function outputReports(
     await open(htmlPath);
   }
 }
+
+export async function runAnalyzeCommand(
+  reportPath: string,
+  options: CommandOptions,
+  command?: Command,
+): Promise<void> {
+  const localOpts = command?.opts() || {};
+  const globalOpts = command?.optsWithGlobals ? command.optsWithGlobals() : options;
+
+  const config: Config = {
+    toolSchemaFile: "",
+    evalsFile: "",
+    backend: localOpts.backend || globalOpts.backend || "vercel",
+    model: localOpts.model || DEFAULT_MODEL_EVAL_ANALYZER,
+    runs: globalOpts.runs,
+    outputDir: globalOpts.outputDir,
+    reporter: globalOpts.reporter,
+  };
+
+  const spinner = ora({ discardStdin: false });
+  spinner.start("Analyzing evals report...");
+
+  try {
+    const analysisText = await analyzeEvalReport(reportPath, config);
+    spinner.stop();
+
+    const timestamp = Date.now();
+    const outputDir = globalOpts.outputDir || ".evals";
+    await mkdir(resolve(process.cwd(), outputDir), { recursive: true });
+
+    // Determine output filename matching the input report filename
+    const base = basename(reportPath, extname(reportPath));
+    const outputPath = resolve(process.cwd(), outputDir, `analysis-${base}-${timestamp}.md`);
+
+    await writeFile(outputPath, analysisText, "utf-8");
+
+    console.log(`\n${chalk.green.bold("📝 Analysis Report Completed:")}`);
+    console.log(`Saved to: ${outputPath}\n`);
+
+    if (localOpts.open) {
+      try {
+        await open(outputPath, { app: { name: "google chrome" } });
+      } catch {
+        await open(outputPath);
+      }
+    }
+
+    // Log a brief snippet of the summary to console
+    const summaryLines = analysisText.split("\n").slice(0, 15).join("\n");
+    console.log(summaryLines);
+    console.log("\n...\n");
+  } catch (error: any) {
+    spinner.stop();
+    console.error(`\n${chalk.red.bold("❌ Error:")} ${error.message || error}\n`);
+    process.exit(1);
+  }
+}