From bb577e086a13668c096be2cdbd037982172673ee Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Thu, 7 May 2026 10:32:21 +0300 Subject: [PATCH 1/5] feat: add agentic eval harness --- eval/agentic/README.md | 86 +++ eval/agentic/result.schema.json | 68 +++ eval/agentic/workloads/express-router.md | 19 + package.json | 1 + scripts/agent-eval.test.ts | 131 +++++ scripts/agent-eval.ts | 705 +++++++++++++++++++++++ 6 files changed, 1010 insertions(+) create mode 100644 eval/agentic/README.md create mode 100644 eval/agentic/result.schema.json create mode 100644 eval/agentic/workloads/express-router.md create mode 100644 scripts/agent-eval.test.ts create mode 100644 scripts/agent-eval.ts diff --git a/eval/agentic/README.md b/eval/agentic/README.md new file mode 100644 index 00000000..20b747dc --- /dev/null +++ b/eval/agentic/README.md @@ -0,0 +1,86 @@ +# Agentic Eval Harness + +This harness runs real coding agents against the real GitHits MCP server and +records whether the agent can use GitHits tools effectively from the MCP +server's own instructions and tool descriptions. + +It is not a smoke test. Smoke tests exercise CLI and MCP contracts directly. +Agentic evals exercise agent behavior end-to-end. + +## What Is Under Test + +- Local mode starts the MCP server from this checkout with + `bun run --cwd dev mcp start`. +- Published mode starts the MCP server with `npx -y githits@latest mcp start` + by default. +- To evaluate MCP instruction changes, change branch/source and run local mode. + +The harness must not add GitHits usage guidance through agent system prompts, +append prompts, alternate MCP instruction files, project instructions, or plugin +commands. Workload prompts may ask the agent to report what happened, but must +not tell the agent how to use GitHits. + +## Isolation + +Runs execute agents from an empty temporary workspace so repository-local files +such as `AGENTS.md`, `.mcp.json`, commands, skills, and plugin payloads do not +contaminate results. The harness keeps the user's normal Claude/GitHits auth +environment so human-driven keychain/OAuth sessions continue to work. + +GitHits authentication follows normal local behavior. Keychain-backed human +login should work by default. Automation can use `GITHITS_API_TOKEN`. + +## Usage + +```bash +bun run agent:e2e --server local --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --server published --workload eval/agentic/workloads/express-router.md +``` + +Useful options: + +```bash +--dry-run Generate artifacts without invoking Claude +--out Output directory, default `.agent-eval/runs/` +--timeout Per-workload timeout, default 300 +--published-package Published package spec, default `githits@latest` +--workload Repeatable workload path +``` + +Normal GitHits backend overrides are passed through when set: + +- `GITHITS_API_URL` +- `GITHITS_MCP_URL` +- `GITHITS_CODE_NAV_URL` +- `PKGSEER_URL` +- `GITHITS_API_TOKEN` +- `GITHITS_AUTH_STORAGE` + +Secret-like values are redacted in run metadata. + +## Workloads + +Workloads are Markdown prompts. They should contain: + +- The task. +- A reporting contract requiring the final answer, GitHits tools used, failed or + unclear tool calls, unclear or missing MCP guidance, usefulness, and + confidence. + +They should not contain instructions such as "call `search` first" or "use +`code_read` after `search`". That guidance must come from the MCP server. + +## Artifacts + +Each run writes: + +- `run.json` with command, git, environment, and timing metadata. +- One workload directory per workload with `prompt.md`, `mcp.json`, + `stdout.json`, `stderr.txt`, and `final.json` when parsing succeeds. + +Claude is launched with `--permission-mode bypassPermissions` so non-interactive +evals can exercise configured MCP tools without a human approval prompt. + +Malformed final JSON, schema mismatches, Claude failures, and timeouts are +harness failures. Raw stdout and stderr are preserved for diagnosis with known +secret values redacted. diff --git a/eval/agentic/result.schema.json b/eval/agentic/result.schema.json new file mode 100644 index 00000000..59630890 --- /dev/null +++ b/eval/agentic/result.schema.json @@ -0,0 +1,68 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "answer", + "githitsToolsUsed", + "toolIssues", + "instructionIssues", + "githitsUsefulness", + "githitsUsefulnessReason", + "confidence" + ], + "properties": { + "status": { + "type": "string", + "enum": ["success", "failure", "inconclusive"] + }, + "answer": { + "type": "string" + }, + "githitsToolsUsed": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["tool", "purpose"], + "properties": { + "tool": { "type": "string" }, + "purpose": { "type": "string" } + } + } + }, + "toolIssues": { + "type": "array", + "items": { + "oneOf": [ + { "type": "string" }, + { + "type": "object", + "additionalProperties": false, + "required": ["tool", "issue"], + "properties": { + "tool": { "type": "string" }, + "issue": { "type": "string" } + } + } + ] + } + }, + "instructionIssues": { + "type": "array", + "items": { "type": "string" } + }, + "githitsUsefulness": { + "type": "string", + "enum": ["helped", "hurt", "unused", "unclear"] + }, + "githitsUsefulnessReason": { + "type": "string" + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + } + } +} diff --git a/eval/agentic/workloads/express-router.md b/eval/agentic/workloads/express-router.md new file mode 100644 index 00000000..3a611668 --- /dev/null +++ b/eval/agentic/workloads/express-router.md @@ -0,0 +1,19 @@ +# Workload: Express Router Investigation + +You are investigating how Express implements routing. Determine where the main +Router implementation lives in the Express package and summarize the evidence +you found. + +At the end, return valid JSON matching the provided schema. Include: + +- `status`: `success`, `failure`, or `inconclusive`. +- `answer`: your final answer with evidence. +- `githitsToolsUsed`: GitHits MCP tools you used, if any, with the purpose of + each call. +- `toolIssues`: GitHits tool calls that failed or returned unclear output, if + any. +- `instructionIssues`: MCP guidance that was unclear, missing, or contradicted + observed tool behavior, if any. +- `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. +- `githitsUsefulnessReason`: why you chose that usefulness value. +- `confidence`: `high`, `medium`, or `low`. diff --git a/package.json b/package.json index f5d66c9e..5043d70d 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "inspector": "npx @modelcontextprotocol/inspector bun run dev mcp", "smoke:cli": "bun run scripts/cli-smoke.ts", "smoke:mcp": "bun run scripts/mcp-smoke.ts", + "agent:e2e": "bun run scripts/agent-eval.ts", "test": "bun test", "typecheck": "tsc", "format": "biome format --write .", diff --git a/scripts/agent-eval.test.ts b/scripts/agent-eval.test.ts new file mode 100644 index 00000000..7b8e6f78 --- /dev/null +++ b/scripts/agent-eval.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "bun:test"; +import { + buildEvalEnv, + buildMcpConfig, + collectSecretValues, + isValidAgentReport, + parseArgs, + redactText, + sanitizedEnvSummary, +} from "./agent-eval.ts"; + +describe("agent eval harness", () => { + it("builds local MCP config with explicit repo cwd", () => { + const config = buildMcpConfig({ + server: "local", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@latest", + }); + + expect(config.mcpServers.githits).toEqual({ + command: "bun", + args: ["run", "--cwd", "/repo/githits-cli", "dev", "mcp", "start"], + }); + }); + + it("builds published MCP config with pinned package spec", () => { + const config = buildMcpConfig({ + server: "published", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@0.4.2", + }); + + expect(config.mcpServers.githits).toEqual({ + command: "npx", + args: ["-y", "githits@0.4.2", "mcp", "start"], + }); + }); + + it("preserves normal Claude and GitHits auth environment while filtering unrelated vars", () => { + const env = buildEvalEnv({ + PATH: "/bin", + HOME: "/real-home", + RANDOM_SECRET: "should-not-pass", + GITHITS_AUTH_STORAGE: "keychain", + GITHITS_API_TOKEN: "secret-token", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + }); + + expect(env.HOME).toBe("/real-home"); + expect(env.GITHITS_AUTH_STORAGE).toBe("keychain"); + expect(env.GITHITS_API_TOKEN).toBe("secret-token"); + expect(env.GITHITS_CODE_NAV_URL).toBe("http://localhost:7070"); + expect(env.RANDOM_SECRET).toBeUndefined(); + }); + + it("redacts secret values from environment summary", () => { + const summary = sanitizedEnvSummary({ + HOME: "/tmp/eval-home", + USERPROFILE: "/tmp/eval-home", + XDG_CONFIG_HOME: "/tmp/eval-home/.config", + APPDATA: "/tmp/eval-home/AppData/Roaming", + GITHITS_API_TOKEN: "secret-token", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + GITHITS_AUTH_STORAGE: "keychain", + }); + + expect(summary.GITHITS_API_TOKEN).toBe(""); + expect(summary.GITHITS_CODE_NAV_URL).toBe("http://localhost:7070"); + expect(summary.GITHITS_AUTH_STORAGE).toBe("keychain"); + expect(summary.HOME).toBe(""); + }); + + it("parses repeatable workloads and dry-run options", () => { + const options = parseArgs( + [ + "--server", + "published", + "--published-package", + "githits@0.4.2", + "--workload", + "eval/agentic/workloads/express-router.md", + "--timeout", + "12", + "--dry-run", + ], + "/repo/githits-cli", + ); + + expect(options.server).toBe("published"); + expect(options.publishedPackage).toBe("githits@0.4.2"); + expect(options.timeoutSeconds).toBe(12); + expect(options.dryRun).toBe(true); + expect(options.workloads).toEqual([ + "/repo/githits-cli/eval/agentic/workloads/express-router.md", + ]); + }); + + it("redacts secret values from persisted text", () => { + const secrets = collectSecretValues({ + GITHITS_API_TOKEN: "secret-token-value", + ANTHROPIC_API_KEY: "anthropic-secret-value", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + }); + + expect( + redactText( + "token=secret-token-value key=anthropic-secret-value", + secrets, + ), + ).toBe("token= key="); + }); + + it("validates final agent report shape", () => { + expect( + isValidAgentReport({ + status: "success", + answer: "Router lives in lib/router/index.js.", + githitsToolsUsed: [{ tool: "search", purpose: "find router" }], + toolIssues: [], + instructionIssues: [], + githitsUsefulness: "helped", + githitsUsefulnessReason: "It located source evidence.", + confidence: "high", + }), + ).toBe(true); + + expect( + isValidAgentReport({ status: "success", answer: "missing fields" }), + ).toBe(false); + }); +}); diff --git a/scripts/agent-eval.ts b/scripts/agent-eval.ts new file mode 100644 index 00000000..641048c7 --- /dev/null +++ b/scripts/agent-eval.ts @@ -0,0 +1,705 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; + +type AgentName = "claude"; +type ServerMode = "local" | "published"; +type RunStatus = "dry-run" | "success" | "failed" | "timeout"; + +export interface AgentEvalOptions { + agent: AgentName; + server: ServerMode; + workloads: string[]; + outDir: string; + timeoutSeconds: number; + publishedPackage: string; + dryRun: boolean; + repoRoot: string; + schemaPath: string; +} + +export interface McpServerConfig { + mcpServers: { + githits: { + command: string; + args: string[]; + env?: Record; + }; + }; +} + +interface WorkloadRunMetadata { + id: string; + path: string; + status: RunStatus; + exitCode?: number; + durationMs?: number; + timedOut?: boolean; + command: string[]; + workspaceDir: string; + workloadDir: string; +} + +const PASSTHROUGH_ENV_KEYS = [ + "GITHITS_API_URL", + "GITHITS_MCP_URL", + "GITHITS_CODE_NAV_URL", + "PKGSEER_URL", + "GITHITS_API_TOKEN", + "GITHITS_AUTH_STORAGE", +] as const; + +const BASE_ENV_KEYS = [ + "PATH", + "HOME", + "USERPROFILE", + "XDG_CONFIG_HOME", + "APPDATA", + "USER", + "LOGNAME", + "USERNAME", + "SystemRoot", + "WINDIR", + "COMSPEC", + "TMPDIR", + "TMP", + "TEMP", + "SHELL", + "LANG", + "LC_ALL", + "SECURITYSESSIONID", + "XPC_FLAGS", + "XPC_SERVICE_NAME", + "__CF_USER_TEXT_ENCODING", + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", +] as const; + +const SECRET_PATTERN = /(TOKEN|API_KEY|SECRET|PASSWORD|CREDENTIAL)/i; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function timestamp(): string { + return new Date().toISOString().replace(/[:.]/g, "-"); +} + +function defaultOutDir(repoRoot: string): string { + return join(repoRoot, ".agent-eval", "runs", timestamp()); +} + +function parsePositiveInteger(value: string, flag: string): number { + const parsed = Number(value); + assert( + Number.isInteger(parsed) && parsed > 0, + `${flag} must be a positive integer`, + ); + return parsed; +} + +export function parseArgs( + argv: string[], + repoRoot = process.cwd(), +): AgentEvalOptions { + const options: AgentEvalOptions = { + agent: "claude", + server: "local", + workloads: [], + outDir: defaultOutDir(repoRoot), + timeoutSeconds: 300, + publishedPackage: "githits@latest", + dryRun: false, + repoRoot, + schemaPath: join(repoRoot, "eval", "agentic", "result.schema.json"), + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case "--agent": { + const value = argv[++i]; + assert(value === "claude", "--agent currently supports only claude"); + options.agent = value; + break; + } + case "--server": { + const value = argv[++i]; + assert( + value === "local" || value === "published", + "--server must be local or published", + ); + options.server = value; + break; + } + case "--workload": { + const value = argv[++i]; + assert(value, "--workload requires a path"); + options.workloads.push(value); + break; + } + case "--out": { + const value = argv[++i]; + assert(value, "--out requires a path"); + options.outDir = resolve(repoRoot, value); + break; + } + case "--timeout": { + const value = argv[++i]; + assert(value, "--timeout requires seconds"); + options.timeoutSeconds = parsePositiveInteger(value, "--timeout"); + break; + } + case "--published-package": { + const value = argv[++i]; + assert(value, "--published-package requires a package spec"); + options.publishedPackage = value; + break; + } + case "--schema": { + const value = argv[++i]; + assert(value, "--schema requires a path"); + options.schemaPath = resolve(repoRoot, value); + break; + } + case "--dry-run": + options.dryRun = true; + break; + case "--help": + case "-h": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (options.workloads.length === 0) { + options.workloads.push( + join(repoRoot, "eval", "agentic", "workloads", "express-router.md"), + ); + } + options.workloads = options.workloads.map((path) => resolve(repoRoot, path)); + return options; +} + +function printHelp(): void { + console.log(`Usage: bun run agent:e2e [options] + +Options: + --agent claude Agent to run (default: claude) + --server local|published MCP server mode (default: local) + --workload Workload markdown path, repeatable + --out Output directory + --timeout Per-workload timeout (default: 300) + --published-package Package for published mode (default: githits@latest) + --schema Result JSON schema path + --dry-run Generate artifacts without invoking Claude +`); +} + +export function buildMcpConfig( + options: Pick, +): McpServerConfig { + if (options.server === "local") { + return { + mcpServers: { + githits: { + command: "bun", + args: ["run", "--cwd", options.repoRoot, "dev", "mcp", "start"], + }, + }, + }; + } + + return { + mcpServers: { + githits: { + command: "npx", + args: ["-y", options.publishedPackage, "mcp", "start"], + }, + }, + }; +} + +export function buildEvalEnv( + baseEnv: NodeJS.ProcessEnv, +): Record { + const env: Record = {}; + + for (const key of [...BASE_ENV_KEYS, ...PASSTHROUGH_ENV_KEYS]) { + const value = baseEnv[key]; + if (value !== undefined) { + env[key] = value; + } + } + + env.NO_COLOR = "1"; + return env; +} + +export function collectSecretValues(env: Record): string[] { + const values = new Set(); + for (const [key, value] of Object.entries(env)) { + if (SECRET_PATTERN.test(key) && value.length >= 8) { + values.add(value); + } + } + return [...values].sort((a, b) => b.length - a.length); +} + +export function redactText(text: string, secretValues: string[]): string { + let redacted = text; + for (const secret of secretValues) { + redacted = redacted.split(secret).join(""); + } + return redacted; +} + +export function sanitizedEnvSummary( + env: Record, +): Record { + const summary: Record = {}; + for (const key of PASSTHROUGH_ENV_KEYS) { + const value = env[key]; + if (value === undefined) { + continue; + } + summary[key] = SECRET_PATTERN.test(key) ? "" : value; + } + if (env.HOME) summary.HOME = ""; + if (env.USERPROFILE) summary.USERPROFILE = ""; + if (env.XDG_CONFIG_HOME) summary.XDG_CONFIG_HOME = ""; + if (env.APPDATA) summary.APPDATA = ""; + return summary; +} + +function workloadId(workloadPath: string): string { + return basename(workloadPath).replace(/\.[^.]+$/, ""); +} + +function writeJson(path: string, value: unknown): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function redactValue(value: unknown, secretValues: string[]): unknown { + if (typeof value === "string") { + return redactText(value, secretValues); + } + if (Array.isArray(value)) { + return value.map((item) => redactValue(item, secretValues)); + } + if (value !== null && typeof value === "object") { + const redacted: Record = {}; + for (const [key, item] of Object.entries(value)) { + redacted[key] = redactValue(item, secretValues); + } + return redacted; + } + return value; +} + +async function commandOutput( + command: string, + args: string[], + cwd: string, +): Promise { + try { + const proc = Bun.spawn([command, ...args], { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + proc.exited, + ]); + if (exitCode !== 0) { + return undefined; + } + return stdout.trim() || undefined; + } catch { + return undefined; + } +} + +async function collectGitMetadata( + repoRoot: string, +): Promise> { + const [branch, sha] = await Promise.all([ + commandOutput("git", ["branch", "--show-current"], repoRoot), + commandOutput("git", ["rev-parse", "HEAD"], repoRoot), + ]); + return { branch, sha }; +} + +async function claudeVersion(): Promise { + return commandOutput("claude", ["--version"], process.cwd()); +} + +async function assertClaudeAvailable(): Promise { + const version = await claudeVersion(); + assert( + version, + "claude CLI not found or not executable. Install Claude Code before running live agent evals.", + ); +} + +function extractFinalJson(stdout: string): unknown | undefined { + const lines = stdout.split("\n").filter((line) => line.trim().length > 0); + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + if (!line) continue; + try { + const event = JSON.parse(line) as Record; + const candidates = [ + event.result, + event.message, + event.content, + event.text, + ]; + for (const candidate of candidates) { + if (typeof candidate === "string") { + const parsed = parseJsonFromText(candidate); + if (parsed !== undefined) return parsed; + } + } + if (event.status || event.answer || event.githitsToolsUsed) { + return event; + } + } catch { + try { + return JSON.parse(line); + } catch { + // Continue scanning older lines. + } + } + } + return undefined; +} + +function parseJsonFromText(text: string): unknown | undefined { + const trimmed = text.trim(); + try { + return JSON.parse(trimmed); + } catch { + // Continue to fenced JSON extraction. + } + + const fence = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (!fence?.[1]) { + return undefined; + } + try { + return JSON.parse(fence[1].trim()); + } catch { + return undefined; + } +} + +function isStringArray(value: unknown): value is string[] { + return ( + Array.isArray(value) && value.every((item) => typeof item === "string") + ); +} + +function isToolUseArray(value: unknown): boolean { + return ( + Array.isArray(value) && + value.every( + (item) => + item !== null && + typeof item === "object" && + typeof (item as Record).tool === "string" && + typeof (item as Record).purpose === "string", + ) + ); +} + +function isToolIssueArray(value: unknown): boolean { + return ( + Array.isArray(value) && + value.every( + (item) => + typeof item === "string" || + (item !== null && + typeof item === "object" && + typeof (item as Record).tool === "string" && + typeof (item as Record).issue === "string"), + ) + ); +} + +export function isValidAgentReport(value: unknown): boolean { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + const report = value as Record; + const allowedKeys = new Set([ + "status", + "answer", + "githitsToolsUsed", + "toolIssues", + "instructionIssues", + "githitsUsefulness", + "githitsUsefulnessReason", + "confidence", + ]); + if (Object.keys(report).some((key) => !allowedKeys.has(key))) { + return false; + } + return ( + (report.status === "success" || + report.status === "failure" || + report.status === "inconclusive") && + typeof report.answer === "string" && + isToolUseArray(report.githitsToolsUsed) && + isToolIssueArray(report.toolIssues) && + isStringArray(report.instructionIssues) && + (report.githitsUsefulness === "helped" || + report.githitsUsefulness === "hurt" || + report.githitsUsefulness === "unused" || + report.githitsUsefulness === "unclear") && + typeof report.githitsUsefulnessReason === "string" && + (report.confidence === "high" || + report.confidence === "medium" || + report.confidence === "low") + ); +} + +async function runWithTimeout( + command: string[], + cwd: string, + env: Record, + timeoutSeconds: number, +): Promise<{ + stdout: string; + stderr: string; + exitCode?: number; + timedOut: boolean; +}> { + const proc = Bun.spawn(command, { + cwd, + env, + stdout: "pipe", + stderr: "pipe", + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill("SIGTERM"); + setTimeout(() => proc.kill("SIGKILL"), 2_000).unref?.(); + }, timeoutSeconds * 1_000); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited.catch(() => undefined), + ]); + clearTimeout(timer); + return { stdout, stderr, exitCode, timedOut }; +} + +function buildClaudeCommand(prompt: string, mcpConfigPath: string): string[] { + return [ + "claude", + "-p", + prompt, + "--mcp-config", + mcpConfigPath, + "--strict-mcp-config", + "--output-format", + "json", + "--permission-mode", + "bypassPermissions", + "--no-session-persistence", + ]; +} + +async function runWorkload( + options: AgentEvalOptions, + workloadPath: string, + runDir: string, + env: Record, + mcpConfig: McpServerConfig, + secretValues: string[], +): Promise { + assert(existsSync(workloadPath), `Workload not found: ${workloadPath}`); + const id = workloadId(workloadPath); + const workloadDir = join(runDir, "workloads", id); + const workspaceDir = mkdtempSync( + join(tmpdir(), "githits-agent-eval-workspace-"), + ); + mkdirSync(workloadDir, { recursive: true }); + + const prompt = readFileSync(workloadPath, "utf8"); + const mcpConfigPath = join(workloadDir, "mcp.json"); + writeFileSync(join(workloadDir, "prompt.md"), prompt); + writeJson(mcpConfigPath, mcpConfig); + + const command = buildClaudeCommand(prompt, mcpConfigPath); + const metadataBase = { + id, + path: workloadPath, + command, + workspaceDir, + workloadDir, + }; + + try { + if (options.dryRun) { + writeJson(join(workloadDir, "dry-run.json"), metadataBase); + return { ...metadataBase, status: "dry-run" }; + } + + const started = Date.now(); + const result = await runWithTimeout( + command, + workspaceDir, + env, + options.timeoutSeconds, + ); + const durationMs = Date.now() - started; + + writeFileSync( + join(workloadDir, "stdout.json"), + redactText(result.stdout, secretValues), + ); + writeFileSync( + join(workloadDir, "stderr.txt"), + redactText(result.stderr, secretValues), + ); + + const finalJson = extractFinalJson(result.stdout); + const validFinalJson = isValidAgentReport(finalJson); + if (validFinalJson) { + writeJson( + join(workloadDir, "final.json"), + redactValue(finalJson, secretValues), + ); + } else if (finalJson !== undefined) { + writeJson( + join(workloadDir, "invalid-final.json"), + redactValue(finalJson, secretValues), + ); + } + + const status: RunStatus = result.timedOut + ? "timeout" + : result.exitCode === 0 && validFinalJson + ? "success" + : "failed"; + + return { + ...metadataBase, + status, + exitCode: result.exitCode, + durationMs, + timedOut: result.timedOut, + }; + } finally { + rmSync(workspaceDir, { recursive: true, force: true }); + } +} + +export async function runAgentEval(options: AgentEvalOptions): Promise { + assert( + existsSync(options.schemaPath), + `Schema not found: ${options.schemaPath}`, + ); + mkdirSync(options.outDir, { recursive: true }); + const env = buildEvalEnv(process.env); + const secretValues = collectSecretValues(env); + const mcpConfig = buildMcpConfig(options); + + if (!options.dryRun) { + await assertClaudeAvailable(); + } + + const [git, claude] = await Promise.all([ + collectGitMetadata(options.repoRoot), + claudeVersion(), + ]); + + const workloadResults: WorkloadRunMetadata[] = []; + for (const workload of options.workloads) { + workloadResults.push( + await runWorkload( + options, + workload, + options.outDir, + env, + mcpConfig, + secretValues, + ), + ); + } + + writeJson(join(options.outDir, "run.json"), { + agent: options.agent, + server: options.server, + publishedPackage: options.publishedPackage, + dryRun: options.dryRun, + timeoutSeconds: options.timeoutSeconds, + repoRoot: options.repoRoot, + schemaPath: options.schemaPath, + git, + claudeVersion: claude, + env: sanitizedEnvSummary(env), + workloads: workloadResults, + }); + + writeJson(join(options.outDir, "summary.json"), { + status: workloadResults.some( + (result) => result.status === "failed" || result.status === "timeout", + ) + ? "failed" + : options.dryRun + ? "dry-run" + : "success", + workloads: workloadResults.map( + ({ id, status, exitCode, durationMs, timedOut }) => ({ + id, + status, + exitCode, + durationMs, + timedOut, + }), + ), + }); +} + +async function main(): Promise { + const repoRoot = process.cwd(); + const options = parseArgs(process.argv.slice(2), repoRoot); + if (!isAbsolute(options.outDir)) { + options.outDir = resolve(repoRoot, options.outDir); + } + await runAgentEval(options); + console.log(`Agent eval artifacts written to ${options.outDir}`); +} + +if (import.meta.main) { + main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exit(1); + }); +} From 9ae2409ebf6c3afd67b28f352cd90e70694dea40 Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Thu, 7 May 2026 11:06:56 +0300 Subject: [PATCH 2/5] feat: add codex agent eval support --- .gitignore | 1 + eval/agentic/README.md | 9 +- eval/agentic/result.schema.json | 13 +- eval/agentic/workloads/express-router.md | 4 +- scripts/agent-eval.test.ts | 32 +++++ scripts/agent-eval.ts | 149 +++++++++++++++++++---- 6 files changed, 168 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 9288a86f..ccbe837a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ coverage # Agents .claude +.agent-eval/ # logs logs diff --git a/eval/agentic/README.md b/eval/agentic/README.md index 20b747dc..c52cc754 100644 --- a/eval/agentic/README.md +++ b/eval/agentic/README.md @@ -35,12 +35,14 @@ login should work by default. Automation can use `GITHITS_API_TOKEN`. ```bash bun run agent:e2e --server local --workload eval/agentic/workloads/express-router.md bun run agent:e2e --server published --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --agent codex --server local --workload eval/agentic/workloads/express-router.md ``` Useful options: ```bash ---dry-run Generate artifacts without invoking Claude +--agent Agent to run, default `claude` +--dry-run Generate artifacts without invoking the agent --out Output directory, default `.agent-eval/runs/` --timeout Per-workload timeout, default 300 --published-package Published package spec, default `githits@latest` @@ -79,7 +81,10 @@ Each run writes: `stdout.json`, `stderr.txt`, and `final.json` when parsing succeeds. Claude is launched with `--permission-mode bypassPermissions` so non-interactive -evals can exercise configured MCP tools without a human approval prompt. +evals can exercise configured MCP tools without a human approval prompt. Codex is +launched with per-run `-c` MCP config overrides, `--ignore-rules`, and a +read-only sandbox so it can use normal human auth without mutating global MCP +configuration. Malformed final JSON, schema mismatches, Claude failures, and timeouts are harness failures. Raw stdout and stderr are preserved for diagnosis with known diff --git a/eval/agentic/result.schema.json b/eval/agentic/result.schema.json index 59630890..d4945f6c 100644 --- a/eval/agentic/result.schema.json +++ b/eval/agentic/result.schema.json @@ -35,18 +35,7 @@ "toolIssues": { "type": "array", "items": { - "oneOf": [ - { "type": "string" }, - { - "type": "object", - "additionalProperties": false, - "required": ["tool", "issue"], - "properties": { - "tool": { "type": "string" }, - "issue": { "type": "string" } - } - } - ] + "type": "string" } }, "instructionIssues": { diff --git a/eval/agentic/workloads/express-router.md b/eval/agentic/workloads/express-router.md index 3a611668..e3d2572f 100644 --- a/eval/agentic/workloads/express-router.md +++ b/eval/agentic/workloads/express-router.md @@ -10,8 +10,8 @@ At the end, return valid JSON matching the provided schema. Include: - `answer`: your final answer with evidence. - `githitsToolsUsed`: GitHits MCP tools you used, if any, with the purpose of each call. -- `toolIssues`: GitHits tool calls that failed or returned unclear output, if - any. +- `toolIssues`: string descriptions of GitHits tool calls that failed or + returned unclear output, if any. - `instructionIssues`: MCP guidance that was unclear, missing, or contradicted observed tool behavior, if any. - `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. diff --git a/scripts/agent-eval.test.ts b/scripts/agent-eval.test.ts index 7b8e6f78..711bd15f 100644 --- a/scripts/agent-eval.test.ts +++ b/scripts/agent-eval.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "bun:test"; import { + buildCodexConfig, + buildCodexConfigArgs, buildEvalEnv, buildMcpConfig, collectSecretValues, @@ -36,6 +38,33 @@ describe("agent eval harness", () => { }); }); + it("builds Codex TOML config from the same MCP command", () => { + expect( + buildCodexConfig({ + server: "local", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@latest", + }), + ).toBe( + '[mcp_servers.githits]\ncommand = "bun"\nargs = ["run","--cwd","/repo/githits-cli","dev","mcp","start"]\n', + ); + }); + + it("builds Codex config override args", () => { + expect( + buildCodexConfigArgs({ + server: "published", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@0.4.2", + }), + ).toEqual([ + "-c", + 'mcp_servers.githits.command="npx"', + "-c", + 'mcp_servers.githits.args=["-y","githits@0.4.2","mcp","start"]', + ]); + }); + it("preserves normal Claude and GitHits auth environment while filtering unrelated vars", () => { const env = buildEvalEnv({ PATH: "/bin", @@ -73,6 +102,8 @@ describe("agent eval harness", () => { it("parses repeatable workloads and dry-run options", () => { const options = parseArgs( [ + "--agent", + "codex", "--server", "published", "--published-package", @@ -86,6 +117,7 @@ describe("agent eval harness", () => { "/repo/githits-cli", ); + expect(options.agent).toBe("codex"); expect(options.server).toBe("published"); expect(options.publishedPackage).toBe("githits@0.4.2"); expect(options.timeoutSeconds).toBe(12); diff --git a/scripts/agent-eval.ts b/scripts/agent-eval.ts index 641048c7..6655de38 100644 --- a/scripts/agent-eval.ts +++ b/scripts/agent-eval.ts @@ -9,7 +9,7 @@ import { import { tmpdir } from "node:os"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; -type AgentName = "claude"; +type AgentName = "claude" | "codex"; type ServerMode = "local" | "published"; type RunStatus = "dry-run" | "success" | "failed" | "timeout"; @@ -134,7 +134,10 @@ export function parseArgs( switch (arg) { case "--agent": { const value = argv[++i]; - assert(value === "claude", "--agent currently supports only claude"); + assert( + value === "claude" || value === "codex", + "--agent must be claude or codex", + ); options.agent = value; break; } @@ -203,7 +206,7 @@ function printHelp(): void { console.log(`Usage: bun run agent:e2e [options] Options: - --agent claude Agent to run (default: claude) + --agent claude|codex Agent to run (default: claude) --server local|published MCP server mode (default: local) --workload Workload markdown path, repeatable --out Output directory @@ -217,27 +220,54 @@ Options: export function buildMcpConfig( options: Pick, ): McpServerConfig { + const command = buildMcpCommand(options); + return { + mcpServers: { + githits: command, + }, + }; +} + +function buildMcpCommand( + options: Pick, +): McpServerConfig["mcpServers"]["githits"] { if (options.server === "local") { return { - mcpServers: { - githits: { - command: "bun", - args: ["run", "--cwd", options.repoRoot, "dev", "mcp", "start"], - }, - }, + command: "bun", + args: ["run", "--cwd", options.repoRoot, "dev", "mcp", "start"], }; } return { - mcpServers: { - githits: { - command: "npx", - args: ["-y", options.publishedPackage, "mcp", "start"], - }, - }, + command: "npx", + args: ["-y", options.publishedPackage, "mcp", "start"], }; } +export function buildCodexConfig( + options: Pick, +): string { + const command = buildMcpCommand(options); + return [ + "[mcp_servers.githits]", + `command = ${JSON.stringify(command.command)}`, + `args = ${JSON.stringify(command.args)}`, + "", + ].join("\n"); +} + +export function buildCodexConfigArgs( + options: Pick, +): string[] { + const command = buildMcpCommand(options); + return [ + "-c", + `mcp_servers.githits.command=${JSON.stringify(command.command)}`, + "-c", + `mcp_servers.githits.args=${JSON.stringify(command.args)}`, + ]; +} + export function buildEvalEnv( baseEnv: NodeJS.ProcessEnv, ): Record { @@ -354,6 +384,10 @@ async function claudeVersion(): Promise { return commandOutput("claude", ["--version"], process.cwd()); } +async function codexVersion(): Promise { + return commandOutput("codex", ["--version"], process.cwd()); +} + async function assertClaudeAvailable(): Promise { const version = await claudeVersion(); assert( @@ -362,6 +396,14 @@ async function assertClaudeAvailable(): Promise { ); } +async function assertCodexAvailable(): Promise { + const version = await codexVersion(); + assert( + version, + "codex CLI not found or not executable. Install Codex before running live agent evals.", + ); +} + function extractFinalJson(stdout: string): unknown | undefined { const lines = stdout.split("\n").filter((line) => line.trim().length > 0); for (let i = lines.length - 1; i >= 0; i -= 1) { @@ -533,6 +575,33 @@ function buildClaudeCommand(prompt: string, mcpConfigPath: string): string[] { ]; } +function buildCodexCommand( + prompt: string, + workspaceDir: string, + finalMessagePath: string, + schemaPath: string, + options: Pick, +): string[] { + return [ + "codex", + "exec", + ...buildCodexConfigArgs(options), + "--cd", + workspaceDir, + "--skip-git-repo-check", + "--ephemeral", + "--json", + "--output-last-message", + finalMessagePath, + "--output-schema", + schemaPath, + "--sandbox", + "read-only", + "--ignore-rules", + prompt, + ]; +} + async function runWorkload( options: AgentEvalOptions, workloadPath: string, @@ -551,10 +620,23 @@ async function runWorkload( const prompt = readFileSync(workloadPath, "utf8"); const mcpConfigPath = join(workloadDir, "mcp.json"); + const codexConfigPath = join(workloadDir, "codex-config.toml"); + const codexFinalPath = join(workloadDir, "codex-final.txt"); writeFileSync(join(workloadDir, "prompt.md"), prompt); writeJson(mcpConfigPath, mcpConfig); - - const command = buildClaudeCommand(prompt, mcpConfigPath); + writeFileSync(codexConfigPath, buildCodexConfig(options)); + + const command = + options.agent === "claude" + ? buildClaudeCommand(prompt, mcpConfigPath) + : buildCodexCommand( + prompt, + workspaceDir, + codexFinalPath, + options.schemaPath, + options, + ); + const workloadEnv = { ...env }; const metadataBase = { id, path: workloadPath, @@ -573,7 +655,7 @@ async function runWorkload( const result = await runWithTimeout( command, workspaceDir, - env, + workloadEnv, options.timeoutSeconds, ); const durationMs = Date.now() - started; @@ -588,16 +670,29 @@ async function runWorkload( ); const finalJson = extractFinalJson(result.stdout); - const validFinalJson = isValidAgentReport(finalJson); + const codexFinalText = existsSync(codexFinalPath) + ? readFileSync(codexFinalPath, "utf8") + : undefined; + if (codexFinalText !== undefined) { + writeFileSync( + join(workloadDir, "codex-final.txt"), + redactText(codexFinalText, secretValues), + ); + } + const reportJson = + options.agent === "codex" && codexFinalText !== undefined + ? parseJsonFromText(codexFinalText) + : finalJson; + const validFinalJson = isValidAgentReport(reportJson); if (validFinalJson) { writeJson( join(workloadDir, "final.json"), - redactValue(finalJson, secretValues), + redactValue(reportJson, secretValues), ); - } else if (finalJson !== undefined) { + } else if (reportJson !== undefined) { writeJson( join(workloadDir, "invalid-final.json"), - redactValue(finalJson, secretValues), + redactValue(reportJson, secretValues), ); } @@ -630,12 +725,17 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { const mcpConfig = buildMcpConfig(options); if (!options.dryRun) { - await assertClaudeAvailable(); + if (options.agent === "claude") { + await assertClaudeAvailable(); + } else { + await assertCodexAvailable(); + } } - const [git, claude] = await Promise.all([ + const [git, claude, codex] = await Promise.all([ collectGitMetadata(options.repoRoot), claudeVersion(), + codexVersion(), ]); const workloadResults: WorkloadRunMetadata[] = []; @@ -662,6 +762,7 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { schemaPath: options.schemaPath, git, claudeVersion: claude, + codexVersion: codex, env: sanitizedEnvSummary(env), workloads: workloadResults, }); From fddcbb01944cf929b0f097e10ba50be5b061ec93 Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Thu, 7 May 2026 11:30:44 +0300 Subject: [PATCH 3/5] test: add targeted agent eval workloads --- AGENTS.md | 1 + eval/agentic/README.md | 48 +++++- eval/agentic/result.schema.json | 21 +-- eval/agentic/workloads/REPORTING.md | 15 ++ .../agentic/workloads/code-file-navigation.md | 5 + .../workloads/code-grep-investigation.md | 5 + eval/agentic/workloads/docs-discovery.md | 5 + eval/agentic/workloads/express-router.md | 14 -- eval/agentic/workloads/global-example.md | 5 + eval/agentic/workloads/package-changelog.md | 6 + .../agentic/workloads/package-dependencies.md | 6 + .../package-overview-vulnerabilities.md | 6 + .../workloads/unified-search-investigation.md | 6 + scripts/agent-eval.test.ts | 58 ++++++- scripts/agent-eval.ts | 157 ++++++++++++++++-- 15 files changed, 305 insertions(+), 53 deletions(-) create mode 100644 eval/agentic/workloads/REPORTING.md create mode 100644 eval/agentic/workloads/code-file-navigation.md create mode 100644 eval/agentic/workloads/code-grep-investigation.md create mode 100644 eval/agentic/workloads/docs-discovery.md create mode 100644 eval/agentic/workloads/global-example.md create mode 100644 eval/agentic/workloads/package-changelog.md create mode 100644 eval/agentic/workloads/package-dependencies.md create mode 100644 eval/agentic/workloads/package-overview-vulnerabilities.md create mode 100644 eval/agentic/workloads/unified-search-investigation.md diff --git a/AGENTS.md b/AGENTS.md index 9a90d004..28afc219 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ Philosophy: "If it is not tested, it is likely broken" - Use `bun test` for running tests - Use `bun run smoke:mcp` and `bun run smoke:cli` when changing MCP tools, CLI commands, shared formatters, auth/error envelopes, or MCP/CLI parity behavior. These are live smoke suites, not the normal unit suite; they must pass unauthenticated by validating auth handling, and provide deeper coverage when authenticated. +- Use `bun run agent:e2e` when changing MCP instructions, tool descriptions, or agent-facing tool behavior. Pick targeted workloads from `eval/agentic/README.md`; run both Claude and Codex for broad instruction changes when practical. Inspect `tool-calls.json` and `final.json` for actual tool use, `toolIssues`, `instructionIssues`, and usefulness, not just harness pass/fail. - Maintain smoke coverage when adding or changing user-facing tools/commands. Prefer structural UX assertions over brittle snapshots, and keep MCP `format: "json"` and CLI `--json` behavior aligned. - Keep tests async and isolated - Mock services at the interface level using factory functions diff --git a/eval/agentic/README.md b/eval/agentic/README.md index c52cc754..663354b5 100644 --- a/eval/agentic/README.md +++ b/eval/agentic/README.md @@ -65,26 +65,58 @@ Secret-like values are redacted in run metadata. Workloads are Markdown prompts. They should contain: - The task. -- A reporting contract requiring the final answer, GitHits tools used, failed or - unclear tool calls, unclear or missing MCP guidance, usefulness, and - confidence. + +The harness appends `eval/agentic/workloads/REPORTING.md` to every workload so +all agents return the same structured report. Workload files should not repeat +that reporting contract. They should not contain instructions such as "call `search` first" or "use `code_read` after `search`". That guidance must come from the MCP server. +### Workload Selection + +Use targeted workloads when a change affects a specific tool family. Use both +Claude and Codex for instruction/tool-description changes when practical; use at +least one agent for quick iteration. + +| Affected Area | Workload | +|---|---| +| Core global examples, `get_example`, `search_language`, `feedback` | `global-example.md` | +| Unified `search` / `search_status` behavior | `unified-search-investigation.md` | +| Package overview or vulnerability UX, `pkg_info`, `pkg_vulns` | `package-overview-vulnerabilities.md` | +| Dependency graph UX, `pkg_deps` | `package-dependencies.md` | +| Release notes UX, `pkg_changelog` | `package-changelog.md` | +| Documentation browsing, `docs_list`, `docs_read` | `docs-discovery.md` | +| File listing / file read UX, `code_files`, `code_read` | `code-file-navigation.md` | +| Deterministic source search UX, `code_grep` | `code-grep-investigation.md` | +| Multi-tool code navigation strategy and MCP instructions | `express-router.md` | + +For broad MCP instruction edits, run at least: + +```bash +bun run agent:e2e --agent claude --server local --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --agent codex --server local --workload eval/agentic/workloads/express-router.md +``` + +For tool-specific edits, add the workload from the table. Compare +`tool-calls.json` plus the final JSON's `toolIssues`, `instructionIssues`, and +`githitsUsefulnessReason` across branches or against a published run. + ## Artifacts Each run writes: - `run.json` with command, git, environment, and timing metadata. - One workload directory per workload with `prompt.md`, `mcp.json`, - `stdout.json`, `stderr.txt`, and `final.json` when parsing succeeds. + `stdout.json`, `stderr.txt`, `tool-calls.json`, and `final.json` when parsing + succeeds. Claude is launched with `--permission-mode bypassPermissions` so non-interactive -evals can exercise configured MCP tools without a human approval prompt. Codex is -launched with per-run `-c` MCP config overrides, `--ignore-rules`, and a -read-only sandbox so it can use normal human auth without mutating global MCP -configuration. +evals can exercise configured MCP tools without a human approval prompt, and +`--disable-slash-commands` to reduce plugin/skill contamination while preserving +normal human auth. Codex is launched with per-run `-c` MCP config overrides, +`--ignore-rules`, and a read-only sandbox so it can use normal human auth +without mutating global MCP configuration. Malformed final JSON, schema mismatches, Claude failures, and timeouts are harness failures. Raw stdout and stderr are preserved for diagnosis with known diff --git a/eval/agentic/result.schema.json b/eval/agentic/result.schema.json index d4945f6c..a8fb8e2d 100644 --- a/eval/agentic/result.schema.json +++ b/eval/agentic/result.schema.json @@ -5,7 +5,6 @@ "required": [ "status", "answer", - "githitsToolsUsed", "toolIssues", "instructionIssues", "githitsUsefulness", @@ -20,24 +19,20 @@ "answer": { "type": "string" }, - "githitsToolsUsed": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["tool", "purpose"], - "properties": { - "tool": { "type": "string" }, - "purpose": { "type": "string" } - } - } - }, "toolIssues": { "type": "array", "items": { "type": "string" } }, + "expectedToolUse": { + "type": "array", + "items": { "type": "string" } + }, + "unexpectedToolUse": { + "type": "array", + "items": { "type": "string" } + }, "instructionIssues": { "type": "array", "items": { "type": "string" } diff --git a/eval/agentic/workloads/REPORTING.md b/eval/agentic/workloads/REPORTING.md new file mode 100644 index 00000000..54684a49 --- /dev/null +++ b/eval/agentic/workloads/REPORTING.md @@ -0,0 +1,15 @@ +At the end, return valid JSON matching the provided schema. Include: + +- `status`: `success`, `failure`, or `inconclusive`. +- `answer`: your final answer with evidence. +- `toolIssues`: string descriptions of GitHits tool calls that failed or + returned unclear output, if any. +- `expectedToolUse`: optional string list of GitHits tools you expected to use + for this workload. +- `unexpectedToolUse`: optional string list of tools or fallback paths you used + but did not expect to need. +- `instructionIssues`: MCP guidance that was unclear, missing, or contradicted + observed tool behavior, if any. +- `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. +- `githitsUsefulnessReason`: why you chose that usefulness value. +- `confidence`: `high`, `medium`, or `low`. diff --git a/eval/agentic/workloads/code-file-navigation.md b/eval/agentic/workloads/code-file-navigation.md new file mode 100644 index 00000000..452bd7ae --- /dev/null +++ b/eval/agentic/workloads/code-file-navigation.md @@ -0,0 +1,5 @@ +# Workload: Source File Navigation + +You are debugging how `npm:express` creates an application object. Identify the +source file that exports the application factory, read the relevant section, and +explain how request and response prototypes are attached. diff --git a/eval/agentic/workloads/code-grep-investigation.md b/eval/agentic/workloads/code-grep-investigation.md new file mode 100644 index 00000000..83fae17e --- /dev/null +++ b/eval/agentic/workloads/code-grep-investigation.md @@ -0,0 +1,5 @@ +# Workload: Deterministic Source Grep + +You are investigating where Express imports the standalone `router` package. +Find the exact files and lines in `npm:express` that import it, then summarize +what each import site does. diff --git a/eval/agentic/workloads/docs-discovery.md b/eval/agentic/workloads/docs-discovery.md new file mode 100644 index 00000000..ad99cdb2 --- /dev/null +++ b/eval/agentic/workloads/docs-discovery.md @@ -0,0 +1,5 @@ +# Workload: Documentation Discovery + +You need to learn how Express documents routing behavior. Find relevant package +documentation for `npm:express`, read enough to summarize how routes are defined, +and include source references for the documentation you relied on. diff --git a/eval/agentic/workloads/express-router.md b/eval/agentic/workloads/express-router.md index e3d2572f..2a85a8cc 100644 --- a/eval/agentic/workloads/express-router.md +++ b/eval/agentic/workloads/express-router.md @@ -3,17 +3,3 @@ You are investigating how Express implements routing. Determine where the main Router implementation lives in the Express package and summarize the evidence you found. - -At the end, return valid JSON matching the provided schema. Include: - -- `status`: `success`, `failure`, or `inconclusive`. -- `answer`: your final answer with evidence. -- `githitsToolsUsed`: GitHits MCP tools you used, if any, with the purpose of - each call. -- `toolIssues`: string descriptions of GitHits tool calls that failed or - returned unclear output, if any. -- `instructionIssues`: MCP guidance that was unclear, missing, or contradicted - observed tool behavior, if any. -- `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. -- `githitsUsefulnessReason`: why you chose that usefulness value. -- `confidence`: `high`, `medium`, or `low`. diff --git a/eval/agentic/workloads/global-example.md b/eval/agentic/workloads/global-example.md new file mode 100644 index 00000000..443cb908 --- /dev/null +++ b/eval/agentic/workloads/global-example.md @@ -0,0 +1,5 @@ +# Workload: Canonical Example Discovery + +You need a current, canonical TypeScript example for validating user input with +Zod and returning typed parsed data. Find a suitable example or pattern from +open source and explain how you would adapt it in a small CLI command. diff --git a/eval/agentic/workloads/package-changelog.md b/eval/agentic/workloads/package-changelog.md new file mode 100644 index 00000000..d5f6ad71 --- /dev/null +++ b/eval/agentic/workloads/package-changelog.md @@ -0,0 +1,6 @@ +# Workload: Release History Assessment + +You are assessing recent `npm:express` releases before upgrading. Summarize the +most recent release notes and call out anything that looks security-relevant, +breaking, or operationally important. If full bodies are unnecessary, keep the +answer concise and say why. diff --git a/eval/agentic/workloads/package-dependencies.md b/eval/agentic/workloads/package-dependencies.md new file mode 100644 index 00000000..173c6f91 --- /dev/null +++ b/eval/agentic/workloads/package-dependencies.md @@ -0,0 +1,6 @@ +# Workload: Dependency Graph Assessment + +You are reviewing `npm:express` for dependency footprint risk. Explain its +runtime dependency shape, whether optional or non-runtime dependency groups are +relevant to the adoption decision, and whether transitive dependency details +change your recommendation. diff --git a/eval/agentic/workloads/package-overview-vulnerabilities.md b/eval/agentic/workloads/package-overview-vulnerabilities.md new file mode 100644 index 00000000..03c61c4d --- /dev/null +++ b/eval/agentic/workloads/package-overview-vulnerabilities.md @@ -0,0 +1,6 @@ +# Workload: Package Overview And Vulnerability Check + +You are evaluating whether `npm:express` is a reasonable dependency for a new +Node.js service. Summarize the latest package metadata and whether known +vulnerabilities should block adoption. Include version, license, repository +health signals if available, active advisory status, and any caveats. diff --git a/eval/agentic/workloads/unified-search-investigation.md b/eval/agentic/workloads/unified-search-investigation.md new file mode 100644 index 00000000..1c15e12c --- /dev/null +++ b/eval/agentic/workloads/unified-search-investigation.md @@ -0,0 +1,6 @@ +# Workload: Cross-Surface Search Investigation + +You are comparing how Express exposes `Router` across package metadata, source, +and public API references. Investigate `npm:express` and explain whether the +search results were enough by themselves or whether you needed follow-up reads. +Report any warnings, incomplete results, or follow-up status checks you saw. diff --git a/scripts/agent-eval.test.ts b/scripts/agent-eval.test.ts index 711bd15f..3283d3d2 100644 --- a/scripts/agent-eval.test.ts +++ b/scripts/agent-eval.test.ts @@ -5,6 +5,7 @@ import { buildEvalEnv, buildMcpConfig, collectSecretValues, + extractToolCalls, isValidAgentReport, parseArgs, redactText, @@ -147,8 +148,9 @@ describe("agent eval harness", () => { isValidAgentReport({ status: "success", answer: "Router lives in lib/router/index.js.", - githitsToolsUsed: [{ tool: "search", purpose: "find router" }], toolIssues: [], + expectedToolUse: ["code_read"], + unexpectedToolUse: [], instructionIssues: [], githitsUsefulness: "helped", githitsUsefulnessReason: "It located source evidence.", @@ -160,4 +162,58 @@ describe("agent eval harness", () => { isValidAgentReport({ status: "success", answer: "missing fields" }), ).toBe(false); }); + + it("extracts Codex MCP tool calls from JSONL events", () => { + const calls = extractToolCalls( + `${JSON.stringify({ + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "githits", + tool: "code_read", + status: "completed", + arguments: { path: "index.js" }, + }, + })}\n`, + "codex", + ); + + expect(calls).toEqual([ + { + agent: "codex", + server: "githits", + tool: "code_read", + status: "completed", + arguments: { path: "index.js" }, + }, + ]); + }); + + it("extracts Claude MCP tool calls from verbose stream events", () => { + const calls = extractToolCalls( + `${JSON.stringify({ + type: "assistant", + message: { + content: [ + { + type: "tool_use", + name: "mcp__githits__pkg_info", + input: { registry: "npm", package_name: "express" }, + }, + ], + }, + })}\n`, + "claude", + ); + + expect(calls).toEqual([ + { + agent: "claude", + server: "githits", + tool: "pkg_info", + status: "started", + arguments: { registry: "npm", package_name: "express" }, + }, + ]); + }); }); diff --git a/scripts/agent-eval.ts b/scripts/agent-eval.ts index 6655de38..d20f6c5c 100644 --- a/scripts/agent-eval.ts +++ b/scripts/agent-eval.ts @@ -23,6 +23,7 @@ export interface AgentEvalOptions { dryRun: boolean; repoRoot: string; schemaPath: string; + reportingPath: string; } export interface McpServerConfig { @@ -45,6 +46,16 @@ interface WorkloadRunMetadata { command: string[]; workspaceDir: string; workloadDir: string; + toolCallCount?: number; +} + +interface ExtractedToolCall { + agent: AgentName; + server?: string; + tool: string; + status?: string; + arguments?: unknown; + error?: unknown; } const PASSTHROUGH_ENV_KEYS = [ @@ -127,6 +138,13 @@ export function parseArgs( dryRun: false, repoRoot, schemaPath: join(repoRoot, "eval", "agentic", "result.schema.json"), + reportingPath: join( + repoRoot, + "eval", + "agentic", + "workloads", + "REPORTING.md", + ), }; for (let i = 0; i < argv.length; i += 1) { @@ -180,6 +198,12 @@ export function parseArgs( options.schemaPath = resolve(repoRoot, value); break; } + case "--reporting": { + const value = argv[++i]; + assert(value, "--reporting requires a path"); + options.reportingPath = resolve(repoRoot, value); + break; + } case "--dry-run": options.dryRun = true; break; @@ -213,6 +237,7 @@ Options: --timeout Per-workload timeout (default: 300) --published-package Package for published mode (default: githits@latest) --schema Result JSON schema path + --reporting Reporting contract markdown path --dry-run Generate artifacts without invoking Claude `); } @@ -411,6 +436,16 @@ function extractFinalJson(stdout: string): unknown | undefined { if (!line) continue; try { const event = JSON.parse(line) as Record; + const message = event.message; + if (message !== null && typeof message === "object") { + const text = extractTextFromContent( + (message as Record).content, + ); + if (text) { + const parsed = parseJsonFromText(text); + if (parsed !== undefined) return parsed; + } + } const candidates = [ event.result, event.message, @@ -437,6 +472,85 @@ function extractFinalJson(stdout: string): unknown | undefined { return undefined; } +function extractTextFromContent(content: unknown): string | undefined { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return undefined; + const parts = content.flatMap((item): string[] => { + if (item === null || typeof item !== "object") return []; + const record = item as Record; + return record.type === "text" && typeof record.text === "string" + ? [record.text] + : []; + }); + return parts.length > 0 ? parts.join("\n") : undefined; +} + +function extractClaudeToolCalls( + event: Record, +): ExtractedToolCall[] { + const message = event.message; + if (message === null || typeof message !== "object") return []; + const content = (message as Record).content; + if (!Array.isArray(content)) return []; + return content.flatMap((item): ExtractedToolCall[] => { + if (item === null || typeof item !== "object") return []; + const record = item as Record; + if (record.type !== "tool_use" || typeof record.name !== "string") + return []; + const match = record.name.match(/^mcp__(.+)__(.+)$/); + return [ + { + agent: "claude", + server: match?.[1], + tool: match?.[2] ?? record.name, + status: "started", + arguments: record.input, + }, + ]; + }); +} + +function extractCodexToolCall( + event: Record, +): ExtractedToolCall | undefined { + const item = event.item; + if (item === null || typeof item !== "object") return undefined; + const record = item as Record; + if (record.type !== "mcp_tool_call" || typeof record.tool !== "string") { + return undefined; + } + return { + agent: "codex", + server: typeof record.server === "string" ? record.server : undefined, + tool: record.tool, + status: typeof record.status === "string" ? record.status : undefined, + arguments: record.arguments, + error: record.error, + }; +} + +export function extractToolCalls( + stdout: string, + agent: AgentName, +): ExtractedToolCall[] { + const calls: ExtractedToolCall[] = []; + for (const line of stdout.split("\n")) { + if (line.trim().length === 0) continue; + try { + const event = JSON.parse(line) as Record; + if (agent === "claude") { + calls.push(...extractClaudeToolCalls(event)); + } else { + const call = extractCodexToolCall(event); + if (call) calls.push(call); + } + } catch { + // Ignore non-JSON lines. + } + } + return calls; +} + function parseJsonFromText(text: string): unknown | undefined { const trimmed = text.trim(); try { @@ -462,19 +576,6 @@ function isStringArray(value: unknown): value is string[] { ); } -function isToolUseArray(value: unknown): boolean { - return ( - Array.isArray(value) && - value.every( - (item) => - item !== null && - typeof item === "object" && - typeof (item as Record).tool === "string" && - typeof (item as Record).purpose === "string", - ) - ); -} - function isToolIssueArray(value: unknown): boolean { return ( Array.isArray(value) && @@ -489,6 +590,10 @@ function isToolIssueArray(value: unknown): boolean { ); } +function isOptionalStringArray(value: unknown): boolean { + return value === undefined || isStringArray(value); +} + export function isValidAgentReport(value: unknown): boolean { if (value === null || typeof value !== "object" || Array.isArray(value)) { return false; @@ -497,12 +602,13 @@ export function isValidAgentReport(value: unknown): boolean { const allowedKeys = new Set([ "status", "answer", - "githitsToolsUsed", "toolIssues", "instructionIssues", "githitsUsefulness", "githitsUsefulnessReason", "confidence", + "expectedToolUse", + "unexpectedToolUse", ]); if (Object.keys(report).some((key) => !allowedKeys.has(key))) { return false; @@ -512,9 +618,10 @@ export function isValidAgentReport(value: unknown): boolean { report.status === "failure" || report.status === "inconclusive") && typeof report.answer === "string" && - isToolUseArray(report.githitsToolsUsed) && isToolIssueArray(report.toolIssues) && isStringArray(report.instructionIssues) && + isOptionalStringArray(report.expectedToolUse) && + isOptionalStringArray(report.unexpectedToolUse) && (report.githitsUsefulness === "helped" || report.githitsUsefulness === "hurt" || report.githitsUsefulness === "unused" || @@ -567,8 +674,10 @@ function buildClaudeCommand(prompt: string, mcpConfigPath: string): string[] { "--mcp-config", mcpConfigPath, "--strict-mcp-config", + "--disable-slash-commands", "--output-format", - "json", + "stream-json", + "--verbose", "--permission-mode", "bypassPermissions", "--no-session-persistence", @@ -611,6 +720,10 @@ async function runWorkload( secretValues: string[], ): Promise { assert(existsSync(workloadPath), `Workload not found: ${workloadPath}`); + assert( + existsSync(options.reportingPath), + `Reporting contract not found: ${options.reportingPath}`, + ); const id = workloadId(workloadPath); const workloadDir = join(runDir, "workloads", id); const workspaceDir = mkdtempSync( @@ -618,7 +731,9 @@ async function runWorkload( ); mkdirSync(workloadDir, { recursive: true }); - const prompt = readFileSync(workloadPath, "utf8"); + const workloadPrompt = readFileSync(workloadPath, "utf8").trimEnd(); + const reportingPrompt = readFileSync(options.reportingPath, "utf8").trim(); + const prompt = `${workloadPrompt}\n\n${reportingPrompt}\n`; const mcpConfigPath = join(workloadDir, "mcp.json"); const codexConfigPath = join(workloadDir, "codex-config.toml"); const codexFinalPath = join(workloadDir, "codex-final.txt"); @@ -669,6 +784,12 @@ async function runWorkload( redactText(result.stderr, secretValues), ); + const toolCalls = extractToolCalls(result.stdout, options.agent); + writeJson( + join(workloadDir, "tool-calls.json"), + redactValue(toolCalls, secretValues), + ); + const finalJson = extractFinalJson(result.stdout); const codexFinalText = existsSync(codexFinalPath) ? readFileSync(codexFinalPath, "utf8") @@ -708,6 +829,7 @@ async function runWorkload( exitCode: result.exitCode, durationMs, timedOut: result.timedOut, + toolCallCount: toolCalls.length, }; } finally { rmSync(workspaceDir, { recursive: true, force: true }); @@ -760,6 +882,7 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { timeoutSeconds: options.timeoutSeconds, repoRoot: options.repoRoot, schemaPath: options.schemaPath, + reportingPath: options.reportingPath, git, claudeVersion: claude, codexVersion: codex, From 331f1fcb9b4f855ba1e7666bdd47fea0829aed83 Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Thu, 7 May 2026 11:58:25 +0300 Subject: [PATCH 4/5] test: tighten agent eval reporting contract --- eval/agentic/README.md | 20 ++++++++++++++++++++ eval/agentic/result.schema.json | 2 ++ eval/agentic/workloads/REPORTING.md | 11 ++++++----- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/eval/agentic/README.md b/eval/agentic/README.md index 663354b5..d2570afa 100644 --- a/eval/agentic/README.md +++ b/eval/agentic/README.md @@ -102,6 +102,26 @@ For tool-specific edits, add the workload from the table. Compare `tool-calls.json` plus the final JSON's `toolIssues`, `instructionIssues`, and `githitsUsefulnessReason` across branches or against a published run. +### Current Baseline Observations + +The initial local baseline ran all workloads against Claude and Codex. Expected +tool families were exercised after tightening the shared reporting contract. +Notable findings to keep in mind when evaluating future changes: + +- `global-example.md` exercises `get_example`; agents may combine it with docs, + source, or package metadata when they need stronger canonical evidence. +- `code-grep-investigation.md` surfaced a real `code_grep` regex limitation for + short/stopword-heavy patterns; literal grep is the reliable path for that + workload. +- `unified-search-investigation.md` intentionally exposes `search` warnings and + follow-up needs. Agents should inspect warnings and use `code_read`, + `docs_read`, or `code_grep` when top search hits are incomplete/noisy. +- Codex sometimes reports a tool as unavailable until it performs additional + tool discovery. Use `tool-calls.json` to distinguish actual unavailable tools + from delayed discovery. +- `tool-calls.json` is the source of truth for tool usage. The final JSON is for + the agent's assessment of clarity, issues, and usefulness. + ## Artifacts Each run writes: diff --git a/eval/agentic/result.schema.json b/eval/agentic/result.schema.json index a8fb8e2d..09453191 100644 --- a/eval/agentic/result.schema.json +++ b/eval/agentic/result.schema.json @@ -6,6 +6,8 @@ "status", "answer", "toolIssues", + "expectedToolUse", + "unexpectedToolUse", "instructionIssues", "githitsUsefulness", "githitsUsefulnessReason", diff --git a/eval/agentic/workloads/REPORTING.md b/eval/agentic/workloads/REPORTING.md index 54684a49..27b27e0c 100644 --- a/eval/agentic/workloads/REPORTING.md +++ b/eval/agentic/workloads/REPORTING.md @@ -1,13 +1,14 @@ -At the end, return valid JSON matching the provided schema. Include: +Return only valid JSON matching the provided schema. Do not include markdown, +prose, code fences, or any text outside the JSON object. Include: - `status`: `success`, `failure`, or `inconclusive`. - `answer`: your final answer with evidence. - `toolIssues`: string descriptions of GitHits tool calls that failed or returned unclear output, if any. -- `expectedToolUse`: optional string list of GitHits tools you expected to use - for this workload. -- `unexpectedToolUse`: optional string list of tools or fallback paths you used - but did not expect to need. +- `expectedToolUse`: string list of GitHits tools you expected to use for this + workload. Use `[]` if none. +- `unexpectedToolUse`: string list of tools or fallback paths you used but did + not expect to need. Use `[]` if none. - `instructionIssues`: MCP guidance that was unclear, missing, or contradicted observed tool behavior, if any. - `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. From cc3c2fa5d1471bfdd064cd3b60ce32e314c35ab8 Mon Sep 17 00:00:00 2001 From: Juha Litola Date: Thu, 7 May 2026 12:05:58 +0300 Subject: [PATCH 5/5] docs: clarify agent eval purpose --- AGENTS.md | 2 +- eval/agentic/README.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 28afc219..89e06355 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Philosophy: "If it is not tested, it is likely broken" - Use `bun test` for running tests - Use `bun run smoke:mcp` and `bun run smoke:cli` when changing MCP tools, CLI commands, shared formatters, auth/error envelopes, or MCP/CLI parity behavior. These are live smoke suites, not the normal unit suite; they must pass unauthenticated by validating auth handling, and provide deeper coverage when authenticated. -- Use `bun run agent:e2e` when changing MCP instructions, tool descriptions, or agent-facing tool behavior. Pick targeted workloads from `eval/agentic/README.md`; run both Claude and Codex for broad instruction changes when practical. Inspect `tool-calls.json` and `final.json` for actual tool use, `toolIssues`, `instructionIssues`, and usefulness, not just harness pass/fail. +- Use `bun run agent:e2e` when changing MCP instructions, tool descriptions, or agent-facing tool behavior. This is a human/agent-driven qualitative eval, not a deterministic CI gate. Pick targeted workloads from `eval/agentic/README.md`; run both Claude and Codex for broad instruction changes when practical. Inspect `tool-calls.json` and `final.json` for actual tool use, `toolIssues`, `instructionIssues`, and usefulness, not just harness pass/fail. - Maintain smoke coverage when adding or changing user-facing tools/commands. Prefer structural UX assertions over brittle snapshots, and keep MCP `format: "json"` and CLI `--json` behavior aligned. - Keep tests async and isolated - Mock services at the interface level using factory functions diff --git a/eval/agentic/README.md b/eval/agentic/README.md index d2570afa..36cbdcce 100644 --- a/eval/agentic/README.md +++ b/eval/agentic/README.md @@ -7,6 +7,14 @@ server's own instructions and tool descriptions. It is not a smoke test. Smoke tests exercise CLI and MCP contracts directly. Agentic evals exercise agent behavior end-to-end. +This harness is intentionally human/agent-driven, not CI. Use it to understand +how MCP instruction or tool-description changes affect real agent behavior. Do +not treat a live agent pass/fail result as a deterministic regression test: model +behavior, backend indexing state, auth state, network conditions, and package +data can all change. The useful output is the artifact set, especially +`tool-calls.json`, `final.json`, `toolIssues`, `instructionIssues`, and the +agent's usefulness assessment. + ## What Is Under Test - Local mode starts the MCP server from this checkout with @@ -15,6 +23,10 @@ Agentic evals exercise agent behavior end-to-end. by default. - To evaluate MCP instruction changes, change branch/source and run local mode. +Smoke tests are the right fit for CI gating. Agentic evals are the right fit for +qualitative review before/after instruction, tool-description, and agent-facing +UX changes. + The harness must not add GitHits usage guidance through agent system prompts, append prompts, alternate MCP instruction files, project instructions, or plugin commands. Workload prompts may ask the agent to report what happened, but must