diff --git a/biome.json b/biome.json index 84afa11c..285e1876 100644 --- a/biome.json +++ b/biome.json @@ -6,6 +6,7 @@ "**/*.json", "**/*.yml", "**/*.yaml", + "!.agent-eval", "!dist", "!node_modules" ], diff --git a/eval/agentic/README.md b/eval/agentic/README.md index dfcac004..c80af055 100644 --- a/eval/agentic/README.md +++ b/eval/agentic/README.md @@ -48,6 +48,8 @@ login should work by default. Automation can use `GITHITS_API_TOKEN`. bun run agent:e2e --server local --workload eval/agentic/workloads/express-router.md bun run agent:e2e --server published --workload eval/agentic/workloads/express-router.md bun run agent:e2e --agent codex --server local --workload eval/agentic/workloads/express-router.md +bun run agent:e2e:report .agent-eval/runs/ +bun run agent:e2e:report --compare .agent-eval/runs/ .agent-eval/runs/ ``` Useful options: @@ -72,6 +74,31 @@ Normal GitHits backend overrides are passed through when set: Secret-like values are redacted in run metadata. +After each run, the harness prints a concise summary with the run directory, +per-workload status, unique GitHits tool count, raw tool event count, +usefulness/confidence when available, key artifact paths, and reported +tool/instruction issues. It also prints a `Next:` block with the exact report, +compare, and raw-call inspection commands an agent should use for follow-up. The +same summary can be regenerated later from persisted artifacts: + +```bash +bun run agent:e2e:report .agent-eval/runs/ +bun run agent:e2e:report --json .agent-eval/runs/ +``` + +Use comparison mode for before/after review against published or a saved main +branch run: + +```bash +bun run agent:e2e --server published --out .agent-eval/runs/published-baseline --workload eval/agentic/workloads/package-overview-vulnerabilities.md +bun run agent:e2e --server local --out .agent-eval/runs/local-change --workload eval/agentic/workloads/package-overview-vulnerabilities.md +bun run agent:e2e:report --compare .agent-eval/runs/published-baseline .agent-eval/runs/local-change +``` + +Same-agent comparisons include normalized aggregate status counts. Cross-agent +comparisons intentionally degrade to tool-name presence with a warning because +Claude and Codex expose different tool-call status events. + ## Workloads Workloads are Markdown prompts. They should contain: @@ -95,7 +122,7 @@ least one agent for quick iteration. |---|---| | Core global examples, `get_example`, `search_language`, `feedback` | `global-example.md` | | Unified `search` / `search_status` behavior | `unified-search-investigation.md` | -| Package overview or vulnerability UX, `pkg_info`, `pkg_vulns` | `package-overview-vulnerabilities.md` | +| Package overview or vulnerability UX, `pkg_info`, `pkg_vulns` | `package-overview-vulnerabilities.md`; use `package-vulnerability-filter.md` for severity/version filtering behavior | | Dependency graph UX, `pkg_deps` | `package-dependencies.md` | | Release notes UX, `pkg_changelog` | `package-changelog.md`; use `package-changelog-range.md` for range/body-preview behavior | | Documentation browsing, `docs_list`, `docs_read` | `docs-discovery.md` | @@ -133,12 +160,17 @@ Notable findings to keep in mind when evaluating future changes: from delayed discovery. - `tool-calls.json` is the source of truth for tool usage. The final JSON is for the agent's assessment of clarity, issues, and usefulness. +- `report.json` and `agent:e2e:report` are derived review aids. They normalize + tool names/statuses for readability but do not replace raw artifacts. ## Artifacts Each run writes: - `run.json` with command, git, environment, and timing metadata. +- `summary.json` with backward-compatible execution status metadata. +- `report.json` with derived review fields, normalized tool summaries, relative + artifact paths, and warnings for missing artifacts or self-report drift. - One workload directory per workload with `prompt.md`, `mcp.json`, `stdout.json`, `stderr.txt`, `tool-calls.json`, and `final.json` when parsing succeeds. diff --git a/package.json b/package.json index 5043d70d..00e205ec 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "smoke:cli": "bun run scripts/cli-smoke.ts", "smoke:mcp": "bun run scripts/mcp-smoke.ts", "agent:e2e": "bun run scripts/agent-eval.ts", + "agent:e2e:report": "bun run scripts/agent-eval-report.ts", "test": "bun test", "typecheck": "tsc", "format": "biome format --write .", diff --git a/scripts/agent-eval-report.ts b/scripts/agent-eval-report.ts new file mode 100644 index 00000000..45568778 --- /dev/null +++ b/scripts/agent-eval-report.ts @@ -0,0 +1,617 @@ +import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { basename, isAbsolute, join, relative, resolve } from "node:path"; + +export type AgentEvalReportMode = "report" | "json" | "compare"; +export type NormalizedToolStatus = + | "started" + | "completed" + | "failed" + | "unknown"; + +export interface AgentEvalReportOptions { + mode: AgentEvalReportMode; + runDir?: string; + beforeRunDir?: string; + afterRunDir?: string; +} + +export interface AgentEvalRunMetadata { + agent?: string; + server?: string; + dryRun?: boolean; + git?: Record; + workloads?: WorkloadRunMetadata[]; +} + +export interface WorkloadRunMetadata { + id: string; + status: string; + durationMs?: number; + exitCode?: number; + timedOut?: boolean; + workloadDir?: string; +} + +export interface ExtractedToolCallForReport { + tool: string; + status?: string; + error?: unknown; +} + +export interface ToolCallSummary { + rawCount: number; + uniqueTools: string[]; + statusCounts: Record; + errors: string[]; +} + +export interface FinalReportSummary { + status: string; + usefulness: string; + usefulnessReason: string; + confidence: string; + expectedToolUse: string[]; + unexpectedToolUse: string[]; + toolIssues: string[]; + instructionIssues: string[]; +} + +export interface WorkloadReport { + id: string; + status: string; + durationMs?: number; + exitCode?: number; + timedOut?: boolean; + artifacts: Record; + missingArtifacts: string[]; + toolCalls: ToolCallSummary; + finalReport?: FinalReportSummary; + warnings: string[]; +} + +export interface AgentEvalReport { + schemaVersion: 1; + status: string; + agent?: string; + server?: string; + dryRun?: boolean; + git?: Record; + runDir: string; + workloads: WorkloadReport[]; + warnings: string[]; +} + +export interface AgentEvalCompareReport { + beforeRunDir: string; + afterRunDir: string; + sameAgent: boolean; + warnings: string[]; + workloads: string[]; + lines: string[]; +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function readJson(path: string): unknown | undefined { + if (!existsSync(path)) return undefined; + return JSON.parse(readFileSync(path, "utf8")); +} + +function asRecord(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function toolIssueArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.flatMap((item): string[] => { + if (typeof item === "string") return [item]; + const record = asRecord(item); + if ( + record && + typeof record.tool === "string" && + typeof record.issue === "string" + ) { + return [`${record.tool}: ${record.issue}`]; + } + return []; + }); +} + +function looksLikeToolName(value: string): boolean { + return /^[a-z][a-z0-9_]*$/.test(value); +} + +function isSafeWorkloadId(value: string): boolean { + return ( + value.length > 0 && + !value.includes("/") && + !value.includes("\\") && + value !== ".." && + !value.includes("..") + ); +} + +function relativeArtifact(runDir: string, path: string): string { + return relative(runDir, path) || basename(path); +} + +export function isContainedRelativePath(relativePath: string): boolean { + return ( + relativePath !== "" && + !relativePath.startsWith("..") && + !isAbsolute(relativePath) && + !/^[A-Za-z]:[\\/]/.test(relativePath) + ); +} + +function realPathInsideRun(runDir: string, path: string): string | undefined { + if (!existsSync(path)) return undefined; + const realRunDir = realpathSync(runDir); + const realPath = realpathSync(path); + const relativePath = relative(realRunDir, realPath); + if (!isContainedRelativePath(relativePath)) return undefined; + return realPath; +} + +export function parseReportArgs(argv: string[]): AgentEvalReportOptions { + if (argv.length === 0) { + throw new Error( + "Usage: bun run agent:e2e:report [--json] | --compare ", + ); + } + if (argv[0] === "--json") { + assert(argv.length === 2, "--json requires exactly one run directory"); + return { mode: "json", runDir: argv[1] }; + } + if (argv[0] === "--compare") { + assert( + argv.length === 3, + "--compare requires before and after run directories", + ); + return { mode: "compare", beforeRunDir: argv[1], afterRunDir: argv[2] }; + } + assert(!argv[0]?.startsWith("--"), `Unknown argument: ${argv[0]}`); + assert(argv.length === 1, "report mode accepts exactly one run directory"); + return { mode: "report", runDir: argv[0] }; +} + +export function normalizeToolName(name: string): string { + const trimmed = name.trim(); + const claude = trimmed.match(/^mcp__[^_.]+__\.?(.+)$/); + if (claude?.[1]) return claude[1].replace(/^\./, ""); + const dotted = trimmed.match(/^mcp__[^.]+\.([^\s]+)$/); + if (dotted?.[1]) return dotted[1]; + return trimmed.replace(/^githits[_.-]/, ""); +} + +export function normalizeToolStatus( + status: string | undefined, + error?: unknown, +): NormalizedToolStatus { + if (error !== undefined && error !== null) return "failed"; + if (!status) return "unknown"; + const normalized = status.toLowerCase().replace(/[\s-]/g, "_"); + if (["completed", "success", "succeeded", "done"].includes(normalized)) { + return "completed"; + } + if (["failed", "failure", "error", "errored"].includes(normalized)) { + return "failed"; + } + if (["started", "in_progress", "running", "pending"].includes(normalized)) { + return "started"; + } + return "unknown"; +} + +export function summarizeToolCalls( + calls: ExtractedToolCallForReport[], +): ToolCallSummary { + const statusCounts: Record = { + started: 0, + completed: 0, + failed: 0, + unknown: 0, + }; + const tools = new Set(); + const errors: string[] = []; + for (const call of calls) { + tools.add(normalizeToolName(call.tool)); + const status = normalizeToolStatus(call.status, call.error); + statusCounts[status] += 1; + if (call.error !== undefined && call.error !== null) { + errors.push( + typeof call.error === "string" + ? call.error + : JSON.stringify(call.error), + ); + } + } + return { + rawCount: calls.length, + uniqueTools: [...tools].sort(), + statusCounts, + errors, + }; +} + +export function summarizeFinalReport( + report: unknown, +): FinalReportSummary | undefined { + const record = asRecord(report); + if (!record) return undefined; + return { + status: typeof record.status === "string" ? record.status : "unknown", + usefulness: + typeof record.githitsUsefulness === "string" + ? record.githitsUsefulness + : "unknown", + usefulnessReason: + typeof record.githitsUsefulnessReason === "string" + ? record.githitsUsefulnessReason + : "", + confidence: + typeof record.confidence === "string" ? record.confidence : "unknown", + expectedToolUse: stringArray(record.expectedToolUse).map(normalizeToolName), + unexpectedToolUse: stringArray(record.unexpectedToolUse).map( + normalizeToolName, + ), + toolIssues: toolIssueArray(record.toolIssues), + instructionIssues: stringArray(record.instructionIssues), + }; +} + +export function workloadIdFromPath(workloadPath: string): string { + return basename(workloadPath).replace(/\.[^.]+$/, ""); +} + +export function assertUniqueWorkloadIds(workloads: string[]): void { + const seen = new Map(); + for (const workload of workloads) { + const id = workloadIdFromPath(workload); + const previous = seen.get(id); + if (previous) { + throw new Error( + `Duplicate workload id "${id}" from ${previous} and ${workload}`, + ); + } + seen.set(id, workload); + } +} + +function workloadDirFor(runDir: string, workload: WorkloadRunMetadata): string { + const safeId = isSafeWorkloadId(workload.id) ? workload.id : "__invalid__"; + const defaultDir = join(runDir, "workloads", safeId); + if (!workload.workloadDir) return defaultDir; + const resolvedRunDir = resolve(runDir); + const resolvedWorkloadDir = resolve(workload.workloadDir); + const relativePath = relative(resolvedRunDir, resolvedWorkloadDir); + if (!isContainedRelativePath(relativePath)) return defaultDir; + return resolvedWorkloadDir; +} + +function readToolCalls(path: string): ExtractedToolCallForReport[] | undefined { + const value = readJson(path); + if (!Array.isArray(value)) return undefined; + return value.flatMap((item): ExtractedToolCallForReport[] => { + const record = asRecord(item); + if (!record || typeof record.tool !== "string") return []; + return [ + { + tool: record.tool, + status: typeof record.status === "string" ? record.status : undefined, + error: record.error, + }, + ]; + }); +} + +function buildWorkloadReport( + runDir: string, + workload: WorkloadRunMetadata, +): WorkloadReport { + const workloadDir = workloadDirFor(runDir, workload); + const paths = { + toolCalls: join(workloadDir, "tool-calls.json"), + final: join(workloadDir, "final.json"), + invalidFinal: join(workloadDir, "invalid-final.json"), + stderr: join(workloadDir, "stderr.txt"), + }; + const artifacts: Record = {}; + const missingArtifacts: string[] = []; + const unsafeArtifacts: string[] = []; + const safePaths: Partial> = {}; + for (const [name, path] of Object.entries(paths)) { + const safePath = realPathInsideRun(runDir, path); + if (safePath) { + safePaths[name as keyof typeof paths] = safePath; + artifacts[name] = relativeArtifact(runDir, path); + } else if (existsSync(path)) { + unsafeArtifacts.push(`${name}: ${path}`); + } + } + if (!artifacts.toolCalls) missingArtifacts.push("tool-calls.json"); + if (!artifacts.final && !artifacts.invalidFinal) + missingArtifacts.push("final.json"); + if (!artifacts.stderr) missingArtifacts.push("stderr.txt"); + + const toolCalls = summarizeToolCalls( + safePaths.toolCalls ? (readToolCalls(safePaths.toolCalls) ?? []) : [], + ); + const finalReport = summarizeFinalReport( + safePaths.final ? readJson(safePaths.final) : undefined, + ); + const warnings: string[] = []; + if (!isSafeWorkloadId(workload.id)) { + warnings.push( + `invalid workload id ignored for artifact paths: ${workload.id}`, + ); + } + for (const unsafe of unsafeArtifacts) { + warnings.push(`artifact path outside run directory ignored: ${unsafe}`); + } + if (missingArtifacts.length > 0 && workload.status === "success") { + warnings.push( + `success workload is missing artifacts: ${missingArtifacts.join(", ")}`, + ); + } + if (finalReport) { + const rawTools = new Set(toolCalls.uniqueTools); + const drift = finalReport.unexpectedToolUse.filter( + (tool) => looksLikeToolName(tool) && !rawTools.has(tool), + ); + if (drift.length > 0) { + warnings.push( + `self-report drift: unexpectedToolUse not present in raw calls: ${drift.join(", ")}`, + ); + } + } + + return { + id: workload.id, + status: workload.status, + durationMs: workload.durationMs, + exitCode: workload.exitCode, + timedOut: workload.timedOut, + artifacts, + missingArtifacts, + toolCalls, + finalReport, + warnings, + }; +} + +export function buildRunReportFromMetadata( + runDir: string, + metadata: AgentEvalRunMetadata, +): AgentEvalReport { + const workloads = metadata.workloads ?? []; + const ids = new Set(); + const warnings: string[] = []; + for (const workload of workloads) { + if (ids.has(workload.id)) + warnings.push(`duplicate workload id in run metadata: ${workload.id}`); + ids.add(workload.id); + } + const reports = workloads.map((workload) => + buildWorkloadReport(runDir, workload), + ); + const status = reports.some((workload) => + ["failed", "timeout"].includes(workload.status), + ) + ? "failed" + : metadata.dryRun + ? "dry-run" + : "success"; + return { + schemaVersion: 1, + status, + agent: metadata.agent, + server: metadata.server, + dryRun: metadata.dryRun, + git: metadata.git, + runDir, + workloads: reports, + warnings, + }; +} + +export function loadRunReport(runDir: string): AgentEvalReport { + const runPath = join(runDir, "run.json"); + const metadata = readJson(runPath); + assert(metadata, `run.json not found in ${runDir}`); + return buildRunReportFromMetadata(runDir, metadata as AgentEvalRunMetadata); +} + +function formatDuration(ms: number | undefined): string { + if (ms === undefined) return "n/a"; + return `${(ms / 1000).toFixed(1)}s`; +} + +export function formatRunReport(report: AgentEvalReport): string { + const lines = [ + `Agent eval: ${report.status} (${report.agent ?? "unknown"}/${report.server ?? "unknown"}) ${report.runDir}`, + ]; + for (const workload of report.workloads) { + const final = workload.finalReport; + const details = final + ? ` usefulness=${final.usefulness} confidence=${final.confidence}` + : ""; + lines.push( + `${workload.id} ${workload.status} ${formatDuration(workload.durationMs)} uniqueTools=${workload.toolCalls.uniqueTools.length} rawEvents=${workload.toolCalls.rawCount}${details}`, + ); + const artifacts = [ + workload.artifacts.toolCalls, + workload.artifacts.final ?? workload.artifacts.invalidFinal, + workload.artifacts.stderr, + ].filter(Boolean); + if (artifacts.length > 0) + lines.push(` artifacts: ${artifacts.join(", ")}`); + for (const warning of workload.warnings) + lines.push(` warning: ${warning}`); + } + for (const warning of report.warnings) lines.push(`Warning: ${warning}`); + const issues = report.workloads.flatMap((workload) => { + const final = workload.finalReport; + if (!final) return []; + return [ + ...final.toolIssues.map((issue) => `${workload.id} tool: ${issue}`), + ...final.instructionIssues.map( + (issue) => `${workload.id} instruction: ${issue}`, + ), + ]; + }); + if (issues.length > 0) { + lines.push("Issues:"); + for (const issue of issues) lines.push(`- ${issue}`); + } + lines.push("Next:"); + lines.push(`- Reopen summary: bun run agent:e2e:report ${report.runDir}`); + lines.push( + `- Compare with baseline: bun run agent:e2e:report --compare ${report.runDir}`, + ); + const firstWorkload = report.workloads.find( + (workload) => workload.artifacts.toolCalls, + ); + if (firstWorkload?.artifacts.toolCalls) { + lines.push(`- Inspect raw calls: ${firstWorkload.artifacts.toolCalls}`); + } + return `${lines.join("\n")}\n`; +} + +function diffStrings(before: string[], after: string[]): string[] { + const beforeSet = new Set(before); + const afterSet = new Set(after); + return [ + ...after.filter((item) => !beforeSet.has(item)).map((item) => `+${item}`), + ...before.filter((item) => !afterSet.has(item)).map((item) => `-${item}`), + ]; +} + +function formatStatusCounts(summary: ToolCallSummary): string { + return Object.entries(summary.statusCounts) + .filter(([, count]) => count > 0) + .map(([status, count]) => `${status}=${count}`) + .join(" "); +} + +export function compareReports( + before: AgentEvalReport, + after: AgentEvalReport, +): AgentEvalCompareReport { + const beforeMap = new Map( + before.workloads.map((workload) => [workload.id, workload]), + ); + const afterMap = new Map( + after.workloads.map((workload) => [workload.id, workload]), + ); + const ids = [...new Set([...beforeMap.keys(), ...afterMap.keys()])].sort(); + const sameAgent = before.agent === after.agent; + const warnings = sameAgent + ? [] + : [ + "cross-agent comparison: status/event counts are not comparable; showing tool-name presence only", + ]; + const lines = [ + `Agent eval compare: before=${before.runDir} after=${after.runDir}`, + ...warnings.map((warning) => `Warning: ${warning}`), + ]; + for (const id of ids) { + const left = beforeMap.get(id); + const right = afterMap.get(id); + if (!left) { + lines.push(`${id} missing in before`); + continue; + } + if (!right) { + lines.push(`${id} missing in after`); + continue; + } + const status = + left.status === right.status + ? `unchanged ${right.status}` + : `${left.status} -> ${right.status}`; + lines.push(`${id} status ${status}`); + const toolDiff = diffStrings( + left.toolCalls.uniqueTools, + right.toolCalls.uniqueTools, + ); + if (sameAgent) { + lines.push( + ` tools: raw events ${left.toolCalls.rawCount} -> ${right.toolCalls.rawCount}; statuses ${formatStatusCounts(left.toolCalls)} -> ${formatStatusCounts(right.toolCalls)}${toolDiff.length > 0 ? `; ${toolDiff.join(", ")}` : ""}`, + ); + } else if (toolDiff.length > 0) { + lines.push(` tools: ${toolDiff.join(", ")}`); + } + const instructionDiff = diffStrings( + left.finalReport?.instructionIssues ?? [], + right.finalReport?.instructionIssues ?? [], + ); + if (instructionDiff.length > 0) + lines.push(` instructionIssues: ${instructionDiff.join(", ")}`); + const toolIssueDiff = diffStrings( + left.finalReport?.toolIssues ?? [], + right.finalReport?.toolIssues ?? [], + ); + if (toolIssueDiff.length > 0) + lines.push(` toolIssues: ${toolIssueDiff.join(", ")}`); + } + return { + beforeRunDir: before.runDir, + afterRunDir: after.runDir, + sameAgent, + warnings, + workloads: ids, + lines, + }; +} + +export function formatCompareReport(report: AgentEvalCompareReport): string { + return `${report.lines.join("\n")}\n`; +} + +export function writeReportJson(runDir: string, report: AgentEvalReport): void { + writeFileSync( + join(runDir, "report.json"), + `${JSON.stringify(report, null, 2)}\n`, + ); +} + +export async function runReportCli(argv: string[]): Promise { + const options = parseReportArgs(argv); + if (options.mode === "compare") { + assert( + options.beforeRunDir && options.afterRunDir, + "compare requires two run directories", + ); + const before = loadRunReport(options.beforeRunDir); + const after = loadRunReport(options.afterRunDir); + console.log(formatCompareReport(compareReports(before, after)).trimEnd()); + return; + } + assert(options.runDir, "report requires a run directory"); + const report = loadRunReport(options.runDir); + writeReportJson(options.runDir, report); + if (options.mode === "json") { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log(formatRunReport(report).trimEnd()); + } +} + +if (import.meta.main) { + runReportCli(process.argv.slice(2)).catch((error) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exit(1); + }); +} diff --git a/scripts/agent-eval.test.ts b/scripts/agent-eval.test.ts index 3283d3d2..5cb3deef 100644 --- a/scripts/agent-eval.test.ts +++ b/scripts/agent-eval.test.ts @@ -1,4 +1,14 @@ import { describe, expect, it } from "bun:test"; +import { + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; import { buildCodexConfig, buildCodexConfigArgs, @@ -11,6 +21,59 @@ import { redactText, sanitizedEnvSummary, } from "./agent-eval.ts"; +import { + assertUniqueWorkloadIds, + buildRunReportFromMetadata, + compareReports, + formatCompareReport, + formatRunReport, + isContainedRelativePath, + normalizeToolName, + normalizeToolStatus, + parseReportArgs, + summarizeFinalReport, + summarizeToolCalls, +} from "./agent-eval-report.ts"; + +function writeJson(path: string, value: unknown): void { + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function createRunFixture(status = "success"): string { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-test-")); + const workloadDir = join(runDir, "workloads", "pkg-vulns"); + mkdirSync(workloadDir, { recursive: true }); + writeJson(join(workloadDir, "tool-calls.json"), [ + { agent: "codex", tool: "pkg_vulns", status: "in_progress" }, + { agent: "codex", tool: "pkg_vulns", status: "completed" }, + ]); + writeJson(join(workloadDir, "final.json"), { + status: "success", + answer: "No active vulnerabilities.", + toolIssues: [], + expectedToolUse: ["mcp__githits__pkg_vulns"], + unexpectedToolUse: [], + instructionIssues: ["Package aliases were unclear"], + githitsUsefulness: "helped", + githitsUsefulnessReason: "It returned advisory details.", + confidence: "high", + }); + writeFileSync(join(workloadDir, "stderr.txt"), ""); + writeJson(join(runDir, "run.json"), { + agent: "codex", + server: "local", + dryRun: false, + workloads: [ + { + id: "pkg-vulns", + status, + durationMs: 1234, + workloadDir, + }, + ], + }); + return runDir; +} describe("agent eval harness", () => { it("builds local MCP config with explicit repo cwd", () => { @@ -216,4 +279,300 @@ describe("agent eval harness", () => { }, ]); }); + + it("ignores non-MCP Claude tool calls", () => { + const calls = extractToolCalls( + `${JSON.stringify({ + type: "assistant", + message: { + content: [ + { + type: "tool_use", + name: "ToolSearch", + input: { query: "pkg_vulns" }, + }, + ], + }, + })}\n`, + "claude", + ); + + expect(calls).toEqual([]); + }); + + it("normalizes tool names and statuses for reports", () => { + expect(normalizeToolName("mcp__githits__pkg_vulns")).toBe("pkg_vulns"); + expect(normalizeToolName("mcp__githits__.pkg_vulns")).toBe("pkg_vulns"); + expect(normalizeToolName("githits.pkg_vulns")).toBe("pkg_vulns"); + expect(normalizeToolName("pkg_vulns")).toBe("pkg_vulns"); + expect(normalizeToolStatus("in_progress")).toBe("started"); + expect(normalizeToolStatus("completed")).toBe("completed"); + expect(normalizeToolStatus(undefined)).toBe("unknown"); + expect(normalizeToolStatus("completed", "boom")).toBe("failed"); + }); + + it("rejects escaped relative paths across platforms", () => { + expect(isContainedRelativePath("workloads/pkg/tool-calls.json")).toBe(true); + expect(isContainedRelativePath("../outside/tool-calls.json")).toBe(false); + expect(isContainedRelativePath("/outside/tool-calls.json")).toBe(false); + expect(isContainedRelativePath("D:\\outside\\tool-calls.json")).toBe(false); + }); + + it("summarizes raw tool calls without hiding duplicate status events", () => { + const summary = summarizeToolCalls([ + { tool: "mcp__githits__pkg_vulns", status: "in_progress" }, + { tool: "pkg_vulns", status: "completed" }, + { tool: "pkg_info", status: "failed", error: { message: "bad" } }, + ]); + + expect(summary.rawCount).toBe(3); + expect(summary.uniqueTools).toEqual(["pkg_info", "pkg_vulns"]); + expect(summary.statusCounts).toEqual({ + started: 1, + completed: 1, + failed: 1, + unknown: 0, + }); + expect(summary.errors[0]).toContain("bad"); + }); + + it("summarizes final reports without treating expected tools as actual calls", () => { + const summary = summarizeFinalReport({ + status: "success", + githitsUsefulness: "helped", + githitsUsefulnessReason: "useful", + confidence: "high", + expectedToolUse: ["mcp__githits__pkg_vulns"], + unexpectedToolUse: ["mcp__githits__.pkg_info"], + toolIssues: ["issue", { tool: "pkg_vulns", issue: "unclear range" }], + instructionIssues: ["instruction"], + }); + + expect(summary?.expectedToolUse).toEqual(["pkg_vulns"]); + expect(summary?.unexpectedToolUse).toEqual(["pkg_info"]); + expect(summary?.toolIssues).toEqual(["issue", "pkg_vulns: unclear range"]); + }); + + it("builds a portable run report from persisted artifacts", () => { + const runDir = createRunFixture(); + const run = JSON.parse(readFileSync(join(runDir, "run.json"), "utf8")); + const report = buildRunReportFromMetadata(runDir, run); + + expect(report.schemaVersion).toBe(1); + expect(report.workloads[0]?.artifacts.toolCalls).toBe( + "workloads/pkg-vulns/tool-calls.json", + ); + expect(report.workloads[0]?.toolCalls.rawCount).toBe(2); + expect(report.workloads[0]?.finalReport?.instructionIssues).toEqual([ + "Package aliases were unclear", + ]); + const formatted = formatRunReport(report); + expect(formatted).toContain( + "pkg-vulns success 1.2s uniqueTools=1 rawEvents=2", + ); + expect(formatted).toContain( + `Reopen summary: bun run agent:e2e:report ${runDir}`, + ); + expect(formatted).toContain( + "Inspect raw calls: workloads/pkg-vulns/tool-calls.json", + ); + }); + + it("reports missing artifacts for dry-run workloads without crashing", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-dry-run-test-")); + const report = buildRunReportFromMetadata(runDir, { + agent: "claude", + server: "local", + dryRun: true, + workloads: [{ id: "express-router", status: "dry-run" }], + }); + + expect(report.status).toBe("dry-run"); + expect(report.workloads[0]?.missingArtifacts).toContain("tool-calls.json"); + expect(formatRunReport(report)).toContain("express-router dry-run"); + }); + + it("does not read workload artifacts outside the run directory", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-safe-test-")); + const outsideDir = mkdtempSync(join(tmpdir(), "agent-eval-outside-test-")); + mkdirSync(join(runDir, "workloads", "safe"), { recursive: true }); + writeJson(join(outsideDir, "tool-calls.json"), [ + { tool: "pkg_vulns", status: "completed" }, + ]); + const report = buildRunReportFromMetadata(runDir, { + workloads: [{ id: "safe", status: "success", workloadDir: outsideDir }], + }); + + expect(report.workloads[0]?.toolCalls.rawCount).toBe(0); + expect(report.workloads[0]?.missingArtifacts).toContain("tool-calls.json"); + }); + + it("does not let workload ids traverse outside the run directory", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-id-safe-test-")); + const outsideDir = join(runDir, "..", "outside-workload"); + mkdirSync(outsideDir, { recursive: true }); + writeJson(join(outsideDir, "tool-calls.json"), [ + { tool: "pkg_vulns", status: "completed" }, + ]); + const report = buildRunReportFromMetadata(runDir, { + workloads: [{ id: "../outside-workload", status: "success" }], + }); + + expect(report.workloads[0]?.toolCalls.rawCount).toBe(0); + expect(report.workloads[0]?.warnings[0]).toContain( + "invalid workload id ignored", + ); + }); + + it("does not follow workload artifact symlinks outside the run directory", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-symlink-test-")); + const outsideDir = mkdtempSync( + join(tmpdir(), "agent-eval-symlink-outside-"), + ); + mkdirSync(join(runDir, "workloads"), { recursive: true }); + writeJson(join(outsideDir, "tool-calls.json"), [ + { tool: "pkg_vulns", status: "completed" }, + ]); + symlinkSync(outsideDir, join(runDir, "workloads", "symlinked")); + const report = buildRunReportFromMetadata(runDir, { + workloads: [{ id: "symlinked", status: "success" }], + }); + + expect(report.workloads[0]?.toolCalls.rawCount).toBe(0); + expect(report.workloads[0]?.warnings[0]).toContain( + "artifact path outside run directory ignored", + ); + }); + + it("warns only on actual-use self-report drift", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-drift-test-")); + const workloadDir = join(runDir, "workloads", "drift"); + mkdirSync(workloadDir, { recursive: true }); + writeJson(join(workloadDir, "tool-calls.json"), [ + { tool: "pkg_vulns", status: "completed" }, + ]); + writeJson(join(workloadDir, "final.json"), { + status: "success", + githitsUsefulness: "helped", + githitsUsefulnessReason: "useful", + confidence: "high", + expectedToolUse: ["pkg_info"], + unexpectedToolUse: ["docs_read"], + toolIssues: [], + instructionIssues: [], + }); + writeFileSync(join(workloadDir, "stderr.txt"), ""); + const report = buildRunReportFromMetadata(runDir, { + workloads: [{ id: "drift", status: "success", workloadDir }], + }); + + expect(report.workloads[0]?.warnings).toEqual([ + "self-report drift: unexpectedToolUse not present in raw calls: docs_read", + ]); + }); + + it("does not treat fallback descriptions as self-report drift", () => { + const runDir = mkdtempSync(join(tmpdir(), "agent-eval-fallback-test-")); + const workloadDir = join(runDir, "workloads", "fallback"); + mkdirSync(workloadDir, { recursive: true }); + writeJson(join(workloadDir, "tool-calls.json"), [ + { tool: "pkg_vulns", status: "completed" }, + ]); + writeJson(join(workloadDir, "final.json"), { + status: "success", + githitsUsefulness: "helped", + githitsUsefulnessReason: "useful", + confidence: "high", + expectedToolUse: [], + unexpectedToolUse: ["web search fallback for public corroboration"], + toolIssues: [], + instructionIssues: [], + }); + writeFileSync(join(workloadDir, "stderr.txt"), ""); + const report = buildRunReportFromMetadata(runDir, { + workloads: [{ id: "fallback", status: "success", workloadDir }], + }); + + expect(report.workloads[0]?.warnings).toEqual([]); + }); + + it("parses report CLI modes and rejects invalid combinations", () => { + expect(parseReportArgs(["/run"])).toEqual({ + mode: "report", + runDir: "/run", + }); + expect(parseReportArgs(["--json", "/run"])).toEqual({ + mode: "json", + runDir: "/run", + }); + expect(parseReportArgs(["--compare", "/before", "/after"])).toEqual({ + mode: "compare", + beforeRunDir: "/before", + afterRunDir: "/after", + }); + expect(() => parseReportArgs(["--json"])).toThrow( + "--json requires exactly one run directory", + ); + expect(() => parseReportArgs(["/one", "/two"])).toThrow( + "report mode accepts exactly one run directory", + ); + }); + + it("compares same-agent reports with aggregate status counts", () => { + const before = buildRunReportFromMetadata("/before", { + agent: "codex", + workloads: [{ id: "pkg-vulns", status: "success" }], + }); + const afterRunDir = createRunFixture(); + const after = buildRunReportFromMetadata( + afterRunDir, + JSON.parse(readFileSync(join(afterRunDir, "run.json"), "utf8")), + ); + const formatted = formatCompareReport(compareReports(before, after)); + + expect(formatted).toContain("pkg-vulns status unchanged success"); + expect(formatted).toContain("raw events 0 -> 2"); + expect(formatted).toContain("+pkg_vulns"); + }); + + it("degrades cross-agent comparisons to tool-name presence", () => { + const before = buildRunReportFromMetadata("/before", { + agent: "claude", + workloads: [{ id: "pkg-vulns", status: "success" }], + }); + const after = buildRunReportFromMetadata("/after", { + agent: "codex", + workloads: [{ id: "pkg-vulns", status: "success" }], + }); + const formatted = formatCompareReport(compareReports(before, after)); + + expect(formatted).toContain("cross-agent comparison"); + expect(formatted).not.toContain("raw events"); + }); + + it("fails fast on duplicate workload ids", () => { + expect(() => + assertUniqueWorkloadIds(["/a/tasks/pkg.md", "/b/other/pkg.md"]), + ).toThrow('Duplicate workload id "pkg"'); + }); + + it("keeps workload selection docs in sync", () => { + const repoRoot = process.cwd(); + const workloadsDir = join(repoRoot, "eval", "agentic", "workloads"); + const readme = readFileSync( + join(repoRoot, "eval", "agentic", "README.md"), + "utf8", + ); + const workloadFiles = readdirSync(workloadsDir) + .filter((file) => file.endsWith(".md") && file !== "REPORTING.md") + .sort(); + const missing = workloadFiles.filter( + (file) => !readme.includes(`\`${file}\``), + ); + + expect(missing).toEqual([]); + expect(workloadFiles.map((file) => basename(file))).toContain( + "package-overview-vulnerabilities.md", + ); + }); }); diff --git a/scripts/agent-eval.ts b/scripts/agent-eval.ts index d20f6c5c..c98da648 100644 --- a/scripts/agent-eval.ts +++ b/scripts/agent-eval.ts @@ -7,7 +7,14 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, dirname, isAbsolute, join, resolve } from "node:path"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { + assertUniqueWorkloadIds, + buildRunReportFromMetadata, + formatRunReport, + workloadIdFromPath, + writeReportJson, +} from "./agent-eval-report.ts"; type AgentName = "claude" | "codex"; type ServerMode = "local" | "published"; @@ -346,7 +353,7 @@ export function sanitizedEnvSummary( } function workloadId(workloadPath: string): string { - return basename(workloadPath).replace(/\.[^.]+$/, ""); + return workloadIdFromPath(workloadPath); } function writeJson(path: string, value: unknown): void { @@ -498,11 +505,15 @@ function extractClaudeToolCalls( if (record.type !== "tool_use" || typeof record.name !== "string") return []; const match = record.name.match(/^mcp__(.+)__(.+)$/); + if (!match) return []; + const server = match[1]; + const tool = match[2]; + if (!server || !tool) return []; return [ { agent: "claude", - server: match?.[1], - tool: match?.[2] ?? record.name, + server, + tool, status: "started", arguments: record.input, }, @@ -842,6 +853,7 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { `Schema not found: ${options.schemaPath}`, ); mkdirSync(options.outDir, { recursive: true }); + assertUniqueWorkloadIds(options.workloads); const env = buildEvalEnv(process.env); const secretValues = collectSecretValues(env); const mcpConfig = buildMcpConfig(options); @@ -874,7 +886,7 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { ); } - writeJson(join(options.outDir, "run.json"), { + const runMetadata = { agent: options.agent, server: options.server, publishedPackage: options.publishedPackage, @@ -888,7 +900,9 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { codexVersion: codex, env: sanitizedEnvSummary(env), workloads: workloadResults, - }); + }; + + writeJson(join(options.outDir, "run.json"), runMetadata); writeJson(join(options.outDir, "summary.json"), { status: workloadResults.some( @@ -908,6 +922,10 @@ export async function runAgentEval(options: AgentEvalOptions): Promise { }), ), }); + + const report = buildRunReportFromMetadata(options.outDir, runMetadata); + writeReportJson(options.outDir, report); + console.log(formatRunReport(report).trimEnd()); } async function main(): Promise { @@ -917,7 +935,6 @@ async function main(): Promise { options.outDir = resolve(repoRoot, options.outDir); } await runAgentEval(options); - console.log(`Agent eval artifacts written to ${options.outDir}`); } if (import.meta.main) {