diff --git a/.gitignore b/.gitignore index 9288a86f..ccbe837a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ coverage # Agents .claude +.agent-eval/ # logs logs diff --git a/AGENTS.md b/AGENTS.md index 9a90d004..89e06355 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ Philosophy: "If it is not tested, it is likely broken" - Use `bun test` for running tests - Use `bun run smoke:mcp` and `bun run smoke:cli` when changing MCP tools, CLI commands, shared formatters, auth/error envelopes, or MCP/CLI parity behavior. These are live smoke suites, not the normal unit suite; they must pass unauthenticated by validating auth handling, and provide deeper coverage when authenticated. +- Use `bun run agent:e2e` when changing MCP instructions, tool descriptions, or agent-facing tool behavior. This is a human/agent-driven qualitative eval, not a deterministic CI gate. Pick targeted workloads from `eval/agentic/README.md`; run both Claude and Codex for broad instruction changes when practical. Inspect `tool-calls.json` and `final.json` for actual tool use, `toolIssues`, `instructionIssues`, and usefulness, not just harness pass/fail. - Maintain smoke coverage when adding or changing user-facing tools/commands. Prefer structural UX assertions over brittle snapshots, and keep MCP `format: "json"` and CLI `--json` behavior aligned. - Keep tests async and isolated - Mock services at the interface level using factory functions diff --git a/eval/agentic/README.md b/eval/agentic/README.md new file mode 100644 index 00000000..36cbdcce --- /dev/null +++ b/eval/agentic/README.md @@ -0,0 +1,155 @@ +# Agentic Eval Harness + +This harness runs real coding agents against the real GitHits MCP server and +records whether the agent can use GitHits tools effectively from the MCP +server's own instructions and tool descriptions. + +It is not a smoke test. Smoke tests exercise CLI and MCP contracts directly. +Agentic evals exercise agent behavior end-to-end. + +This harness is intentionally human/agent-driven, not CI. Use it to understand +how MCP instruction or tool-description changes affect real agent behavior. Do +not treat a live agent pass/fail result as a deterministic regression test: model +behavior, backend indexing state, auth state, network conditions, and package +data can all change. The useful output is the artifact set, especially +`tool-calls.json`, `final.json`, `toolIssues`, `instructionIssues`, and the +agent's usefulness assessment. + +## What Is Under Test + +- Local mode starts the MCP server from this checkout with + `bun run --cwd dev mcp start`. +- Published mode starts the MCP server with `npx -y githits@latest mcp start` + by default. +- To evaluate MCP instruction changes, change branch/source and run local mode. + +Smoke tests are the right fit for CI gating. Agentic evals are the right fit for +qualitative review before/after instruction, tool-description, and agent-facing +UX changes. + +The harness must not add GitHits usage guidance through agent system prompts, +append prompts, alternate MCP instruction files, project instructions, or plugin +commands. Workload prompts may ask the agent to report what happened, but must +not tell the agent how to use GitHits. + +## Isolation + +Runs execute agents from an empty temporary workspace so repository-local files +such as `AGENTS.md`, `.mcp.json`, commands, skills, and plugin payloads do not +contaminate results. The harness keeps the user's normal Claude/GitHits auth +environment so human-driven keychain/OAuth sessions continue to work. + +GitHits authentication follows normal local behavior. Keychain-backed human +login should work by default. Automation can use `GITHITS_API_TOKEN`. + +## Usage + +```bash +bun run agent:e2e --server local --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --server published --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --agent codex --server local --workload eval/agentic/workloads/express-router.md +``` + +Useful options: + +```bash +--agent Agent to run, default `claude` +--dry-run Generate artifacts without invoking the agent +--out Output directory, default `.agent-eval/runs/` +--timeout Per-workload timeout, default 300 +--published-package Published package spec, default `githits@latest` +--workload Repeatable workload path +``` + +Normal GitHits backend overrides are passed through when set: + +- `GITHITS_API_URL` +- `GITHITS_MCP_URL` +- `GITHITS_CODE_NAV_URL` +- `PKGSEER_URL` +- `GITHITS_API_TOKEN` +- `GITHITS_AUTH_STORAGE` + +Secret-like values are redacted in run metadata. + +## Workloads + +Workloads are Markdown prompts. They should contain: + +- The task. + +The harness appends `eval/agentic/workloads/REPORTING.md` to every workload so +all agents return the same structured report. Workload files should not repeat +that reporting contract. + +They should not contain instructions such as "call `search` first" or "use +`code_read` after `search`". That guidance must come from the MCP server. + +### Workload Selection + +Use targeted workloads when a change affects a specific tool family. Use both +Claude and Codex for instruction/tool-description changes when practical; use at +least one agent for quick iteration. + +| Affected Area | Workload | +|---|---| +| Core global examples, `get_example`, `search_language`, `feedback` | `global-example.md` | +| Unified `search` / `search_status` behavior | `unified-search-investigation.md` | +| Package overview or vulnerability UX, `pkg_info`, `pkg_vulns` | `package-overview-vulnerabilities.md` | +| Dependency graph UX, `pkg_deps` | `package-dependencies.md` | +| Release notes UX, `pkg_changelog` | `package-changelog.md` | +| Documentation browsing, `docs_list`, `docs_read` | `docs-discovery.md` | +| File listing / file read UX, `code_files`, `code_read` | `code-file-navigation.md` | +| Deterministic source search UX, `code_grep` | `code-grep-investigation.md` | +| Multi-tool code navigation strategy and MCP instructions | `express-router.md` | + +For broad MCP instruction edits, run at least: + +```bash +bun run agent:e2e --agent claude --server local --workload eval/agentic/workloads/express-router.md +bun run agent:e2e --agent codex --server local --workload eval/agentic/workloads/express-router.md +``` + +For tool-specific edits, add the workload from the table. Compare +`tool-calls.json` plus the final JSON's `toolIssues`, `instructionIssues`, and +`githitsUsefulnessReason` across branches or against a published run. + +### Current Baseline Observations + +The initial local baseline ran all workloads against Claude and Codex. Expected +tool families were exercised after tightening the shared reporting contract. +Notable findings to keep in mind when evaluating future changes: + +- `global-example.md` exercises `get_example`; agents may combine it with docs, + source, or package metadata when they need stronger canonical evidence. +- `code-grep-investigation.md` surfaced a real `code_grep` regex limitation for + short/stopword-heavy patterns; literal grep is the reliable path for that + workload. +- `unified-search-investigation.md` intentionally exposes `search` warnings and + follow-up needs. Agents should inspect warnings and use `code_read`, + `docs_read`, or `code_grep` when top search hits are incomplete/noisy. +- Codex sometimes reports a tool as unavailable until it performs additional + tool discovery. Use `tool-calls.json` to distinguish actual unavailable tools + from delayed discovery. +- `tool-calls.json` is the source of truth for tool usage. The final JSON is for + the agent's assessment of clarity, issues, and usefulness. + +## Artifacts + +Each run writes: + +- `run.json` with command, git, environment, and timing metadata. +- One workload directory per workload with `prompt.md`, `mcp.json`, + `stdout.json`, `stderr.txt`, `tool-calls.json`, and `final.json` when parsing + succeeds. + +Claude is launched with `--permission-mode bypassPermissions` so non-interactive +evals can exercise configured MCP tools without a human approval prompt, and +`--disable-slash-commands` to reduce plugin/skill contamination while preserving +normal human auth. Codex is launched with per-run `-c` MCP config overrides, +`--ignore-rules`, and a read-only sandbox so it can use normal human auth +without mutating global MCP configuration. + +Malformed final JSON, schema mismatches, Claude failures, and timeouts are +harness failures. Raw stdout and stderr are preserved for diagnosis with known +secret values redacted. diff --git a/eval/agentic/result.schema.json b/eval/agentic/result.schema.json new file mode 100644 index 00000000..09453191 --- /dev/null +++ b/eval/agentic/result.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "answer", + "toolIssues", + "expectedToolUse", + "unexpectedToolUse", + "instructionIssues", + "githitsUsefulness", + "githitsUsefulnessReason", + "confidence" + ], + "properties": { + "status": { + "type": "string", + "enum": ["success", "failure", "inconclusive"] + }, + "answer": { + "type": "string" + }, + "toolIssues": { + "type": "array", + "items": { + "type": "string" + } + }, + "expectedToolUse": { + "type": "array", + "items": { "type": "string" } + }, + "unexpectedToolUse": { + "type": "array", + "items": { "type": "string" } + }, + "instructionIssues": { + "type": "array", + "items": { "type": "string" } + }, + "githitsUsefulness": { + "type": "string", + "enum": ["helped", "hurt", "unused", "unclear"] + }, + "githitsUsefulnessReason": { + "type": "string" + }, + "confidence": { + "type": "string", + "enum": ["high", "medium", "low"] + } + } +} diff --git a/eval/agentic/workloads/REPORTING.md b/eval/agentic/workloads/REPORTING.md new file mode 100644 index 00000000..27b27e0c --- /dev/null +++ b/eval/agentic/workloads/REPORTING.md @@ -0,0 +1,16 @@ +Return only valid JSON matching the provided schema. Do not include markdown, +prose, code fences, or any text outside the JSON object. Include: + +- `status`: `success`, `failure`, or `inconclusive`. +- `answer`: your final answer with evidence. +- `toolIssues`: string descriptions of GitHits tool calls that failed or + returned unclear output, if any. +- `expectedToolUse`: string list of GitHits tools you expected to use for this + workload. Use `[]` if none. +- `unexpectedToolUse`: string list of tools or fallback paths you used but did + not expect to need. Use `[]` if none. +- `instructionIssues`: MCP guidance that was unclear, missing, or contradicted + observed tool behavior, if any. +- `githitsUsefulness`: `helped`, `hurt`, `unused`, or `unclear`. +- `githitsUsefulnessReason`: why you chose that usefulness value. +- `confidence`: `high`, `medium`, or `low`. diff --git a/eval/agentic/workloads/code-file-navigation.md b/eval/agentic/workloads/code-file-navigation.md new file mode 100644 index 00000000..452bd7ae --- /dev/null +++ b/eval/agentic/workloads/code-file-navigation.md @@ -0,0 +1,5 @@ +# Workload: Source File Navigation + +You are debugging how `npm:express` creates an application object. Identify the +source file that exports the application factory, read the relevant section, and +explain how request and response prototypes are attached. diff --git a/eval/agentic/workloads/code-grep-investigation.md b/eval/agentic/workloads/code-grep-investigation.md new file mode 100644 index 00000000..83fae17e --- /dev/null +++ b/eval/agentic/workloads/code-grep-investigation.md @@ -0,0 +1,5 @@ +# Workload: Deterministic Source Grep + +You are investigating where Express imports the standalone `router` package. +Find the exact files and lines in `npm:express` that import it, then summarize +what each import site does. diff --git a/eval/agentic/workloads/docs-discovery.md b/eval/agentic/workloads/docs-discovery.md new file mode 100644 index 00000000..ad99cdb2 --- /dev/null +++ b/eval/agentic/workloads/docs-discovery.md @@ -0,0 +1,5 @@ +# Workload: Documentation Discovery + +You need to learn how Express documents routing behavior. Find relevant package +documentation for `npm:express`, read enough to summarize how routes are defined, +and include source references for the documentation you relied on. diff --git a/eval/agentic/workloads/express-router.md b/eval/agentic/workloads/express-router.md new file mode 100644 index 00000000..2a85a8cc --- /dev/null +++ b/eval/agentic/workloads/express-router.md @@ -0,0 +1,5 @@ +# Workload: Express Router Investigation + +You are investigating how Express implements routing. Determine where the main +Router implementation lives in the Express package and summarize the evidence +you found. diff --git a/eval/agentic/workloads/global-example.md b/eval/agentic/workloads/global-example.md new file mode 100644 index 00000000..443cb908 --- /dev/null +++ b/eval/agentic/workloads/global-example.md @@ -0,0 +1,5 @@ +# Workload: Canonical Example Discovery + +You need a current, canonical TypeScript example for validating user input with +Zod and returning typed parsed data. Find a suitable example or pattern from +open source and explain how you would adapt it in a small CLI command. diff --git a/eval/agentic/workloads/package-changelog.md b/eval/agentic/workloads/package-changelog.md new file mode 100644 index 00000000..d5f6ad71 --- /dev/null +++ b/eval/agentic/workloads/package-changelog.md @@ -0,0 +1,6 @@ +# Workload: Release History Assessment + +You are assessing recent `npm:express` releases before upgrading. Summarize the +most recent release notes and call out anything that looks security-relevant, +breaking, or operationally important. If full bodies are unnecessary, keep the +answer concise and say why. diff --git a/eval/agentic/workloads/package-dependencies.md b/eval/agentic/workloads/package-dependencies.md new file mode 100644 index 00000000..173c6f91 --- /dev/null +++ b/eval/agentic/workloads/package-dependencies.md @@ -0,0 +1,6 @@ +# Workload: Dependency Graph Assessment + +You are reviewing `npm:express` for dependency footprint risk. Explain its +runtime dependency shape, whether optional or non-runtime dependency groups are +relevant to the adoption decision, and whether transitive dependency details +change your recommendation. diff --git a/eval/agentic/workloads/package-overview-vulnerabilities.md b/eval/agentic/workloads/package-overview-vulnerabilities.md new file mode 100644 index 00000000..03c61c4d --- /dev/null +++ b/eval/agentic/workloads/package-overview-vulnerabilities.md @@ -0,0 +1,6 @@ +# Workload: Package Overview And Vulnerability Check + +You are evaluating whether `npm:express` is a reasonable dependency for a new +Node.js service. Summarize the latest package metadata and whether known +vulnerabilities should block adoption. Include version, license, repository +health signals if available, active advisory status, and any caveats. diff --git a/eval/agentic/workloads/unified-search-investigation.md b/eval/agentic/workloads/unified-search-investigation.md new file mode 100644 index 00000000..1c15e12c --- /dev/null +++ b/eval/agentic/workloads/unified-search-investigation.md @@ -0,0 +1,6 @@ +# Workload: Cross-Surface Search Investigation + +You are comparing how Express exposes `Router` across package metadata, source, +and public API references. Investigate `npm:express` and explain whether the +search results were enough by themselves or whether you needed follow-up reads. +Report any warnings, incomplete results, or follow-up status checks you saw. diff --git a/package.json b/package.json index f5d66c9e..5043d70d 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "inspector": "npx @modelcontextprotocol/inspector bun run dev mcp", "smoke:cli": "bun run scripts/cli-smoke.ts", "smoke:mcp": "bun run scripts/mcp-smoke.ts", + "agent:e2e": "bun run scripts/agent-eval.ts", "test": "bun test", "typecheck": "tsc", "format": "biome format --write .", diff --git a/scripts/agent-eval.test.ts b/scripts/agent-eval.test.ts new file mode 100644 index 00000000..3283d3d2 --- /dev/null +++ b/scripts/agent-eval.test.ts @@ -0,0 +1,219 @@ +import { describe, expect, it } from "bun:test"; +import { + buildCodexConfig, + buildCodexConfigArgs, + buildEvalEnv, + buildMcpConfig, + collectSecretValues, + extractToolCalls, + isValidAgentReport, + parseArgs, + redactText, + sanitizedEnvSummary, +} from "./agent-eval.ts"; + +describe("agent eval harness", () => { + it("builds local MCP config with explicit repo cwd", () => { + const config = buildMcpConfig({ + server: "local", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@latest", + }); + + expect(config.mcpServers.githits).toEqual({ + command: "bun", + args: ["run", "--cwd", "/repo/githits-cli", "dev", "mcp", "start"], + }); + }); + + it("builds published MCP config with pinned package spec", () => { + const config = buildMcpConfig({ + server: "published", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@0.4.2", + }); + + expect(config.mcpServers.githits).toEqual({ + command: "npx", + args: ["-y", "githits@0.4.2", "mcp", "start"], + }); + }); + + it("builds Codex TOML config from the same MCP command", () => { + expect( + buildCodexConfig({ + server: "local", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@latest", + }), + ).toBe( + '[mcp_servers.githits]\ncommand = "bun"\nargs = ["run","--cwd","/repo/githits-cli","dev","mcp","start"]\n', + ); + }); + + it("builds Codex config override args", () => { + expect( + buildCodexConfigArgs({ + server: "published", + repoRoot: "/repo/githits-cli", + publishedPackage: "githits@0.4.2", + }), + ).toEqual([ + "-c", + 'mcp_servers.githits.command="npx"', + "-c", + 'mcp_servers.githits.args=["-y","githits@0.4.2","mcp","start"]', + ]); + }); + + it("preserves normal Claude and GitHits auth environment while filtering unrelated vars", () => { + const env = buildEvalEnv({ + PATH: "/bin", + HOME: "/real-home", + RANDOM_SECRET: "should-not-pass", + GITHITS_AUTH_STORAGE: "keychain", + GITHITS_API_TOKEN: "secret-token", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + }); + + expect(env.HOME).toBe("/real-home"); + expect(env.GITHITS_AUTH_STORAGE).toBe("keychain"); + expect(env.GITHITS_API_TOKEN).toBe("secret-token"); + expect(env.GITHITS_CODE_NAV_URL).toBe("http://localhost:7070"); + expect(env.RANDOM_SECRET).toBeUndefined(); + }); + + it("redacts secret values from environment summary", () => { + const summary = sanitizedEnvSummary({ + HOME: "/tmp/eval-home", + USERPROFILE: "/tmp/eval-home", + XDG_CONFIG_HOME: "/tmp/eval-home/.config", + APPDATA: "/tmp/eval-home/AppData/Roaming", + GITHITS_API_TOKEN: "secret-token", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + GITHITS_AUTH_STORAGE: "keychain", + }); + + expect(summary.GITHITS_API_TOKEN).toBe(""); + expect(summary.GITHITS_CODE_NAV_URL).toBe("http://localhost:7070"); + expect(summary.GITHITS_AUTH_STORAGE).toBe("keychain"); + expect(summary.HOME).toBe(""); + }); + + it("parses repeatable workloads and dry-run options", () => { + const options = parseArgs( + [ + "--agent", + "codex", + "--server", + "published", + "--published-package", + "githits@0.4.2", + "--workload", + "eval/agentic/workloads/express-router.md", + "--timeout", + "12", + "--dry-run", + ], + "/repo/githits-cli", + ); + + expect(options.agent).toBe("codex"); + expect(options.server).toBe("published"); + expect(options.publishedPackage).toBe("githits@0.4.2"); + expect(options.timeoutSeconds).toBe(12); + expect(options.dryRun).toBe(true); + expect(options.workloads).toEqual([ + "/repo/githits-cli/eval/agentic/workloads/express-router.md", + ]); + }); + + it("redacts secret values from persisted text", () => { + const secrets = collectSecretValues({ + GITHITS_API_TOKEN: "secret-token-value", + ANTHROPIC_API_KEY: "anthropic-secret-value", + GITHITS_CODE_NAV_URL: "http://localhost:7070", + }); + + expect( + redactText( + "token=secret-token-value key=anthropic-secret-value", + secrets, + ), + ).toBe("token= key="); + }); + + it("validates final agent report shape", () => { + expect( + isValidAgentReport({ + status: "success", + answer: "Router lives in lib/router/index.js.", + toolIssues: [], + expectedToolUse: ["code_read"], + unexpectedToolUse: [], + instructionIssues: [], + githitsUsefulness: "helped", + githitsUsefulnessReason: "It located source evidence.", + confidence: "high", + }), + ).toBe(true); + + expect( + isValidAgentReport({ status: "success", answer: "missing fields" }), + ).toBe(false); + }); + + it("extracts Codex MCP tool calls from JSONL events", () => { + const calls = extractToolCalls( + `${JSON.stringify({ + type: "item.completed", + item: { + type: "mcp_tool_call", + server: "githits", + tool: "code_read", + status: "completed", + arguments: { path: "index.js" }, + }, + })}\n`, + "codex", + ); + + expect(calls).toEqual([ + { + agent: "codex", + server: "githits", + tool: "code_read", + status: "completed", + arguments: { path: "index.js" }, + }, + ]); + }); + + it("extracts Claude MCP tool calls from verbose stream events", () => { + const calls = extractToolCalls( + `${JSON.stringify({ + type: "assistant", + message: { + content: [ + { + type: "tool_use", + name: "mcp__githits__pkg_info", + input: { registry: "npm", package_name: "express" }, + }, + ], + }, + })}\n`, + "claude", + ); + + expect(calls).toEqual([ + { + agent: "claude", + server: "githits", + tool: "pkg_info", + status: "started", + arguments: { registry: "npm", package_name: "express" }, + }, + ]); + }); +}); diff --git a/scripts/agent-eval.ts b/scripts/agent-eval.ts new file mode 100644 index 00000000..d20f6c5c --- /dev/null +++ b/scripts/agent-eval.ts @@ -0,0 +1,929 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; + +type AgentName = "claude" | "codex"; +type ServerMode = "local" | "published"; +type RunStatus = "dry-run" | "success" | "failed" | "timeout"; + +export interface AgentEvalOptions { + agent: AgentName; + server: ServerMode; + workloads: string[]; + outDir: string; + timeoutSeconds: number; + publishedPackage: string; + dryRun: boolean; + repoRoot: string; + schemaPath: string; + reportingPath: string; +} + +export interface McpServerConfig { + mcpServers: { + githits: { + command: string; + args: string[]; + env?: Record; + }; + }; +} + +interface WorkloadRunMetadata { + id: string; + path: string; + status: RunStatus; + exitCode?: number; + durationMs?: number; + timedOut?: boolean; + command: string[]; + workspaceDir: string; + workloadDir: string; + toolCallCount?: number; +} + +interface ExtractedToolCall { + agent: AgentName; + server?: string; + tool: string; + status?: string; + arguments?: unknown; + error?: unknown; +} + +const PASSTHROUGH_ENV_KEYS = [ + "GITHITS_API_URL", + "GITHITS_MCP_URL", + "GITHITS_CODE_NAV_URL", + "PKGSEER_URL", + "GITHITS_API_TOKEN", + "GITHITS_AUTH_STORAGE", +] as const; + +const BASE_ENV_KEYS = [ + "PATH", + "HOME", + "USERPROFILE", + "XDG_CONFIG_HOME", + "APPDATA", + "USER", + "LOGNAME", + "USERNAME", + "SystemRoot", + "WINDIR", + "COMSPEC", + "TMPDIR", + "TMP", + "TEMP", + "SHELL", + "LANG", + "LC_ALL", + "SECURITYSESSIONID", + "XPC_FLAGS", + "XPC_SERVICE_NAME", + "__CF_USER_TEXT_ENCODING", + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "GOOGLE_APPLICATION_CREDENTIALS", +] as const; + +const SECRET_PATTERN = /(TOKEN|API_KEY|SECRET|PASSWORD|CREDENTIAL)/i; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function timestamp(): string { + return new Date().toISOString().replace(/[:.]/g, "-"); +} + +function defaultOutDir(repoRoot: string): string { + return join(repoRoot, ".agent-eval", "runs", timestamp()); +} + +function parsePositiveInteger(value: string, flag: string): number { + const parsed = Number(value); + assert( + Number.isInteger(parsed) && parsed > 0, + `${flag} must be a positive integer`, + ); + return parsed; +} + +export function parseArgs( + argv: string[], + repoRoot = process.cwd(), +): AgentEvalOptions { + const options: AgentEvalOptions = { + agent: "claude", + server: "local", + workloads: [], + outDir: defaultOutDir(repoRoot), + timeoutSeconds: 300, + publishedPackage: "githits@latest", + dryRun: false, + repoRoot, + schemaPath: join(repoRoot, "eval", "agentic", "result.schema.json"), + reportingPath: join( + repoRoot, + "eval", + "agentic", + "workloads", + "REPORTING.md", + ), + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case "--agent": { + const value = argv[++i]; + assert( + value === "claude" || value === "codex", + "--agent must be claude or codex", + ); + options.agent = value; + break; + } + case "--server": { + const value = argv[++i]; + assert( + value === "local" || value === "published", + "--server must be local or published", + ); + options.server = value; + break; + } + case "--workload": { + const value = argv[++i]; + assert(value, "--workload requires a path"); + options.workloads.push(value); + break; + } + case "--out": { + const value = argv[++i]; + assert(value, "--out requires a path"); + options.outDir = resolve(repoRoot, value); + break; + } + case "--timeout": { + const value = argv[++i]; + assert(value, "--timeout requires seconds"); + options.timeoutSeconds = parsePositiveInteger(value, "--timeout"); + break; + } + case "--published-package": { + const value = argv[++i]; + assert(value, "--published-package requires a package spec"); + options.publishedPackage = value; + break; + } + case "--schema": { + const value = argv[++i]; + assert(value, "--schema requires a path"); + options.schemaPath = resolve(repoRoot, value); + break; + } + case "--reporting": { + const value = argv[++i]; + assert(value, "--reporting requires a path"); + options.reportingPath = resolve(repoRoot, value); + break; + } + case "--dry-run": + options.dryRun = true; + break; + case "--help": + case "-h": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + if (options.workloads.length === 0) { + options.workloads.push( + join(repoRoot, "eval", "agentic", "workloads", "express-router.md"), + ); + } + options.workloads = options.workloads.map((path) => resolve(repoRoot, path)); + return options; +} + +function printHelp(): void { + console.log(`Usage: bun run agent:e2e [options] + +Options: + --agent claude|codex Agent to run (default: claude) + --server local|published MCP server mode (default: local) + --workload Workload markdown path, repeatable + --out Output directory + --timeout Per-workload timeout (default: 300) + --published-package Package for published mode (default: githits@latest) + --schema Result JSON schema path + --reporting Reporting contract markdown path + --dry-run Generate artifacts without invoking Claude +`); +} + +export function buildMcpConfig( + options: Pick, +): McpServerConfig { + const command = buildMcpCommand(options); + return { + mcpServers: { + githits: command, + }, + }; +} + +function buildMcpCommand( + options: Pick, +): McpServerConfig["mcpServers"]["githits"] { + if (options.server === "local") { + return { + command: "bun", + args: ["run", "--cwd", options.repoRoot, "dev", "mcp", "start"], + }; + } + + return { + command: "npx", + args: ["-y", options.publishedPackage, "mcp", "start"], + }; +} + +export function buildCodexConfig( + options: Pick, +): string { + const command = buildMcpCommand(options); + return [ + "[mcp_servers.githits]", + `command = ${JSON.stringify(command.command)}`, + `args = ${JSON.stringify(command.args)}`, + "", + ].join("\n"); +} + +export function buildCodexConfigArgs( + options: Pick, +): string[] { + const command = buildMcpCommand(options); + return [ + "-c", + `mcp_servers.githits.command=${JSON.stringify(command.command)}`, + "-c", + `mcp_servers.githits.args=${JSON.stringify(command.args)}`, + ]; +} + +export function buildEvalEnv( + baseEnv: NodeJS.ProcessEnv, +): Record { + const env: Record = {}; + + for (const key of [...BASE_ENV_KEYS, ...PASSTHROUGH_ENV_KEYS]) { + const value = baseEnv[key]; + if (value !== undefined) { + env[key] = value; + } + } + + env.NO_COLOR = "1"; + return env; +} + +export function collectSecretValues(env: Record): string[] { + const values = new Set(); + for (const [key, value] of Object.entries(env)) { + if (SECRET_PATTERN.test(key) && value.length >= 8) { + values.add(value); + } + } + return [...values].sort((a, b) => b.length - a.length); +} + +export function redactText(text: string, secretValues: string[]): string { + let redacted = text; + for (const secret of secretValues) { + redacted = redacted.split(secret).join(""); + } + return redacted; +} + +export function sanitizedEnvSummary( + env: Record, +): Record { + const summary: Record = {}; + for (const key of PASSTHROUGH_ENV_KEYS) { + const value = env[key]; + if (value === undefined) { + continue; + } + summary[key] = SECRET_PATTERN.test(key) ? "" : value; + } + if (env.HOME) summary.HOME = ""; + if (env.USERPROFILE) summary.USERPROFILE = ""; + if (env.XDG_CONFIG_HOME) summary.XDG_CONFIG_HOME = ""; + if (env.APPDATA) summary.APPDATA = ""; + return summary; +} + +function workloadId(workloadPath: string): string { + return basename(workloadPath).replace(/\.[^.]+$/, ""); +} + +function writeJson(path: string, value: unknown): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function redactValue(value: unknown, secretValues: string[]): unknown { + if (typeof value === "string") { + return redactText(value, secretValues); + } + if (Array.isArray(value)) { + return value.map((item) => redactValue(item, secretValues)); + } + if (value !== null && typeof value === "object") { + const redacted: Record = {}; + for (const [key, item] of Object.entries(value)) { + redacted[key] = redactValue(item, secretValues); + } + return redacted; + } + return value; +} + +async function commandOutput( + command: string, + args: string[], + cwd: string, +): Promise { + try { + const proc = Bun.spawn([command, ...args], { + cwd, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + proc.exited, + ]); + if (exitCode !== 0) { + return undefined; + } + return stdout.trim() || undefined; + } catch { + return undefined; + } +} + +async function collectGitMetadata( + repoRoot: string, +): Promise> { + const [branch, sha] = await Promise.all([ + commandOutput("git", ["branch", "--show-current"], repoRoot), + commandOutput("git", ["rev-parse", "HEAD"], repoRoot), + ]); + return { branch, sha }; +} + +async function claudeVersion(): Promise { + return commandOutput("claude", ["--version"], process.cwd()); +} + +async function codexVersion(): Promise { + return commandOutput("codex", ["--version"], process.cwd()); +} + +async function assertClaudeAvailable(): Promise { + const version = await claudeVersion(); + assert( + version, + "claude CLI not found or not executable. Install Claude Code before running live agent evals.", + ); +} + +async function assertCodexAvailable(): Promise { + const version = await codexVersion(); + assert( + version, + "codex CLI not found or not executable. Install Codex before running live agent evals.", + ); +} + +function extractFinalJson(stdout: string): unknown | undefined { + const lines = stdout.split("\n").filter((line) => line.trim().length > 0); + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + if (!line) continue; + try { + const event = JSON.parse(line) as Record; + const message = event.message; + if (message !== null && typeof message === "object") { + const text = extractTextFromContent( + (message as Record).content, + ); + if (text) { + const parsed = parseJsonFromText(text); + if (parsed !== undefined) return parsed; + } + } + const candidates = [ + event.result, + event.message, + event.content, + event.text, + ]; + for (const candidate of candidates) { + if (typeof candidate === "string") { + const parsed = parseJsonFromText(candidate); + if (parsed !== undefined) return parsed; + } + } + if (event.status || event.answer || event.githitsToolsUsed) { + return event; + } + } catch { + try { + return JSON.parse(line); + } catch { + // Continue scanning older lines. + } + } + } + return undefined; +} + +function extractTextFromContent(content: unknown): string | undefined { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return undefined; + const parts = content.flatMap((item): string[] => { + if (item === null || typeof item !== "object") return []; + const record = item as Record; + return record.type === "text" && typeof record.text === "string" + ? [record.text] + : []; + }); + return parts.length > 0 ? parts.join("\n") : undefined; +} + +function extractClaudeToolCalls( + event: Record, +): ExtractedToolCall[] { + const message = event.message; + if (message === null || typeof message !== "object") return []; + const content = (message as Record).content; + if (!Array.isArray(content)) return []; + return content.flatMap((item): ExtractedToolCall[] => { + if (item === null || typeof item !== "object") return []; + const record = item as Record; + if (record.type !== "tool_use" || typeof record.name !== "string") + return []; + const match = record.name.match(/^mcp__(.+)__(.+)$/); + return [ + { + agent: "claude", + server: match?.[1], + tool: match?.[2] ?? record.name, + status: "started", + arguments: record.input, + }, + ]; + }); +} + +function extractCodexToolCall( + event: Record, +): ExtractedToolCall | undefined { + const item = event.item; + if (item === null || typeof item !== "object") return undefined; + const record = item as Record; + if (record.type !== "mcp_tool_call" || typeof record.tool !== "string") { + return undefined; + } + return { + agent: "codex", + server: typeof record.server === "string" ? record.server : undefined, + tool: record.tool, + status: typeof record.status === "string" ? record.status : undefined, + arguments: record.arguments, + error: record.error, + }; +} + +export function extractToolCalls( + stdout: string, + agent: AgentName, +): ExtractedToolCall[] { + const calls: ExtractedToolCall[] = []; + for (const line of stdout.split("\n")) { + if (line.trim().length === 0) continue; + try { + const event = JSON.parse(line) as Record; + if (agent === "claude") { + calls.push(...extractClaudeToolCalls(event)); + } else { + const call = extractCodexToolCall(event); + if (call) calls.push(call); + } + } catch { + // Ignore non-JSON lines. + } + } + return calls; +} + +function parseJsonFromText(text: string): unknown | undefined { + const trimmed = text.trim(); + try { + return JSON.parse(trimmed); + } catch { + // Continue to fenced JSON extraction. + } + + const fence = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (!fence?.[1]) { + return undefined; + } + try { + return JSON.parse(fence[1].trim()); + } catch { + return undefined; + } +} + +function isStringArray(value: unknown): value is string[] { + return ( + Array.isArray(value) && value.every((item) => typeof item === "string") + ); +} + +function isToolIssueArray(value: unknown): boolean { + return ( + Array.isArray(value) && + value.every( + (item) => + typeof item === "string" || + (item !== null && + typeof item === "object" && + typeof (item as Record).tool === "string" && + typeof (item as Record).issue === "string"), + ) + ); +} + +function isOptionalStringArray(value: unknown): boolean { + return value === undefined || isStringArray(value); +} + +export function isValidAgentReport(value: unknown): boolean { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + const report = value as Record; + const allowedKeys = new Set([ + "status", + "answer", + "toolIssues", + "instructionIssues", + "githitsUsefulness", + "githitsUsefulnessReason", + "confidence", + "expectedToolUse", + "unexpectedToolUse", + ]); + if (Object.keys(report).some((key) => !allowedKeys.has(key))) { + return false; + } + return ( + (report.status === "success" || + report.status === "failure" || + report.status === "inconclusive") && + typeof report.answer === "string" && + isToolIssueArray(report.toolIssues) && + isStringArray(report.instructionIssues) && + isOptionalStringArray(report.expectedToolUse) && + isOptionalStringArray(report.unexpectedToolUse) && + (report.githitsUsefulness === "helped" || + report.githitsUsefulness === "hurt" || + report.githitsUsefulness === "unused" || + report.githitsUsefulness === "unclear") && + typeof report.githitsUsefulnessReason === "string" && + (report.confidence === "high" || + report.confidence === "medium" || + report.confidence === "low") + ); +} + +async function runWithTimeout( + command: string[], + cwd: string, + env: Record, + timeoutSeconds: number, +): Promise<{ + stdout: string; + stderr: string; + exitCode?: number; + timedOut: boolean; +}> { + const proc = Bun.spawn(command, { + cwd, + env, + stdout: "pipe", + stderr: "pipe", + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill("SIGTERM"); + setTimeout(() => proc.kill("SIGKILL"), 2_000).unref?.(); + }, timeoutSeconds * 1_000); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited.catch(() => undefined), + ]); + clearTimeout(timer); + return { stdout, stderr, exitCode, timedOut }; +} + +function buildClaudeCommand(prompt: string, mcpConfigPath: string): string[] { + return [ + "claude", + "-p", + prompt, + "--mcp-config", + mcpConfigPath, + "--strict-mcp-config", + "--disable-slash-commands", + "--output-format", + "stream-json", + "--verbose", + "--permission-mode", + "bypassPermissions", + "--no-session-persistence", + ]; +} + +function buildCodexCommand( + prompt: string, + workspaceDir: string, + finalMessagePath: string, + schemaPath: string, + options: Pick, +): string[] { + return [ + "codex", + "exec", + ...buildCodexConfigArgs(options), + "--cd", + workspaceDir, + "--skip-git-repo-check", + "--ephemeral", + "--json", + "--output-last-message", + finalMessagePath, + "--output-schema", + schemaPath, + "--sandbox", + "read-only", + "--ignore-rules", + prompt, + ]; +} + +async function runWorkload( + options: AgentEvalOptions, + workloadPath: string, + runDir: string, + env: Record, + mcpConfig: McpServerConfig, + secretValues: string[], +): Promise { + assert(existsSync(workloadPath), `Workload not found: ${workloadPath}`); + assert( + existsSync(options.reportingPath), + `Reporting contract not found: ${options.reportingPath}`, + ); + const id = workloadId(workloadPath); + const workloadDir = join(runDir, "workloads", id); + const workspaceDir = mkdtempSync( + join(tmpdir(), "githits-agent-eval-workspace-"), + ); + mkdirSync(workloadDir, { recursive: true }); + + const workloadPrompt = readFileSync(workloadPath, "utf8").trimEnd(); + const reportingPrompt = readFileSync(options.reportingPath, "utf8").trim(); + const prompt = `${workloadPrompt}\n\n${reportingPrompt}\n`; + const mcpConfigPath = join(workloadDir, "mcp.json"); + const codexConfigPath = join(workloadDir, "codex-config.toml"); + const codexFinalPath = join(workloadDir, "codex-final.txt"); + writeFileSync(join(workloadDir, "prompt.md"), prompt); + writeJson(mcpConfigPath, mcpConfig); + writeFileSync(codexConfigPath, buildCodexConfig(options)); + + const command = + options.agent === "claude" + ? buildClaudeCommand(prompt, mcpConfigPath) + : buildCodexCommand( + prompt, + workspaceDir, + codexFinalPath, + options.schemaPath, + options, + ); + const workloadEnv = { ...env }; + const metadataBase = { + id, + path: workloadPath, + command, + workspaceDir, + workloadDir, + }; + + try { + if (options.dryRun) { + writeJson(join(workloadDir, "dry-run.json"), metadataBase); + return { ...metadataBase, status: "dry-run" }; + } + + const started = Date.now(); + const result = await runWithTimeout( + command, + workspaceDir, + workloadEnv, + options.timeoutSeconds, + ); + const durationMs = Date.now() - started; + + writeFileSync( + join(workloadDir, "stdout.json"), + redactText(result.stdout, secretValues), + ); + writeFileSync( + join(workloadDir, "stderr.txt"), + redactText(result.stderr, secretValues), + ); + + const toolCalls = extractToolCalls(result.stdout, options.agent); + writeJson( + join(workloadDir, "tool-calls.json"), + redactValue(toolCalls, secretValues), + ); + + const finalJson = extractFinalJson(result.stdout); + const codexFinalText = existsSync(codexFinalPath) + ? readFileSync(codexFinalPath, "utf8") + : undefined; + if (codexFinalText !== undefined) { + writeFileSync( + join(workloadDir, "codex-final.txt"), + redactText(codexFinalText, secretValues), + ); + } + const reportJson = + options.agent === "codex" && codexFinalText !== undefined + ? parseJsonFromText(codexFinalText) + : finalJson; + const validFinalJson = isValidAgentReport(reportJson); + if (validFinalJson) { + writeJson( + join(workloadDir, "final.json"), + redactValue(reportJson, secretValues), + ); + } else if (reportJson !== undefined) { + writeJson( + join(workloadDir, "invalid-final.json"), + redactValue(reportJson, secretValues), + ); + } + + const status: RunStatus = result.timedOut + ? "timeout" + : result.exitCode === 0 && validFinalJson + ? "success" + : "failed"; + + return { + ...metadataBase, + status, + exitCode: result.exitCode, + durationMs, + timedOut: result.timedOut, + toolCallCount: toolCalls.length, + }; + } finally { + rmSync(workspaceDir, { recursive: true, force: true }); + } +} + +export async function runAgentEval(options: AgentEvalOptions): Promise { + assert( + existsSync(options.schemaPath), + `Schema not found: ${options.schemaPath}`, + ); + mkdirSync(options.outDir, { recursive: true }); + const env = buildEvalEnv(process.env); + const secretValues = collectSecretValues(env); + const mcpConfig = buildMcpConfig(options); + + if (!options.dryRun) { + if (options.agent === "claude") { + await assertClaudeAvailable(); + } else { + await assertCodexAvailable(); + } + } + + const [git, claude, codex] = await Promise.all([ + collectGitMetadata(options.repoRoot), + claudeVersion(), + codexVersion(), + ]); + + const workloadResults: WorkloadRunMetadata[] = []; + for (const workload of options.workloads) { + workloadResults.push( + await runWorkload( + options, + workload, + options.outDir, + env, + mcpConfig, + secretValues, + ), + ); + } + + writeJson(join(options.outDir, "run.json"), { + agent: options.agent, + server: options.server, + publishedPackage: options.publishedPackage, + dryRun: options.dryRun, + timeoutSeconds: options.timeoutSeconds, + repoRoot: options.repoRoot, + schemaPath: options.schemaPath, + reportingPath: options.reportingPath, + git, + claudeVersion: claude, + codexVersion: codex, + env: sanitizedEnvSummary(env), + workloads: workloadResults, + }); + + writeJson(join(options.outDir, "summary.json"), { + status: workloadResults.some( + (result) => result.status === "failed" || result.status === "timeout", + ) + ? "failed" + : options.dryRun + ? "dry-run" + : "success", + workloads: workloadResults.map( + ({ id, status, exitCode, durationMs, timedOut }) => ({ + id, + status, + exitCode, + durationMs, + timedOut, + }), + ), + }); +} + +async function main(): Promise { + const repoRoot = process.cwd(); + const options = parseArgs(process.argv.slice(2), repoRoot); + if (!isAbsolute(options.outDir)) { + options.outDir = resolve(repoRoot, options.outDir); + } + await runAgentEval(options); + console.log(`Agent eval artifacts written to ${options.outDir}`); +} + +if (import.meta.main) { + main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exit(1); + }); +}