diff --git a/.gitignore b/.gitignore index 478ac3f..b34bc9f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ __pycache__/ *.egg-info/ .pytest_cache/ *.pyc + +# local Claude Code worktrees/scratch +.claude/ diff --git a/README.md b/README.md index d725e70..f170a2d 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,11 @@ **Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** · [arena.oxagen.sh](https://arena.oxagen.sh) -Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. +> **Conflict of interest.** Arena is built by the team behind [Oxagen](https://oxagen.sh) and [Stella](https://github.com/macanderson/stella) — two of the agents it benchmarks. Assume bias and check us: every run ships its manifest, per-trial JSON, transcripts, and diffs, and the [`reproduce` command](#reproducing-a-run) runs on your machine with your keys. Don't take a number from us you can't regenerate yourself. This is the same disclosure the [publish bar](#what-it-takes-to-publish) requires of anyone. + +Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests copied in only after the agent exits** (never present in the workspace while it runs), and reports each metric separately with real statistics and full receipts. + +> The local suite is not adversarially sandboxed: the verify tests exist elsewhere on disk (and are public in this repo), so "held out" means *not in the agent's workspace*, not *unreachable*. For contamination-proof, network-isolated grading, run the [Harbor adapter](harbor/) — its Docker verifier never exposes the grader to the agent. ```bash git clone https://github.com/macanderson/arena @@ -54,7 +58,7 @@ Because most agent-vs-agent numbers don't survive five minutes of skeptical read | `gemini` | `gemini -p --output-format json --approval-mode auto_edit` | envelope-tested; re-verify flags against your installed version | | `mock` | in-process (CI / smoke) | built-in | -Autonomy levels are matched as closely as each CLI allows (auto-approve edits, no interactive prompts). Binary overrides: `--agents oxagen …` + `ARENA_OXAGEN_BIN=/path/to/oxagen`, etc. API keys come from each CLI's normal environment (`ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, …). +Autonomy is **not** yet fully matched, and the asymmetry favors the author's own agents — disclosed here rather than hidden: `claude-code` (`--permission-mode acceptEdits`) and `gemini` (`--approval-mode auto_edit`) auto-approve file edits but **cannot run arbitrary shell**, while `oxagen` and `stella` run one-shot **ungated** (shell included). On tasks graded by running tests, that is a real advantage for Oxagen/Stella. Until each adapter is pinned to a matched autonomy level (edits-only for all, or full-auto for all via `claude --dangerously-skip-permissions` / `gemini --approval-mode yolo`), do not use local-suite results to back an Oxagen/Stella-vs-competitor claim. The exact flags per adapter are in each adapter's header comment and recorded in the manifest. Binary overrides: `--agents oxagen …` + `ARENA_OXAGEN_BIN=/path/to/oxagen`, etc. API keys come from each CLI's normal environment (`ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, …) — note this means every spawned CLI inherits the full shell environment, including other vendors' keys. ### Adding your agent diff --git a/pricing.json b/pricing.json index 6762f1e..78e633e 100644 --- a/pricing.json +++ b/pricing.json @@ -3,19 +3,28 @@ "inputPerM": 3, "outputPerM": 15, "cacheReadPerM": 0.3, - "cacheWritePerM": 3.75 + "cacheWritePerM": 3.75, + "source": "https://platform.claude.com/docs/en/pricing", + "asOf": "2026-07-19", + "note": "Sticker rate. An introductory $2/$10 per MTok applies through 2026-08-31; cacheWrite is the 5-minute-TTL rate (1h TTL is 2x input). Verify before publishing." }, "anthropic/claude-opus-4.8": { - "inputPerM": 15, - "outputPerM": 75, - "cacheReadPerM": 1.5, - "cacheWritePerM": 18.75 + "inputPerM": 5, + "outputPerM": 25, + "cacheReadPerM": 0.5, + "cacheWritePerM": 6.25, + "source": "https://platform.claude.com/docs/en/pricing", + "asOf": "2026-07-19", + "note": "cacheWrite is the 5-minute-TTL rate (1h TTL is 2x input). Verify before publishing." }, "anthropic/claude-haiku-4.5": { - "inputPerM": 0.8, - "outputPerM": 4, - "cacheReadPerM": 0.08, - "cacheWritePerM": 1 + "inputPerM": 1, + "outputPerM": 5, + "cacheReadPerM": 0.1, + "cacheWritePerM": 1.25, + "source": "https://platform.claude.com/docs/en/pricing", + "asOf": "2026-07-19", + "note": "cacheWrite is the 5-minute-TTL rate (1h TTL is 2x input). Verify before publishing." }, "zai/glm-5.2": { "inputPerM": 0.6, @@ -27,6 +36,6 @@ "inputPerM": 1.25, "outputPerM": 10, "cacheReadPerM": 0.31, - "note": "Verify against current Google AI pricing before publishing." + "note": "Verify against current Google AI pricing before publishing. Ignores the >200k-token context tier ($2.50/$15)." } } diff --git a/src/stats.ts b/src/stats.ts index 9bc2729..619ed33 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -7,7 +7,11 @@ /** 95% Wilson score interval for a binomial proportion. */ export function wilsonInterval(successes: number, n: number, z = 1.96): [number, number] { - if (n === 0) return [0, 0]; + // No observations → maximal uncertainty, not false certainty. Returning [0,0] + // reads as "provably 0% success" and makes the regression gate certify a + // "100% drop" when a run has zero scored trials (e.g. every trial was an + // excluded agent-error) — the exact case the gate must NOT fail on. + if (n === 0) return [0, 1]; const p = successes / n; const denom = 1 + (z * z) / n; const center = p + (z * z) / (2 * n); @@ -117,7 +121,12 @@ export function pairedBootstrapDelta( if (deltas.length < iterations / 2) return null; deltas.sort((x, y) => x - y); - const lo = deltas[Math.floor(deltas.length * 0.025)] as number; - const hi = deltas[Math.min(deltas.length - 1, Math.ceil(deltas.length * 0.975))] as number; + // Type-1 empirical quantile (index ceil(q·N)-1), symmetric on both tails. + // The previous floor/ceil pair trimmed 2.5% below but only 2.45% above, + // biasing the interval — the kind of asymmetry a hostile reviewer diffs. + const quantileIdx = (q: number): number => + Math.min(deltas.length - 1, Math.max(0, Math.ceil(q * deltas.length) - 1)); + const lo = deltas[quantileIdx(0.025)] as number; + const hi = deltas[quantileIdx(0.975)] as number; return { relativeDelta: point, ci: [lo, hi], n }; } diff --git a/test/stats.test.ts b/test/stats.test.ts index eda8c3d..0aab541 100644 --- a/test/stats.test.ts +++ b/test/stats.test.ts @@ -9,8 +9,8 @@ import { } from "../src/stats.js"; describe("wilsonInterval", () => { - it("is [0,0] for n=0", () => { - expect(wilsonInterval(0, 0)).toEqual([0, 0]); + it("is [0,1] (maximal uncertainty) for n=0, not false certainty", () => { + expect(wilsonInterval(0, 0)).toEqual([0, 1]); }); it("brackets the point estimate and stays in [0,1]", () => { diff --git a/web/index.html b/web/index.html index ba595f0..7be6624 100644 --- a/web/index.html +++ b/web/index.html @@ -50,9 +50,15 @@

Head-to-head benchmarks for agentic coding CLIs — built to survive scrutin

Arena runs two or more coding agents — Claude Code, Gemini CLI, Oxagen, Stella, or your own — on the same tasks with the same model, budget, and timeout. - Held-out tests the agent can never see or author decide every outcome. Each metric + Held-out tests, copied in only after the agent exits, decide every outcome. Each metric is reported separately, with real statistics and full receipts.

+

+ Arena is built by the team behind Oxagen and Stella — two of the agents it benchmarks. + Assume bias and check us: every run ships a reproduce command that runs on your machine + with your keys. (The local suite is not network-sandboxed; for contamination-proof + grading use the Harbor adapter.) +

View on GitHub Read the methodology @@ -227,8 +233,12 @@

Score

Supported agents

-

Autonomy levels are matched as closely as each CLI allows — auto-approved - edits, no interactive prompts. Adding your own agent is one ~80-line adapter.

+

Autonomy is not yet fully matched, and today the gap favors the + author's own agents: Claude Code and Gemini auto-approve edits but can't run + arbitrary shell, while Oxagen and Stella run one-shot ungated. On tasks graded by + running tests that's a real edge — so don't back an Oxagen/Stella-vs-competitor claim + on the local suite until autonomy is pinned to a matched level. Adding your own agent + is one ~80-line adapter.