From 46f6fd26bee65acd2fcfb3430c50a472df283b0e Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 13:13:07 -0700 Subject: [PATCH 01/18] Initial release: head-to-head benchmark harness for agentic coding CLIs Adapters for Claude Code, Gemini CLI, Oxagen, Stella (+ mock for CI); held-out verification the agent can never author; normalized token and uniform cost accounting; matched-config enforcement with unmatched-model warnings; agent-error quarantine; Wilson/McNemar/seeded-bootstrap stats; per-run manifests, transcripts, diffs, and reproduce commands. --- .github/workflows/ci.yml | 25 + .gitignore | 5 + LICENSE | 21 + METHODOLOGY.md | 53 + README.md | 90 ++ package.json | 35 + pnpm-lock.yaml | 1197 +++++++++++++++++ pnpm-workspace.yaml | 2 + pricing.json | 32 + src/adapters/base.ts | 149 ++ src/adapters/claude-code.ts | 77 ++ src/adapters/gemini.ts | 84 ++ src/adapters/index.ts | 31 + src/adapters/mock.ts | 69 + src/adapters/oxagen.ts | 66 + src/adapters/stella.ts | 77 ++ src/cli.ts | 211 +++ src/orchestrator.ts | 280 ++++ src/parse.ts | 77 ++ src/pricing.ts | 55 + src/report.ts | 266 ++++ src/stats.ts | 130 ++ src/types.ts | 152 +++ src/workspace.ts | 159 +++ .../solution/src/intervals.mjs | 18 + tasks/bug-merge-intervals/task.json | 8 + .../verify/intervals.test.mjs | 38 + .../workspace/src/intervals.mjs | 16 + tasks/bug-paginate/solution/src/paginate.mjs | 20 + tasks/bug-paginate/task.json | 8 + tasks/bug-paginate/verify/paginate.test.mjs | 37 + tasks/bug-paginate/workspace/src/paginate.mjs | 11 + .../solution/src/query-string.mjs | 27 + tasks/bug-query-string/task.json | 8 + .../verify/query-string.test.mjs | 37 + .../workspace/src/query-string.mjs | 11 + .../solution/src/lru-cache.mjs | 44 + tasks/feature-lru-cache/task.json | 8 + .../verify/lru-cache.test.mjs | 74 + .../workspace/src/lru-cache.mjs | 9 + .../solution/src/rate-limiter.mjs | 40 + tasks/feature-rate-limiter/task.json | 8 + .../verify/rate-limiter.test.mjs | 60 + .../workspace/src/rate-limiter.mjs | 9 + .../solution/src/users.mjs | 25 + tasks/robustness-fetch-users/task.json | 8 + .../verify/users.test.mjs | 68 + .../workspace/src/users.mjs | 10 + test/adapters.test.ts | 170 +++ test/parse.test.ts | 72 + test/pipeline.test.ts | 147 ++ test/stats.test.ts | 91 ++ tsconfig.json | 17 + vitest.config.ts | 9 + 54 files changed, 4451 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 METHODOLOGY.md create mode 100644 README.md create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 pnpm-workspace.yaml create mode 100644 pricing.json create mode 100644 src/adapters/base.ts create mode 100644 src/adapters/claude-code.ts create mode 100644 src/adapters/gemini.ts create mode 100644 src/adapters/index.ts create mode 100644 src/adapters/mock.ts create mode 100644 src/adapters/oxagen.ts create mode 100644 src/adapters/stella.ts create mode 100644 src/cli.ts create mode 100644 src/orchestrator.ts create mode 100644 src/parse.ts create mode 100644 src/pricing.ts create mode 100644 src/report.ts create mode 100644 src/stats.ts create mode 100644 src/types.ts create mode 100644 src/workspace.ts create mode 100644 tasks/bug-merge-intervals/solution/src/intervals.mjs create mode 100644 tasks/bug-merge-intervals/task.json create mode 100644 tasks/bug-merge-intervals/verify/intervals.test.mjs create mode 100644 tasks/bug-merge-intervals/workspace/src/intervals.mjs create mode 100644 tasks/bug-paginate/solution/src/paginate.mjs create mode 100644 tasks/bug-paginate/task.json create mode 100644 tasks/bug-paginate/verify/paginate.test.mjs create mode 100644 tasks/bug-paginate/workspace/src/paginate.mjs create mode 100644 tasks/bug-query-string/solution/src/query-string.mjs create mode 100644 tasks/bug-query-string/task.json create mode 100644 tasks/bug-query-string/verify/query-string.test.mjs create mode 100644 tasks/bug-query-string/workspace/src/query-string.mjs create mode 100644 tasks/feature-lru-cache/solution/src/lru-cache.mjs create mode 100644 tasks/feature-lru-cache/task.json create mode 100644 tasks/feature-lru-cache/verify/lru-cache.test.mjs create mode 100644 tasks/feature-lru-cache/workspace/src/lru-cache.mjs create mode 100644 tasks/feature-rate-limiter/solution/src/rate-limiter.mjs create mode 100644 tasks/feature-rate-limiter/task.json create mode 100644 tasks/feature-rate-limiter/verify/rate-limiter.test.mjs create mode 100644 tasks/feature-rate-limiter/workspace/src/rate-limiter.mjs create mode 100644 tasks/robustness-fetch-users/solution/src/users.mjs create mode 100644 tasks/robustness-fetch-users/task.json create mode 100644 tasks/robustness-fetch-users/verify/users.test.mjs create mode 100644 tasks/robustness-fetch-users/workspace/src/users.mjs create mode 100644 test/adapters.test.ts create mode 100644 test/parse.test.ts create mode 100644 test/pipeline.test.ts create mode 100644 test/stats.test.ts create mode 100644 tsconfig.json create mode 100644 vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..64c0d06 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm typecheck + - run: pnpm test + # Task audit: every task's held-out tests must FAIL on the pristine + # workspace and PASS on the reference solution. + - run: pnpm arena verify diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..492dd79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +results/ +dist/ +*.tsbuildinfo +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7787eea --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Mac Anderson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/METHODOLOGY.md b/METHODOLOGY.md new file mode 100644 index 0000000..49e638d --- /dev/null +++ b/METHODOLOGY.md @@ -0,0 +1,53 @@ +# Methodology + +This document is the contract behind any number produced with Arena. If a published claim violates it, the claim — not the reader — is wrong. + +## What a run measures + +One run executes N agents × T tasks × K trials. Every trial: + +1. **Seed** — the task's `workspace/` fixture is copied to a fresh temp directory, committed to a fresh git repo. `TASK.md` contains the identical prompt every agent receives. +2. **Agent** — the agent CLI runs headlessly in that directory with the shared model/budget/timeout. Arguments are passed as argv arrays (no shell). +3. **Diff** — `git diff` against the seed commit captures exactly what the agent changed. +4. **Verify** — the harness deletes anything at `.arena-verify/`, copies the task's held-out `verify/` tests in, and runs them with `node --test`. The tests were never on disk while the agent ran. +5. **Score** — `passed` iff the held-out tests pass. `timeout` if the agent hit the wall-clock cap. `agent-error` if the CLI could not be invoked at all (spawn failure, or non-zero exit with zero diff and zero tokens) — these are harness-side failures, excluded from every comparison and listed separately. + +## Fairness controls + +- **Matched configuration.** One `--model`, `--budget`, `--timeout` for all agents. Per-agent model overrides are possible (`--agents a=m1,b=m2`) but stamp the manifest `matchedModels: false` and the report leads with a warning: such runs conflate harness quality with model quality and must not back harness-vs-harness claims. +- **Ordering.** Agents interleave within each task and the order reverses on alternate trials (ABBA), so provider load drift cannot systematically favor one agent. Trials run sequentially — wall-clock numbers are never measured under host contention. +- **Autonomy parity.** Each adapter runs its CLI at the closest available autonomy level (auto-approve file edits, non-interactive). The exact flags are in each adapter's header comment and recorded in the manifest via the CLI version. + +## Metric definitions + +- **Accuracy** — fraction of scored trials (excluding `agent-error`) whose held-out tests pass. Reported with a 95% Wilson interval. +- **Speed** — wall-clock seconds from spawn to exit, measured by the harness clock (the agent's self-reported duration is recorded separately, never used for comparison). +- **Tokens** — normalized: `input` excludes cache reads; `cacheRead`/`cacheWrite` tracked separately; `total = input + output + cacheRead + cacheWrite`. Per-adapter normalization rules are documented in the adapter source (e.g. oxagen and gemini report combined input, so cached tokens are subtracted; Claude Code already reports them separately). +- **Cost** — two numbers, never merged: `computedUsd` (one shared `pricing.json` applied to normalized tokens; null when the model has no entry — Arena never prices one vendor's tokens with another's rate card) and `agentReportedUsd` (whatever the CLI claimed). + +## Statistics + +- Success rates: **Wilson score intervals** (95%). +- Head-to-head success: **exact McNemar test** on paired (task, trial) outcomes — the correct test when both agents attempt identical work items. Discordant-pair counts are printed so readers can recompute it by hand. +- Continuous metrics (wall clock, tokens, cost): **paired bootstrap** (2000 resamples, seeded PRNG) percentile CIs on the relative difference in medians. Deterministic given the raw trials and the run seed. +- **No blended score.** "X% better" claims must name one metric and carry its CI. + +## What it takes to publish "Agent A beats Agent B" + +1. Same model on every agent (`matchedModels: true` in the manifest). +2. Task set and trial count committed **before** running (pre-registration — link the commit). +3. Enough trials that the McNemar p-value on success (or the bootstrap CI on your claimed metric) excludes chance at α = 0.05. The report tells you when it doesn't. +4. Publish the entire run directory: manifest, per-trial JSON, transcripts, diffs, report. +5. Disclose conflicts of interest (e.g. "we build Agent A") and pin CLI versions in the text. +6. Invite reruns: the manifest's reproduce command must work for a stranger with their own API keys. + +## Known limitations + +- The built-in task suite is small and JavaScript-only — a fast, cheap calibration set. Treat its results as directional, not as a research-grade resolve rate. +- Hidden tests can only assert behavior stated in the prompt; prompts therefore fully specify the contract. Agents that guess unstated conventions are neither rewarded nor punished. +- Wall-clock speed depends on provider-side load; ABBA interleaving and multiple trials mitigate but cannot eliminate this. Report medians, not means. +- Budget flags are passed to agents that support them (`claude --max-budget-usd`, `oxagen --budget`); agents without a budget flag are bounded by the timeout only. The manifest records exactly what was passed. + +## Scaling up + +For headline resolve-rate claims, run this same protocol over [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) with a containerized runner such as [Harbor](https://github.com/laude-institute/harbor) and the **official** evaluator, with a pre-registered instance list (published random seed over the 500 verified instances, n ≥ 100). Arena's local suite is for harness development, smoke comparisons, and metric plumbing — the statistics and receipt discipline are identical either way. diff --git a/README.md b/README.md new file mode 100644 index 0000000..51c2dab --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# Agent Arena + +**Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** + +Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. + +```bash +git clone https://github.com/macanderson/agent-arena +cd agent-arena && pnpm install + +pnpm arena doctor # which agent CLIs are installed? +pnpm arena list # available tasks + +# The only comparison that isolates the harness: same model everywhere. +pnpm arena run \ + --agents oxagen,claude-code \ + --model anthropic/claude-sonnet-5 \ + --trials 3 --budget 5 + +open results/run-*/report.md +``` + +## Why another benchmark? + +Because most agent-vs-agent numbers don't survive five minutes of skeptical reading. Arena is engineered around the specific ways benchmarks get (rightly) torn apart: + +| Attack | Arena's defense | +|---|---| +| "The agent graded itself" | Verification tests live **outside** the workspace, are copied in only **after** the agent exits, and anything the agent planted at the verify path is deleted first. CI proves every task's tests **fail on the pristine workspace** and **pass on the reference solution**. | +| "Different models / budgets / timeouts" | One config drives every agent. Runs with unmatched models are stamped `matchedModels: false` and the report leads with a warning banner. | +| "Your harness broke the competitor" | A CLI that can't even be invoked is scored **`agent-error`** and excluded from every comparison — it is never counted as the agent losing. | +| "Token counts aren't comparable" | Adapters normalize usage: `input` never includes cache reads; cache reads/writes are tracked separately. Documented per adapter. | +| "You priced their tokens wrong" | One shared pricing table applied to every agent; models without an entry get **no** computed cost (never a guessed one). Agent-self-reported cost is shown alongside. | +| "n=1" / "no stats" | Multiple trials per (task, agent); Wilson 95% CIs on success rates; **exact McNemar** on paired outcomes; seeded paired-bootstrap CIs on wall-clock/token/cost deltas. Deterministic: same raw data + seed ⇒ same report, bit for bit. | +| "First-mover advantage / time-of-day drift" | Agents interleave and their order flips every trial (ABBA). Tasks run sequentially so agents never contend for the host. | +| "Where's the raw data?" | Every trial writes its full stdout/stderr transcript, workspace diff, and a manifest with CLI versions, models, host, seed, and a one-line reproduce command. | + +## Supported agents + +| Adapter | Invocation | Status | +|---|---|---| +| `claude-code` | `claude -p --output-format json --permission-mode acceptEdits` | verified | +| `oxagen` | `oxagen --local --output-format json -- ` | verified | +| `stella` | `stella run ` + `STELLA_*` env | verified | +| `gemini` | `gemini -p --output-format json --approval-mode auto_edit` | envelope-tested; re-verify flags against your installed version | +| `mock` | in-process (CI / smoke) | built-in | + +Autonomy levels are matched as closely as each CLI allows (auto-approve edits, no interactive prompts). Binary overrides: `--agents oxagen …` + `ARENA_OXAGEN_BIN=/path/to/oxagen`, etc. API keys come from each CLI's normal environment (`ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, …). + +### Adding your agent + +Implement one small adapter (see `src/adapters/`): how to invoke your CLI headlessly in a directory, and how to map its output envelope to normalized tokens. ~80 lines; PRs welcome. Your agent must run non-interactively and edit files in the working directory. + +## Tasks + +Tasks are self-contained fixtures under `tasks//`: + +``` +tasks/bug-merge-intervals/ +├── task.json # the full behavior contract given to every agent +├── workspace/ # real starting code (buggy or stubbed) +├── verify/ # held-out node:test suite — never visible to the agent +└── solution/ # reference solution proving the task is solvable +``` + +Two invariants, enforced by `pnpm arena verify` in CI: + +1. The held-out tests **fail** against the pristine workspace (no tautological tasks). +2. They **pass** against the reference solution (no impossible tasks). + +The prompt states the complete behavior contract; the hidden tests assert only what the prompt states. Verification needs nothing but Node — no npm installs inside workspaces, no network. + +The built-in suite is deliberately small, fast, and cheap — a calibration set, not a research benchmark. For publishable resolve-rate claims, pair Arena's protocol with [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) under a containerized runner (e.g. [Harbor](https://github.com/laude-institute/harbor)) and use the official evaluator; see [METHODOLOGY.md](METHODOLOGY.md#scaling-up). + +## Reading the report + +`results//report.md` contains: per-agent success rates with Wilson CIs; head-to-head paired outcomes with the exact McNemar p-value; median wall-clock/token/cost deltas with bootstrap CIs; a per-task, per-trial outcome matrix; excluded `agent-error` trials; and the reproduce command. **Each metric is reported separately — Arena never emits a blended score.** + +Before you publish a number, read [METHODOLOGY.md](METHODOLOGY.md). Short version: same model on every agent, pre-committed task set and trial count, p < 0.05 on the metric you're claiming, raw run directory published, and a conflict-of-interest note if you built one of the agents. + +## Development + +```bash +pnpm install +pnpm typecheck +pnpm test # unit + end-to-end pipeline (mock agents, no API keys) +pnpm arena verify # audit every task's discrimination invariants +``` + +MIT © Mac Anderson diff --git a/package.json b/package.json new file mode 100644 index 0000000..6d85953 --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "agent-arena", + "version": "0.1.0", + "description": "Head-to-head benchmark harness for agentic coding CLIs (Claude Code, Gemini CLI, Oxagen, Stella, and yours) with held-out verification, matched configs, paired statistics, and full receipts.", + "license": "MIT", + "type": "module", + "repository": { + "type": "git", + "url": "https://github.com/macanderson/agent-arena.git" + }, + "keywords": [ + "benchmark", + "ai-agents", + "coding-agents", + "claude-code", + "gemini-cli", + "evaluation", + "llm" + ], + "engines": { + "node": ">=22" + }, + "scripts": { + "arena": "tsx src/cli.ts", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "verify:tasks": "tsx src/cli.ts verify" + }, + "devDependencies": { + "@types/node": "^25.9.1", + "tsx": "^4.19.2", + "typescript": "^5.8.3", + "vitest": "^2.1.9" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..d64c772 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,1197 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@types/node': + specifier: ^25.9.1 + version: 25.9.5 + tsx: + specifier: ^4.19.2 + version: 4.23.0 + typescript: + specifier: ^5.8.3 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@25.9.5) + +packages: + + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@rollup/rollup-android-arm-eabi@4.62.2': + resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.62.2': + resolution: {integrity: sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.62.2': + resolution: {integrity: sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.62.2': + resolution: {integrity: sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.62.2': + resolution: {integrity: sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.62.2': + resolution: {integrity: sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.62.2': + resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.62.2': + resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.62.2': + resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.62.2': + resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.62.2': + resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.62.2': + resolution: {integrity: sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + resolution: {integrity: sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + resolution: {integrity: sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.62.2': + resolution: {integrity: sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.62.2': + resolution: {integrity: sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==} + cpu: [x64] + os: [win32] + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@25.9.5': + resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + + '@vitest/expect@2.1.9': + resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + + '@vitest/mocker@2.1.9': + resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@2.1.9': + resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + + '@vitest/runner@2.1.9': + resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + + '@vitest/snapshot@2.1.9': + resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + + '@vitest/spy@2.1.9': + resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + + '@vitest/utils@2.1.9': + resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.15: + resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + postcss@8.5.17: + resolution: {integrity: sha512-J7EF+8X+CzRPaJPOv9Ck2wNWJvGnnl3PcNPAdGg6GTLjyVpyQ0yATMSXRFRV01BviT/9Gwuc3rjEyJbDJG9a4w==} + engines: {node: ^10 || ^12 || >=14} + + rollup@4.62.2: + resolution: {integrity: sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@1.2.0: + resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + engines: {node: '>=14.0.0'} + + tinyspy@3.0.2: + resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + engines: {node: '>=14.0.0'} + + tsx@4.23.0: + resolution: {integrity: sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + + vite-node@2.1.9: + resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + + vitest@2.1.9: + resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 2.1.9 + '@vitest/ui': 2.1.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + +snapshots: + + '@esbuild/aix-ppc64@0.21.5': + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.21.5': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.21.5': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.21.5': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.21.5': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.21.5': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.21.5': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.21.5': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.21.5': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.21.5': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.21.5': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.21.5': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.21.5': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.21.5': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.21.5': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.21.5': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.21.5': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.21.5': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.21.5': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.21.5': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.21.5': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.21.5': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.21.5': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@rollup/rollup-android-arm-eabi@4.62.2': + optional: true + + '@rollup/rollup-android-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-x64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-arm64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-x64@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-musl@4.62.2': + optional: true + + '@rollup/rollup-openbsd-x64@4.62.2': + optional: true + + '@rollup/rollup-openharmony-arm64@4.62.2': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.62.2': + optional: true + + '@types/estree@1.0.9': {} + + '@types/node@25.9.5': + dependencies: + undici-types: 7.24.6 + + '@vitest/expect@2.1.9': + dependencies: + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + tinyrainbow: 1.2.0 + + '@vitest/mocker@2.1.9(vite@5.4.21(@types/node@25.9.5))': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21(@types/node@25.9.5) + + '@vitest/pretty-format@2.1.9': + dependencies: + tinyrainbow: 1.2.0 + + '@vitest/runner@2.1.9': + dependencies: + '@vitest/utils': 2.1.9 + pathe: 1.1.2 + + '@vitest/snapshot@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + magic-string: 0.30.21 + pathe: 1.1.2 + + '@vitest/spy@2.1.9': + dependencies: + tinyspy: 3.0.2 + + '@vitest/utils@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + loupe: 3.2.1 + tinyrainbow: 1.2.0 + + assertion-error@2.0.1: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-eql@5.0.2: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fsevents@2.3.3: + optional: true + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + ms@2.1.3: {} + + nanoid@3.3.15: {} + + pathe@1.1.2: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + postcss@8.5.17: + dependencies: + nanoid: 3.3.15 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + rollup@4.62.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.62.2 + '@rollup/rollup-android-arm64': 4.62.2 + '@rollup/rollup-darwin-arm64': 4.62.2 + '@rollup/rollup-darwin-x64': 4.62.2 + '@rollup/rollup-freebsd-arm64': 4.62.2 + '@rollup/rollup-freebsd-x64': 4.62.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.2 + '@rollup/rollup-linux-arm-musleabihf': 4.62.2 + '@rollup/rollup-linux-arm64-gnu': 4.62.2 + '@rollup/rollup-linux-arm64-musl': 4.62.2 + '@rollup/rollup-linux-loong64-gnu': 4.62.2 + '@rollup/rollup-linux-loong64-musl': 4.62.2 + '@rollup/rollup-linux-ppc64-gnu': 4.62.2 + '@rollup/rollup-linux-ppc64-musl': 4.62.2 + '@rollup/rollup-linux-riscv64-gnu': 4.62.2 + '@rollup/rollup-linux-riscv64-musl': 4.62.2 + '@rollup/rollup-linux-s390x-gnu': 4.62.2 + '@rollup/rollup-linux-x64-gnu': 4.62.2 + '@rollup/rollup-linux-x64-musl': 4.62.2 + '@rollup/rollup-openbsd-x64': 4.62.2 + '@rollup/rollup-openharmony-arm64': 4.62.2 + '@rollup/rollup-win32-arm64-msvc': 4.62.2 + '@rollup/rollup-win32-ia32-msvc': 4.62.2 + '@rollup/rollup-win32-x64-gnu': 4.62.2 + '@rollup/rollup-win32-x64-msvc': 4.62.2 + fsevents: 2.3.3 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinypool@1.1.1: {} + + tinyrainbow@1.2.0: {} + + tinyspy@3.0.2: {} + + tsx@4.23.0: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@7.24.6: {} + + vite-node@2.1.9(@types/node@25.9.5): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 1.1.2 + vite: 5.4.21(@types/node@25.9.5) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + vite@5.4.21(@types/node@25.9.5): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.17 + rollup: 4.62.2 + optionalDependencies: + '@types/node': 25.9.5 + fsevents: 2.3.3 + + vitest@2.1.9(@types/node@25.9.5): + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@25.9.5)) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 1.1.2 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.1.1 + tinyrainbow: 1.2.0 + vite: 5.4.21(@types/node@25.9.5) + vite-node: 2.1.9(@types/node@25.9.5) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 25.9.5 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..5ed0b5a --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true diff --git a/pricing.json b/pricing.json new file mode 100644 index 0000000..6762f1e --- /dev/null +++ b/pricing.json @@ -0,0 +1,32 @@ +{ + "anthropic/claude-sonnet-5": { + "inputPerM": 3, + "outputPerM": 15, + "cacheReadPerM": 0.3, + "cacheWritePerM": 3.75 + }, + "anthropic/claude-opus-4.8": { + "inputPerM": 15, + "outputPerM": 75, + "cacheReadPerM": 1.5, + "cacheWritePerM": 18.75 + }, + "anthropic/claude-haiku-4.5": { + "inputPerM": 0.8, + "outputPerM": 4, + "cacheReadPerM": 0.08, + "cacheWritePerM": 1 + }, + "zai/glm-5.2": { + "inputPerM": 0.6, + "outputPerM": 2.2, + "cacheReadPerM": 0.11, + "note": "Verify against current z.ai coding-plan rate card before publishing." + }, + "google/gemini-2.5-pro": { + "inputPerM": 1.25, + "outputPerM": 10, + "cacheReadPerM": 0.31, + "note": "Verify against current Google AI pricing before publishing." + } +} diff --git a/src/adapters/base.ts b/src/adapters/base.ts new file mode 100644 index 0000000..c055d81 --- /dev/null +++ b/src/adapters/base.ts @@ -0,0 +1,149 @@ +/** + * Adapter contract. + * + * An adapter knows how to invoke one agent CLI in non-interactive ("headless") + * mode inside a prepared workspace and how to normalize its stdout envelope + * into Arena metrics. Adapters never decide success — the harness's held-out + * verification does. Adapters must normalize token counts so `tokens.input` + * excludes cache reads (see types.ts). + */ + +import { execFileSync, spawn } from "node:child_process"; + +import type { AgentSpec, ParsedEnvelope } from "../types.js"; + +export interface AdapterRunArgs { + prompt: string; + /** Arena-canonical model slug (e.g. "anthropic/claude-sonnet-5"). */ + model: string; + budgetUsd: number | undefined; + timeoutSeconds: number; + workDir: string; +} + +export interface ExecOutcome { + stdout: string; + stderr: string; + exitCode: number | null; + timedOut: boolean; + /** Set when the process could not be spawned at all. */ + spawnError?: string; +} + +export abstract class Adapter { + abstract readonly name: string; + protected abstract readonly defaultBinary: string; + + protected binOverride: string | undefined; + + constructor(spec?: Pick) { + this.binOverride = spec?.bin; + } + + bin(): string { + return ( + this.binOverride ?? + process.env[`ARENA_${this.name.replace(/-/g, "_").toUpperCase()}_BIN`] ?? + this.defaultBinary + ); + } + + /** Map the canonical model slug to what this CLI expects. */ + resolveModel(model: string): string { + return model; + } + + abstract buildArgs(args: AdapterRunArgs): string[]; + + /** Extra environment merged over process.env for the spawned agent. */ + env(_args: AdapterRunArgs): Record { + return {}; + } + + abstract parseEnvelope(stdout: string): ParsedEnvelope; + + isAvailable(): boolean { + try { + execFileSync(this.bin(), ["--version"], { stdio: "ignore", timeout: 10_000 }); + return true; + } catch { + return false; + } + } + + version(): string { + try { + return execFileSync(this.bin(), ["--version"], { + encoding: "utf8", + timeout: 10_000, + }) + .trim() + .split("\n")[0] as string; + } catch { + return "unknown"; + } + } + + /** + * Run the agent. Overridable (the mock adapter runs in-process). argv arrays + * only — nothing is ever interpolated through a shell. + */ + execute(args: AdapterRunArgs): Promise { + return new Promise((resolve) => { + const child = spawn(this.bin(), this.buildArgs(args), { + cwd: args.workDir, + env: { ...process.env, ...this.env(args) }, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, args.timeoutSeconds * 1000); + + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + + child.on("error", (err) => { + clearTimeout(timer); + resolve({ + stdout, + stderr, + exitCode: null, + timedOut: false, + spawnError: err.message, + }); + }); + + child.on("close", (code) => { + clearTimeout(timer); + resolve({ stdout, stderr, exitCode: code, timedOut }); + }); + }); + } +} + +export function emptyEnvelope(): ParsedEnvelope { + return { + tokens: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + agentReportedUsd: null, + agentReportedSeconds: null, + toolCalls: null, + iterations: null, + }; +} + +export function totalize(tokens: { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; +}): { input: number; output: number; cacheRead: number; cacheWrite: number; total: number } { + return { + ...tokens, + total: tokens.input + tokens.output + tokens.cacheRead + tokens.cacheWrite, + }; +} diff --git a/src/adapters/claude-code.ts b/src/adapters/claude-code.ts new file mode 100644 index 0000000..a06e95d --- /dev/null +++ b/src/adapters/claude-code.ts @@ -0,0 +1,77 @@ +/** + * Claude Code adapter. + * + * Invocation (verified against claude CLI ≥ 2.x): + * claude -p --output-format json --model + * --permission-mode acceptEdits [--max-budget-usd N] + * + * Result envelope: { type:"result", total_cost_usd, duration_ms, num_turns, + * usage:{ input_tokens, output_tokens, cache_read_input_tokens, + * cache_creation_input_tokens } } + * + * Token normalization: Claude's `input_tokens` already EXCLUDES cache reads, + * so counts map through directly. + */ + +import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; +import { isRecord, num, parseJsonEnvelope } from "../parse.js"; +import type { ParsedEnvelope } from "../types.js"; + +export class ClaudeCodeAdapter extends Adapter { + readonly name = "claude-code"; + protected readonly defaultBinary = "claude"; + + /** `anthropic/claude-sonnet-5` → `sonnet`; unrecognized slugs pass through. */ + override resolveModel(model: string): string { + const bare = model.replace(/^anthropic\//, ""); + const family = bare.match(/^claude-(opus|sonnet|haiku|fable)/)?.[1]; + return family ?? bare; + } + + buildArgs(args: AdapterRunArgs): string[] { + const argv = [ + "-p", + "--output-format", + "json", + "--model", + this.resolveModel(args.model), + "--permission-mode", + "acceptEdits", + ]; + if (args.budgetUsd !== undefined) { + argv.push("--max-budget-usd", String(args.budgetUsd)); + } + argv.push(args.prompt); + return argv; + } + + parseEnvelope(stdout: string): ParsedEnvelope { + const env = parseJsonEnvelope(stdout); + if (!env) return emptyEnvelope(); + + const usage = isRecord(env["usage"]) ? env["usage"] : {}; + const tokens = totalize({ + input: num(usage["input_tokens"]), + output: num(usage["output_tokens"]), + cacheRead: num(usage["cache_read_input_tokens"]), + cacheWrite: num(usage["cache_creation_input_tokens"]), + }); + + const reportedUsd = env["total_cost_usd"]; + const durationMs = env["duration_ms"]; + + return { + tokens, + agentReportedUsd: + typeof reportedUsd === "number" && Number.isFinite(reportedUsd) + ? reportedUsd + : null, + agentReportedSeconds: + typeof durationMs === "number" && Number.isFinite(durationMs) + ? durationMs / 1000 + : null, + toolCalls: null, // Claude's json envelope does not report tool calls. + iterations: num(env["num_turns"], 0) || null, + }; + } +} diff --git a/src/adapters/gemini.ts b/src/adapters/gemini.ts new file mode 100644 index 0000000..b914a0b --- /dev/null +++ b/src/adapters/gemini.ts @@ -0,0 +1,84 @@ +/** + * Gemini CLI adapter. + * + * Invocation: + * gemini -p --output-format json -m --approval-mode auto_edit + * + * `--approval-mode auto_edit` is the closest autonomy match to Claude Code's + * `--permission-mode acceptEdits` (edits auto-approved, arbitrary shell not). + * + * JSON output shape (gemini-cli ≥ 0.x): { response, stats: { models: { + * "": { tokens: { prompt, candidates, cached, total, ... } } } } } + * Multiple model entries are summed. Gemini's `prompt` count INCLUDES cached + * tokens, so normalized input = prompt − cached. + * + * NOTE: gemini-cli flags/envelope evolve quickly; `arena doctor` surfaces the + * installed version, and this adapter is covered by unit tests over a captured + * envelope. Re-verify against your installed version before publishing runs. + */ + +import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; +import { isRecord, num, parseJsonEnvelope } from "../parse.js"; +import type { ParsedEnvelope } from "../types.js"; + +export class GeminiAdapter extends Adapter { + readonly name = "gemini"; + protected readonly defaultBinary = "gemini"; + + /** `google/gemini-2.5-pro` → `gemini-2.5-pro`; bare ids pass through. */ + override resolveModel(model: string): string { + return model.replace(/^google\//, ""); + } + + buildArgs(args: AdapterRunArgs): string[] { + return [ + "-p", + args.prompt, + "--output-format", + "json", + "-m", + this.resolveModel(args.model), + "--approval-mode", + "auto_edit", + ]; + } + + parseEnvelope(stdout: string): ParsedEnvelope { + const env = parseJsonEnvelope(stdout); + if (!env) return emptyEnvelope(); + + const stats = isRecord(env["stats"]) ? env["stats"] : {}; + const models = isRecord(stats["models"]) ? stats["models"] : {}; + + let prompt = 0; + let candidates = 0; + let cached = 0; + let toolCalls: number | null = null; + for (const entry of Object.values(models)) { + if (!isRecord(entry)) continue; + const tokens = isRecord(entry["tokens"]) ? entry["tokens"] : {}; + prompt += num(tokens["prompt"]); + candidates += num(tokens["candidates"]); + cached += num(tokens["cached"]); + } + const tools = isRecord(stats["tools"]) ? stats["tools"] : {}; + const totalCalls = tools["totalCalls"]; + if (typeof totalCalls === "number" && Number.isFinite(totalCalls)) { + toolCalls = totalCalls; + } + + const cacheRead = Math.min(cached, prompt); + return { + tokens: totalize({ + input: prompt - cacheRead, + output: candidates, + cacheRead, + cacheWrite: 0, + }), + agentReportedUsd: null, + agentReportedSeconds: null, + toolCalls, + iterations: null, + }; + } +} diff --git a/src/adapters/index.ts b/src/adapters/index.ts new file mode 100644 index 0000000..3cbb0da --- /dev/null +++ b/src/adapters/index.ts @@ -0,0 +1,31 @@ +import type { AgentSpec } from "../types.js"; +import { Adapter } from "./base.js"; +import { ClaudeCodeAdapter } from "./claude-code.js"; +import { GeminiAdapter } from "./gemini.js"; +import { MockAdapter } from "./mock.js"; +import { OxagenAdapter } from "./oxagen.js"; +import { StellaAdapter } from "./stella.js"; + +export { Adapter } from "./base.js"; + +const registry: Record) => Adapter> = { + "claude-code": ClaudeCodeAdapter, + gemini: GeminiAdapter, + oxagen: OxagenAdapter, + stella: StellaAdapter, + mock: MockAdapter, +}; + +export function adapterNames(): string[] { + return Object.keys(registry); +} + +export function createAdapter(spec: AgentSpec): Adapter { + const Ctor = registry[spec.adapter]; + if (!Ctor) { + throw new Error( + `Unknown adapter "${spec.adapter}". Available: ${adapterNames().join(", ")}`, + ); + } + return new Ctor(spec); +} diff --git a/src/adapters/mock.ts b/src/adapters/mock.ts new file mode 100644 index 0000000..87dc421 --- /dev/null +++ b/src/adapters/mock.ts @@ -0,0 +1,69 @@ +/** + * Mock adapter — exercises the ENTIRE pipeline (workspace seeding, diff + * capture, held-out verification, metrics, reporting) without any API key or + * network access. Used by the test suite and CI. + * + * model "solve" → copies the task's reference solution into the workspace + * model "fail" → touches nothing + * + * It reports deterministic fake token counts so report math is testable. + */ + +import { cpSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +import { Adapter, totalize, type AdapterRunArgs, type ExecOutcome } from "./base.js"; +import type { ParsedEnvelope } from "../types.js"; + +export class MockAdapter extends Adapter { + readonly name = "mock"; + protected readonly defaultBinary = "mock"; + + /** The orchestrator records the active task dir here before each execute. */ + static currentTaskDir: string | null = null; + + override isAvailable(): boolean { + return true; + } + + override version(): string { + return "mock-1.0.0"; + } + + buildArgs(): string[] { + return []; + } + + override execute(args: AdapterRunArgs): Promise { + const behavior = args.model; + if (behavior === "solve") { + const taskDir = MockAdapter.currentTaskDir; + const solution = taskDir ? join(taskDir, "solution") : null; + if (solution && existsSync(solution)) { + cpSync(solution, args.workDir, { recursive: true }); + } + } + const envelope = { + type: "result", + usage: { inputTokens: 1000, outputTokens: 200, cachedInputTokens: 400 }, + steps: 3, + durationMs: 1500, + }; + return Promise.resolve({ + stdout: JSON.stringify(envelope), + stderr: "", + exitCode: 0, + timedOut: false, + }); + } + + parseEnvelope(): ParsedEnvelope { + return { + tokens: totalize({ input: 600, output: 200, cacheRead: 400, cacheWrite: 0 }), + agentReportedUsd: null, + agentReportedSeconds: 1.5, + toolCalls: 3, + iterations: 3, + }; + } +} diff --git a/src/adapters/oxagen.ts b/src/adapters/oxagen.ts new file mode 100644 index 0000000..114c529 --- /dev/null +++ b/src/adapters/oxagen.ts @@ -0,0 +1,66 @@ +/** + * Oxagen adapter. + * + * Invocation: + * oxagen --local --output-format json --model [--budget N] -- + * + * Result envelope: { type:"result", steps, usage:{ inputTokens, outputTokens, + * cachedInputTokens }, filesTouched:[], commandsRun:[], durationMs } + * + * Token normalization: oxagen's `inputTokens` INCLUDES cache reads + * (`cachedInputTokens` is a subset), so normalized input = + * inputTokens − cachedInputTokens. + */ + +import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; +import { countOf, isRecord, num, parseJsonEnvelope } from "../parse.js"; +import type { ParsedEnvelope } from "../types.js"; + +export class OxagenAdapter extends Adapter { + readonly name = "oxagen"; + protected readonly defaultBinary = "oxagen"; + + buildArgs(args: AdapterRunArgs): string[] { + const argv = ["--local", "--output-format", "json", "--model", args.model]; + if (args.budgetUsd !== undefined) argv.push("--budget", String(args.budgetUsd)); + argv.push("--", args.prompt); + return argv; + } + + override env(args: AdapterRunArgs): Record { + return { + OXAGEN_MODEL_SLUG: args.model, + ...(args.budgetUsd !== undefined + ? { OXAGEN_BUDGET: String(args.budgetUsd) } + : {}), + }; + } + + parseEnvelope(stdout: string): ParsedEnvelope { + const env = parseJsonEnvelope(stdout); + if (!env) return emptyEnvelope(); + + const usage = isRecord(env["usage"]) ? env["usage"] : {}; + const rawInput = num(usage["inputTokens"]); + const cacheRead = Math.min(num(usage["cachedInputTokens"]), rawInput); + const tokens = totalize({ + input: rawInput - cacheRead, + output: num(usage["outputTokens"]), + cacheRead, + cacheWrite: 0, + }); + + const durationMs = env["durationMs"]; + + return { + tokens, + agentReportedUsd: null, // envelope carries no cost + agentReportedSeconds: + typeof durationMs === "number" && Number.isFinite(durationMs) + ? durationMs / 1000 + : null, + toolCalls: countOf(env["commandsRun"]), + iterations: num(env["steps"], 0) || null, + }; + } +} diff --git a/src/adapters/stella.ts b/src/adapters/stella.ts new file mode 100644 index 0000000..22ad767 --- /dev/null +++ b/src/adapters/stella.ts @@ -0,0 +1,77 @@ +/** + * Stella adapter. + * + * Invocation: `stella run ` with configuration via STELLA_* env vars + * (STELLA_MODEL, STELLA_OUTPUT_FORMAT=json, optional STELLA_BUDGET) — the CLI + * documents env equivalents for its global flags. + * + * Envelope: mirrors the oxagen family but field names may drift between + * camelCase and snake_case across releases, so both spellings are read. + * Token normalization: cached input is reported separately; when a combined + * field is detected (inputTokens >= cachedInputTokens > 0) the cached share is + * subtracted, mirroring the oxagen rule. + */ + +import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; +import { countOf, isRecord, num, parseJsonEnvelope } from "../parse.js"; +import type { ParsedEnvelope } from "../types.js"; + +export class StellaAdapter extends Adapter { + readonly name = "stella"; + protected readonly defaultBinary = "stella"; + + /** Bare model ids get the zai/ provider prefix; scoped ids pass through. */ + override resolveModel(model: string): string { + return model.includes("/") ? model : `zai/${model}`; + } + + buildArgs(args: AdapterRunArgs): string[] { + return ["run", args.prompt]; + } + + override env(args: AdapterRunArgs): Record { + return { + STELLA_MODEL: this.resolveModel(args.model), + STELLA_OUTPUT_FORMAT: "json", + ...(args.budgetUsd !== undefined + ? { STELLA_BUDGET: String(args.budgetUsd) } + : {}), + }; + } + + parseEnvelope(stdout: string): ParsedEnvelope { + const env = parseJsonEnvelope(stdout); + if (!env) return emptyEnvelope(); + + const usage = isRecord(env["usage"]) ? env["usage"] : env; + const rawInput = num(usage["inputTokens"] ?? usage["input_tokens"]); + const cacheRead = Math.min( + num( + usage["cachedInputTokens"] ?? + usage["cacheReadTokens"] ?? + usage["cache_read_tokens"], + ), + rawInput, + ); + const tokens = totalize({ + input: rawInput - cacheRead, + output: num(usage["outputTokens"] ?? usage["output_tokens"]), + cacheRead, + cacheWrite: 0, + }); + + const reportedUsd = num( + env["costUsd"] ?? env["cost_usd"] ?? usage["costUsd"] ?? usage["cost_usd"], + -1, + ); + const durationMs = num(env["durationMs"] ?? env["duration_ms"], -1); + + return { + tokens, + agentReportedUsd: reportedUsd >= 0 ? reportedUsd : null, + agentReportedSeconds: durationMs >= 0 ? durationMs / 1000 : null, + toolCalls: countOf(env["toolCalls"] ?? env["commandsRun"]), + iterations: num(env["steps"] ?? env["iterations"], 0) || null, + }; + } +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..db1653d --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,211 @@ +#!/usr/bin/env tsx +/** + * Arena CLI. + * + * arena list — list tasks + * arena doctor — check which agent CLIs are installed + * arena run --agents a,b --model M — run a benchmark + * arena report — (re)generate report.md for a run + * arena verify [taskId…] — prove tasks discriminate: held-out + * tests FAIL on the pristine workspace + * and PASS on the reference solution + */ + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +import { adapterNames, createAdapter } from "./adapters/index.js"; +import { executeRun } from "./orchestrator.js"; +import { generateReport } from "./report.js"; +import { + applySolution, + loadTasks, + runVerification, + seedWorkspace, +} from "./workspace.js"; +import type { AgentSpec, LoadedTask } from "./types.js"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const TASK_ROOT = join(HERE, "..", "tasks"); + +const [command, ...rest] = process.argv.slice(2); + +switch (command) { + case "list": + cmdList(); + break; + case "doctor": + cmdDoctor(); + break; + case "run": + await cmdRun(rest); + break; + case "report": + cmdReport(rest); + break; + case "verify": + cmdVerify(rest); + break; + default: + console.log( + [ + "agent-arena — head-to-head benchmarks for agentic coding CLIs", + "", + "Commands:", + " list List available tasks", + " doctor Check installed agent CLIs and versions", + " run --agents [...] Run a benchmark (see below)", + " report Regenerate report.md for a run", + " verify [taskId...] Audit tasks: pristine must fail, solution must pass", + "", + "Run options:", + " --agents Comma list; each entry `adapter` or `adapter=model`", + ` adapters: ${adapterNames().join(", ")}`, + " --model Model slug applied to agents without an explicit =model", + " --tasks Comma list of task ids, or 'all' (default: all)", + " --trials Trials per (task, agent) pair (default: 3)", + " --budget Per-trial USD cap passed to agents that support one", + " --timeout Per-trial seconds (default: 600)", + " --seed RNG seed for deterministic statistics (default: 42)", + " --out Results root (default: ./results)", + "", + "Example:", + " pnpm arena run --agents oxagen,claude-code --model anthropic/claude-sonnet-5 --trials 3", + ].join("\n"), + ); +} + +function cmdList(): void { + for (const task of loadTasks(TASK_ROOT)) { + console.log(`${task.id}`); + console.log(` ${task.name} · ${task.category} · ${task.difficulty}`); + } +} + +function cmdDoctor(): void { + for (const name of adapterNames()) { + if (name === "mock") continue; + const adapter = createAdapter({ adapter: name, model: "" }); + const ok = adapter.isAvailable(); + console.log( + `${ok ? "✅" : "❌"} ${name.padEnd(12)} bin=${adapter.bin()} ${ok ? `version=${adapter.version()}` : "(not found)"}`, + ); + } +} + +async function cmdRun(argv: string[]): Promise { + const { values } = parseArgs({ + args: argv, + options: { + agents: { type: "string" }, + model: { type: "string" }, + tasks: { type: "string" }, + trials: { type: "string" }, + budget: { type: "string" }, + timeout: { type: "string" }, + seed: { type: "string" }, + out: { type: "string" }, + }, + }); + + if (!values.agents) { + console.error("--agents is required, e.g. --agents oxagen,claude-code"); + process.exit(1); + } + + const defaultModel = values.model; + const agents: AgentSpec[] = values.agents.split(",").map((entry) => { + const [adapter, model] = entry.split("=") as [string, string | undefined]; + const resolved = model ?? defaultModel; + if (!resolved) { + console.error( + `No model for agent "${adapter}". Pass --model or use ${adapter}=.`, + ); + process.exit(1); + } + return { adapter: adapter.trim(), model: resolved.trim() }; + }); + + const allTasks = loadTasks(TASK_ROOT); + let tasks: LoadedTask[] = allTasks; + if (values.tasks && values.tasks !== "all") { + const wanted = new Set(values.tasks.split(",").map((t) => t.trim())); + tasks = allTasks.filter((t) => wanted.has(t.id)); + const missing = [...wanted].filter((id) => !tasks.some((t) => t.id === id)); + if (missing.length > 0) { + console.error(`Unknown task ids: ${missing.join(", ")}`); + process.exit(1); + } + } + + const config = { + agents, + tasks, + trials: values.trials ? parseInt(values.trials, 10) : 3, + ...(values.budget !== undefined ? { budgetUsd: parseFloat(values.budget) } : {}), + timeoutSeconds: values.timeout ? parseInt(values.timeout, 10) : 600, + seed: values.seed ? parseInt(values.seed, 10) : 42, + outDir: resolve(values.out ?? "results"), + }; + + const { runDir } = await executeRun(config, (msg) => console.log(msg)); + const report = generateReport(runDir); + writeFileSync(join(runDir, "report.md"), report); + console.log(`\nRun complete. Report: ${join(runDir, "report.md")}`); +} + +function cmdReport(argv: string[]): void { + const runDir = argv[0]; + if (!runDir) { + console.error("Usage: arena report "); + process.exit(1); + } + const report = generateReport(resolve(runDir)); + writeFileSync(join(resolve(runDir), "report.md"), report); + console.log(report); +} + +/** + * Task audit: for each task, the held-out tests must FAIL against the pristine + * workspace (no tautological tests) and PASS against the reference solution + * (the task is actually solvable). CI runs this on every push. + */ +function cmdVerify(argv: string[]): void { + const all = loadTasks(TASK_ROOT); + const tasks = argv.length > 0 ? all.filter((t) => argv.includes(t.id)) : all; + let failures = 0; + + for (const task of tasks) { + const pristineDir = mkdtempSync(join(tmpdir(), "arena-audit-")); + const solvedDir = mkdtempSync(join(tmpdir(), "arena-audit-")); + try { + seedWorkspace(task, pristineDir); + const pristine = runVerification(task, pristineDir); + + seedWorkspace(task, solvedDir); + applySolution(task, solvedDir); + const solved = runVerification(task, solvedDir); + + const ok = !pristine.passed && solved.passed; + if (!ok) failures++; + console.log( + `${ok ? "✅" : "❌"} ${task.id}: pristine ${pristine.passed ? "PASSED (bad — tests don't discriminate)" : "fails (good)"}, solution ${solved.passed ? "passes (good)" : "FAILED (bad — task unsolvable)"}`, + ); + if (!solved.passed) { + console.log(solved.output.split("\n").slice(-15).join("\n")); + } + } finally { + rmSync(pristineDir, { recursive: true, force: true }); + rmSync(solvedDir, { recursive: true, force: true }); + } + } + + if (failures > 0) { + console.error(`\n${String(failures)} task(s) failed the audit.`); + process.exit(1); + } + console.log(`\nAll ${String(tasks.length)} task(s) discriminate correctly.`); +} diff --git a/src/orchestrator.ts b/src/orchestrator.ts new file mode 100644 index 0000000..888d615 --- /dev/null +++ b/src/orchestrator.ts @@ -0,0 +1,280 @@ +/** + * Run orchestration. + * + * Fairness rules enforced here rather than left to operator discipline: + * - Every agent gets the identical prompt, budget, and timeout. + * - Agents are interleaved and their order flips on alternate trials, so + * time-of-day drift (provider load, rate limits) cannot systematically + * favor whichever agent ran first. + * - Tasks run sequentially — no resource contention between agents on the + * same host skewing wall-clock numbers. + * - A run whose agent process could not even be invoked correctly is scored + * "agent-error", not "failed": adapter bugs must never read as one agent + * beating another. + */ + +import { execFileSync } from "node:child_process"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir, platform, arch } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; + +import { createAdapter, type Adapter } from "./adapters/index.js"; +import { MockAdapter } from "./adapters/mock.js"; +import { diffStats, sanitizeSegment } from "./parse.js"; +import { computeCost, loadPricing } from "./pricing.js"; +import { + buildPrompt, + collectDiff, + runVerification, + seedWorkspace, +} from "./workspace.js"; +import type { + AgentSpec, + LoadedTask, + Outcome, + RunConfig, + RunManifest, + TrialResult, +} from "./types.js"; + +export const HARNESS_VERSION = "0.1.0"; + +export interface RunProgress { + (message: string): void; +} + +export async function executeRun( + config: RunConfig, + log: RunProgress = () => {}, +): Promise<{ runDir: string; manifest: RunManifest; results: TrialResult[] }> { + const pricing = loadPricing(); + const runId = `run-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}-${randomUUID().slice(0, 8)}`; + const runDir = join(config.outDir, runId); + mkdirSync(join(runDir, "trials"), { recursive: true }); + mkdirSync(join(runDir, "transcripts"), { recursive: true }); + mkdirSync(join(runDir, "diffs"), { recursive: true }); + + const adapters = config.agents.map((spec) => ({ + spec, + adapter: createAdapter(spec), + })); + + for (const { spec, adapter } of adapters) { + if (!adapter.isAvailable()) { + throw new Error( + `Agent "${spec.adapter}" is not available (binary: ${adapter.bin()}). ` + + `Install it or point ARENA_${spec.adapter.replace(/-/g, "_").toUpperCase()}_BIN at it.`, + ); + } + } + + const manifest: RunManifest = { + runId, + createdAt: new Date().toISOString(), + harness: { + name: "agent-arena", + version: HARNESS_VERSION, + gitSha: gitSha(), + }, + host: { platform: platform(), arch: arch(), node: process.version }, + seed: config.seed, + trials: config.trials, + budgetUsd: config.budgetUsd ?? null, + timeoutSeconds: config.timeoutSeconds, + agents: adapters.map(({ spec, adapter }) => ({ + adapter: spec.adapter, + model: spec.model, + resolvedModel: adapter.resolveModel(spec.model), + version: adapter.version(), + bin: adapter.bin(), + })), + taskIds: config.tasks.map((t) => t.id), + matchedModels: new Set(config.agents.map((a) => a.model)).size <= 1, + reproduceCommand: buildReproduceCommand(config), + }; + writeFileSync(join(runDir, "manifest.json"), JSON.stringify(manifest, null, 2)); + + const results: TrialResult[] = []; + + for (let trial = 1; trial <= config.trials; trial++) { + // Flip agent order on alternate trials (ABBA) to null out drift. + const ordered = trial % 2 === 1 ? adapters : [...adapters].reverse(); + for (const task of config.tasks) { + for (const { spec, adapter } of ordered) { + log(`▶ trial ${trial}/${config.trials} · ${task.id} · ${spec.adapter} (${spec.model})`); + const result = await runOne(task, spec, adapter, trial, config, runId, runDir); + results.push(result); + writeFileSync( + join(runDir, "trials", `${result.id}.json`), + JSON.stringify(result, null, 2), + ); + const mark = + result.outcome === "passed" ? "✅" : result.outcome === "agent-error" ? "⚠️" : "❌"; + log( + ` ${mark} ${result.outcome} · ${result.timing.wallClockSeconds.toFixed(1)}s · ` + + `${result.tokens.total.toLocaleString()} tokens`, + ); + } + } + } + + writeFileSync( + join(runDir, "results.json"), + JSON.stringify({ manifest, results }, null, 2), + ); + + return { runDir, manifest, results }; +} + +async function runOne( + task: LoadedTask, + spec: AgentSpec, + adapter: Adapter, + trial: number, + config: RunConfig, + runId: string, + runDir: string, +): Promise { + const workDir = join(tmpdir(), `arena-${sanitizeSegment(task.id)}-${randomUUID()}`); + const startedAt = new Date(); + const id = `${task.id}-${spec.adapter}-${sanitizeSegment(spec.model)}-t${trial}`; + + try { + seedWorkspace(task, workDir); + MockAdapter.currentTaskDir = task.dir; + + const exec = await adapter.execute({ + prompt: buildPrompt(task), + model: spec.model, + budgetUsd: config.budgetUsd, + timeoutSeconds: config.timeoutSeconds, + workDir, + }); + + const finishedAt = new Date(); + const wallClockSeconds = (finishedAt.getTime() - startedAt.getTime()) / 1000; + + const diff = collectDiff(workDir); + const envelope = adapter.parseEnvelope(exec.stdout); + + // Invocation-level failure: the agent never actually ran (bad flags, + // missing binary, immediate non-zero exit with no work product). + const invocationFailure = + exec.spawnError !== undefined || + (exec.exitCode !== 0 && !exec.timedOut && diff.length === 0 && envelope.tokens.total === 0); + + let outcome: Outcome; + let verify = { passed: false, output: "" }; + if (invocationFailure) { + outcome = "agent-error"; + } else { + verify = runVerification(task, workDir); + if (verify.passed) outcome = "passed"; + else outcome = exec.timedOut ? "timeout" : "failed"; + } + + const transcriptPath = join("transcripts", `${id}.txt`); + writeFileSync( + join(runDir, transcriptPath), + [ + `# ${id}`, + `# exit=${String(exec.exitCode)} timedOut=${String(exec.timedOut)} spawnError=${exec.spawnError ?? "none"}`, + "", + "## stdout", + exec.stdout, + "", + "## stderr", + exec.stderr, + ].join("\n"), + ); + const diffPath = join("diffs", `${id}.patch`); + writeFileSync(join(runDir, diffPath), diff); + + const stats = diffStats(diff); + const errorText = + exec.spawnError ?? + (exec.timedOut + ? `timed out after ${config.timeoutSeconds}s` + : exec.exitCode !== 0 + ? `agent exited ${String(exec.exitCode)}: ${exec.stderr.trim().slice(0, 500)}` + : undefined); + + return { + id, + taskId: task.id, + trial, + agent: { + adapter: spec.adapter, + model: spec.model, + resolvedModel: adapter.resolveModel(spec.model), + version: adapter.version(), + bin: adapter.bin(), + }, + outcome, + verify, + timing: { + wallClockSeconds, + agentReportedSeconds: envelope.agentReportedSeconds, + }, + tokens: envelope.tokens, + cost: { + computedUsd: computeCost(envelope.tokens, spec.model, loadPricingCached()), + agentReportedUsd: envelope.agentReportedUsd, + pricingModel: loadPricingCached()[spec.model] ? spec.model : null, + }, + activity: { + toolCalls: envelope.toolCalls, + iterations: envelope.iterations, + filesTouched: stats.filesTouched, + linesAdded: stats.linesAdded, + linesRemoved: stats.linesRemoved, + diffBytes: Buffer.byteLength(diff), + }, + provenance: { + runId, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + }, + transcriptPath, + diffPath, + ...(errorText !== undefined ? { error: errorText } : {}), + }; + } finally { + MockAdapter.currentTaskDir = null; + rmSync(workDir, { recursive: true, force: true }); + } +} + +let pricingCache: ReturnType | null = null; +function loadPricingCached(): ReturnType { + pricingCache ??= loadPricing(); + return pricingCache; +} + +function gitSha(): string { + try { + return execFileSync("git", ["rev-parse", "--short", "HEAD"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return "unknown"; + } +} + +function buildReproduceCommand(config: RunConfig): string { + const agents = config.agents + .map((a) => (a.model ? `${a.adapter}=${a.model}` : a.adapter)) + .join(","); + const parts = [ + "pnpm arena run", + `--agents ${agents}`, + `--tasks ${config.tasks.map((t) => t.id).join(",")}`, + `--trials ${String(config.trials)}`, + `--timeout ${String(config.timeoutSeconds)}`, + `--seed ${String(config.seed)}`, + ]; + if (config.budgetUsd !== undefined) parts.push(`--budget ${String(config.budgetUsd)}`); + return parts.join(" "); +} diff --git a/src/parse.ts b/src/parse.ts new file mode 100644 index 0000000..b8901c7 --- /dev/null +++ b/src/parse.ts @@ -0,0 +1,77 @@ +/** + * Envelope-parsing helpers shared by all adapters. Pure functions, unit-tested + * without spawning any agent binary. + */ + +/** Coerce an unknown JSON value into a finite non-negative number, else fallback. */ +export function num(value: unknown, fallback = 0): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : fallback; +} + +/** A count that may arrive as an array (length) or a number. Null when absent. */ +export function countOf(value: unknown): number | null { + if (Array.isArray(value)) return value.length; + if (typeof value === "number" && Number.isFinite(value)) return value; + return null; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Extract the machine-readable JSON envelope from an agent's stdout. + * + * Headless agent CLIs print either a single JSON object or JSONL (one event + * per line). We want the last `type: "result"` object, falling back to the + * last parseable object. Banners and log lines are ignored. + */ +export function parseJsonEnvelope( + stdout: string, +): Record | null { + if (!stdout) return null; + + const lines = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.startsWith("{")); + + let fallback: Record | null = null; + + for (let i = lines.length - 1; i >= 0; i--) { + try { + const obj: unknown = JSON.parse(lines[i] as string); + if (!isRecord(obj)) continue; + if (fallback === null) fallback = obj; + if (obj["type"] === "result") return obj; + } catch { + // Not a complete JSON object line — skip. + } + } + + return fallback; +} + +/** Make a string safe as a single path segment (model slugs contain "/"). */ +export function sanitizeSegment(value: string): string { + return value.replace(/[^a-zA-Z0-9._-]+/g, "_").replace(/^_+|_+$/g, ""); +} + +/** Count +/- lines and files touched from a unified diff. */ +export function diffStats(diff: string): { + filesTouched: number; + linesAdded: number; + linesRemoved: number; +} { + let filesTouched = 0; + let linesAdded = 0; + let linesRemoved = 0; + for (const line of diff.split("\n")) { + if (line.startsWith("diff --git ")) filesTouched += 1; + else if (line.startsWith("+") && !line.startsWith("+++")) linesAdded += 1; + else if (line.startsWith("-") && !line.startsWith("---")) linesRemoved += 1; + } + return { filesTouched, linesAdded, linesRemoved }; +} diff --git a/src/pricing.ts b/src/pricing.ts new file mode 100644 index 0000000..9f1e183 --- /dev/null +++ b/src/pricing.ts @@ -0,0 +1,55 @@ +/** + * Uniform cost accounting. + * + * One pricing table (pricing.json, USD per million tokens) is applied to every + * agent's NORMALIZED token counts. If a model has no entry, computed cost is + * null — the harness never silently prices one vendor's tokens with another + * vendor's rate card. Agent-self-reported cost is recorded separately and the + * report shows both. + */ + +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { PricingTable, TokenCounts } from "./types.js"; +import { isRecord, num } from "./parse.js"; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +export function loadPricing(path?: string): PricingTable { + const file = path ?? join(HERE, "..", "pricing.json"); + const raw: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!isRecord(raw)) return {}; + const table: PricingTable = {}; + for (const [model, entry] of Object.entries(raw)) { + if (!isRecord(entry)) continue; + table[model] = { + inputPerM: num(entry["inputPerM"]), + outputPerM: num(entry["outputPerM"]), + cacheReadPerM: num(entry["cacheReadPerM"]), + ...(entry["cacheWritePerM"] !== undefined + ? { cacheWritePerM: num(entry["cacheWritePerM"]) } + : {}), + ...(typeof entry["note"] === "string" ? { note: entry["note"] } : {}), + }; + } + return table; +} + +/** Compute USD cost from normalized tokens, or null if the model is unpriced. */ +export function computeCost( + tokens: TokenCounts, + model: string, + pricing: PricingTable, +): number | null { + const rate = pricing[model]; + if (!rate) return null; + const cacheWriteRate = rate.cacheWritePerM ?? rate.inputPerM; + return ( + (tokens.input / 1e6) * rate.inputPerM + + (tokens.output / 1e6) * rate.outputPerM + + (tokens.cacheRead / 1e6) * rate.cacheReadPerM + + (tokens.cacheWrite / 1e6) * cacheWriteRate + ); +} diff --git a/src/report.ts b/src/report.ts new file mode 100644 index 0000000..c8e2ab6 --- /dev/null +++ b/src/report.ts @@ -0,0 +1,266 @@ +/** + * Markdown report generation from a run directory. + * + * Reporting rules: + * - "agent-error" trials (the harness failed to invoke the CLI) are excluded + * from every headline number and listed separately — an adapter bug must + * never be presented as an agent losing. + * - Success rates carry 95% Wilson intervals; head-to-head success uses the + * exact McNemar test on paired (task, trial) outcomes; latency/token/cost + * deltas use a seeded paired bootstrap. Each metric is reported separately — + * no blended score. + * - The report embeds the reproduce command and per-trial receipt paths. + */ + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + mcnemarExact, + median, + pairedBootstrapDelta, + wilsonInterval, +} from "./stats.js"; +import { isRecord } from "./parse.js"; +import type { RunManifest, TrialResult } from "./types.js"; + +export function loadRun(runDir: string): { + manifest: RunManifest; + results: TrialResult[]; +} { + const raw: unknown = JSON.parse( + readFileSync(join(runDir, "results.json"), "utf8"), + ); + if (!isRecord(raw) || !isRecord(raw["manifest"]) || !Array.isArray(raw["results"])) { + throw new Error(`Malformed results.json in ${runDir}`); + } + return { + manifest: raw["manifest"] as unknown as RunManifest, + results: raw["results"] as unknown as TrialResult[], + }; +} + +function agentKey(r: TrialResult): string { + return `${r.agent.adapter} (${r.agent.model})`; +} + +function pct(x: number): string { + return `${(x * 100).toFixed(1)}%`; +} + +function fmtUsd(x: number | null): string { + return x === null ? "—" : `$${x.toFixed(4)}`; +} + +function fmtDelta(x: number): string { + const sign = x > 0 ? "+" : ""; + return `${sign}${(x * 100).toFixed(1)}%`; +} + +export function generateReport(runDir: string): string { + const { manifest, results } = loadRun(runDir); + + const keys = [...new Set(results.map(agentKey))]; + const byAgent = new Map( + keys.map((k) => [k, results.filter((r) => agentKey(r) === k)]), + ); + + const lines: string[] = []; + lines.push(`# Arena run report — \`${manifest.runId}\``, ""); + lines.push(`- Harness: ${manifest.harness.name} v${manifest.harness.version} (git ${manifest.harness.gitSha})`); + lines.push(`- Host: ${manifest.host.platform}/${manifest.host.arch}, node ${manifest.host.node}`); + lines.push(`- Created: ${manifest.createdAt}`); + lines.push(`- Trials per (task, agent): ${String(manifest.trials)} · timeout ${String(manifest.timeoutSeconds)}s · budget ${manifest.budgetUsd === null ? "none" : `$${String(manifest.budgetUsd)}`} · seed ${String(manifest.seed)}`); + lines.push(`- Tasks (${String(manifest.taskIds.length)}): ${manifest.taskIds.join(", ")}`); + lines.push(""); + lines.push("Agents:"); + for (const a of manifest.agents) { + lines.push(`- **${a.adapter}** · model \`${a.model}\` (resolved \`${a.resolvedModel}\`) · version \`${a.version}\``); + } + lines.push(""); + + if (!manifest.matchedModels) { + lines.push( + "> ⚠️ **Unmatched models.** Agents in this run used different models, so", + "> differences conflate harness and model quality. For harness-vs-harness", + "> claims, rerun with the same model on every agent.", + "", + ); + } + + const errorTrials = results.filter((r) => r.outcome === "agent-error"); + if (errorTrials.length > 0) { + lines.push( + `> ⚠️ **${String(errorTrials.length)} trial(s) excluded as \`agent-error\`** — the CLI could not be`, + "> invoked (bad flags / missing binary). These are harness-side failures and", + "> are excluded from all comparisons below. See “Excluded trials”.", + "", + ); + } + + // ── Per-agent summary ── + lines.push("## Results by agent", ""); + lines.push( + "| Agent | Scored trials | Passed | Success rate (95% CI) | Median wall clock | Median tokens (billed) | Median computed cost | Self-reported cost |", + "|---|---|---|---|---|---|---|---|", + ); + for (const key of keys) { + const all = byAgent.get(key) ?? []; + const scored = all.filter((r) => r.outcome !== "agent-error"); + const passed = scored.filter((r) => r.outcome === "passed").length; + const [lo, hi] = wilsonInterval(passed, scored.length); + const wall = median(scored.map((r) => r.timing.wallClockSeconds)); + const toks = median(scored.map((r) => r.tokens.total)); + const costs = scored + .map((r) => r.cost.computedUsd) + .filter((c): c is number => c !== null); + const self = scored + .map((r) => r.cost.agentReportedUsd) + .filter((c): c is number => c !== null); + lines.push( + `| ${key} | ${String(scored.length)} | ${String(passed)} | ${pct(scored.length ? passed / scored.length : 0)} (${pct(lo)}–${pct(hi)}) | ${wall.toFixed(1)}s | ${Math.round(toks).toLocaleString()} | ${fmtUsd(costs.length ? median(costs) : null)} | ${fmtUsd(self.length ? median(self) : null)} |`, + ); + } + lines.push(""); + lines.push( + "Token counts are normalized (input excludes cache reads; cache reads and writes tracked separately). Computed cost applies one shared pricing table to every agent; “—” means the model has no pricing entry (cost is never guessed).", + "", + ); + + // ── Pairwise comparisons ── + if (keys.length >= 2) { + lines.push("## Head-to-head (paired)", ""); + for (let i = 0; i < keys.length; i++) { + for (let j = i + 1; j < keys.length; j++) { + const a = keys[i] as string; + const b = keys[j] as string; + lines.push(...pairwiseSection(a, b, byAgent, manifest)); + } + } + } + + // ── Per-task matrix ── + lines.push("## Per-task outcomes", ""); + lines.push(`| Task | ${keys.join(" | ")} |`); + lines.push(`|---|${keys.map(() => "---").join("|")}|`); + for (const taskId of manifest.taskIds) { + const cells = keys.map((key) => { + const trials = (byAgent.get(key) ?? []).filter((r) => r.taskId === taskId); + if (trials.length === 0) return "—"; + return trials + .map((r) => + r.outcome === "passed" + ? "✅" + : r.outcome === "agent-error" + ? "⚠️" + : r.outcome === "timeout" + ? "⏱" + : "❌", + ) + .join(""); + }); + lines.push(`| ${taskId} | ${cells.join(" | ")} |`); + } + lines.push("", "One symbol per trial: ✅ passed · ❌ failed · ⏱ timeout · ⚠️ agent-error (excluded).", ""); + + // ── Excluded trials ── + if (errorTrials.length > 0) { + lines.push("## Excluded trials (agent-error)", ""); + for (const r of errorTrials) { + lines.push(`- \`${r.id}\`: ${r.error ?? "unknown"} (transcript: \`${r.transcriptPath}\`)`); + } + lines.push(""); + } + + // ── Receipts ── + lines.push("## Receipts", ""); + lines.push("```"); + lines.push(manifest.reproduceCommand); + lines.push("```"); + lines.push( + "", + "Every trial's full stdout/stderr transcript and workspace diff are stored under `transcripts/` and `diffs/` in this run directory. Statistics are deterministic given the raw trials (seeded bootstrap).", + "", + ); + + return lines.join("\n"); +} + +function pairwiseSection( + aKey: string, + bKey: string, + byAgent: Map, + manifest: RunManifest, +): string[] { + const lines: string[] = [`### ${aKey} vs ${bKey}`, ""]; + + const aTrials = (byAgent.get(aKey) ?? []).filter((r) => r.outcome !== "agent-error"); + const bTrials = (byAgent.get(bKey) ?? []).filter((r) => r.outcome !== "agent-error"); + const index = (rs: TrialResult[]): Map => + new Map(rs.map((r) => [`${r.taskId}#${String(r.trial)}`, r])); + const ai = index(aTrials); + const bi = index(bTrials); + const pairKeys = [...ai.keys()].filter((k) => bi.has(k)).sort(); + + if (pairKeys.length === 0) { + lines.push("No shared (task, trial) pairs with both agents scored — nothing to compare.", ""); + return lines; + } + + let aOnly = 0; + let bOnly = 0; + let both = 0; + let neither = 0; + for (const k of pairKeys) { + const ap = (ai.get(k) as TrialResult).outcome === "passed"; + const bp = (bi.get(k) as TrialResult).outcome === "passed"; + if (ap && bp) both++; + else if (ap) aOnly++; + else if (bp) bOnly++; + else neither++; + } + const p = mcnemarExact(aOnly, bOnly); + lines.push( + `Paired (task, trial) outcomes over ${String(pairKeys.length)} pairs: both passed ${String(both)}, only ${aKey} ${String(aOnly)}, only ${bKey} ${String(bOnly)}, neither ${String(neither)}.`, + ); + lines.push( + p === null + ? "McNemar test: not applicable (no discordant pairs)." + : `Exact McNemar (two-sided): p = ${p.toFixed(4)}${p < 0.05 ? " — statistically significant at α=0.05" : " — not significant at α=0.05; collect more trials before claiming a success-rate difference"}.`, + "", + ); + + const metrics: { label: string; get: (r: TrialResult) => number | null }[] = [ + { label: "Wall clock (s)", get: (r) => r.timing.wallClockSeconds }, + { label: "Total tokens", get: (r) => r.tokens.total }, + { label: "Output tokens", get: (r) => r.tokens.output }, + { label: "Computed cost (USD)", get: (r) => r.cost.computedUsd }, + ]; + lines.push(`| Metric | ${aKey} (median) | ${bKey} (median) | Δ relative to ${aKey} (95% CI) |`); + lines.push("|---|---|---|---|"); + for (const metric of metrics) { + const av: number[] = []; + const bv: number[] = []; + for (const k of pairKeys) { + const x = metric.get(ai.get(k) as TrialResult); + const y = metric.get(bi.get(k) as TrialResult); + if (x === null || y === null) continue; + av.push(x); + bv.push(y); + } + if (av.length === 0) { + lines.push(`| ${metric.label} | — | — | — |`); + continue; + } + const delta = pairedBootstrapDelta(av, bv, manifest.seed); + lines.push( + `| ${metric.label} | ${median(av).toFixed(2)} | ${median(bv).toFixed(2)} | ${delta ? `${fmtDelta(delta.relativeDelta)} (${fmtDelta(delta.ci[0])} to ${fmtDelta(delta.ci[1])})` : "n too small"} |`, + ); + } + lines.push( + "", + `Negative Δ means ${bKey} used less (faster / fewer tokens / cheaper). CIs from a seeded paired bootstrap over (task, trial) pairs.`, + "", + ); + return lines; +} diff --git a/src/stats.ts b/src/stats.ts new file mode 100644 index 0000000..776f372 --- /dev/null +++ b/src/stats.ts @@ -0,0 +1,130 @@ +/** + * Statistics for paired agent comparisons. + * + * Everything here is deterministic: the bootstrap uses a seeded PRNG so a + * published report can be regenerated bit-for-bit from the raw trial data. + */ + +/** 95% Wilson score interval for a binomial proportion. */ +export function wilsonInterval( + successes: number, + n: number, + z = 1.96, +): [number, number] { + if (n === 0) return [0, 0]; + const p = successes / n; + const denom = 1 + (z * z) / n; + const center = p + (z * z) / (2 * n); + const margin = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n)); + return [ + Math.max(0, (center - margin) / denom), + Math.min(1, (center + margin) / denom), + ]; +} + +/** + * Exact McNemar test (two-sided) on paired pass/fail outcomes. + * + * `b` = pairs where A passed and B failed; `c` = pairs where B passed and A + * failed. Under H0 (no difference), discordant pairs are Binomial(b+c, 0.5). + * Returns the two-sided exact p-value, or null when there are no discordant + * pairs (the test is undefined; the agents never disagreed). + */ +export function mcnemarExact(b: number, c: number): number | null { + const n = b + c; + if (n === 0) return null; + const k = Math.min(b, c); + // P(X <= k) for X ~ Binomial(n, 0.5), computed in log space for stability. + let tail = 0; + for (let i = 0; i <= k; i++) { + tail += Math.exp(logChoose(n, i) - n * Math.LN2); + } + return Math.min(1, 2 * tail); +} + +function logChoose(n: number, k: number): number { + return logFactorial(n) - logFactorial(k) - logFactorial(n - k); +} + +const logFactCache: number[] = [0, 0]; +function logFactorial(n: number): number { + for (let i = logFactCache.length; i <= n; i++) { + logFactCache.push((logFactCache[i - 1] as number) + Math.log(i)); + } + return logFactCache[n] as number; +} + +/** Deterministic 32-bit PRNG (mulberry32). */ +export function mulberry32(seed: number): () => number { + let a = seed >>> 0; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function median(values: number[]): number { + if (values.length === 0) return NaN; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? (sorted[mid] as number) + : ((sorted[mid - 1] as number) + (sorted[mid] as number)) / 2; +} + +export function mean(values: number[]): number { + if (values.length === 0) return NaN; + return values.reduce((s, v) => s + v, 0) / values.length; +} + +export interface PairedDeltaCI { + /** Point estimate: (median(b) - median(a)) / median(a). */ + relativeDelta: number; + /** 95% percentile bootstrap CI on the relative delta. */ + ci: [number, number]; + n: number; +} + +/** + * Paired bootstrap CI for the relative difference in medians between two + * agents measured on the SAME (task, trial) pairs. Resampling is over pairs, + * preserving the pairing structure. Returns null when fewer than 3 pairs or + * when the baseline median is 0 (relative delta undefined). + */ +export function pairedBootstrapDelta( + a: number[], + b: number[], + seed: number, + iterations = 2000, +): PairedDeltaCI | null { + const n = Math.min(a.length, b.length); + if (n < 3) return null; + const baseMedian = median(a.slice(0, n)); + if (baseMedian === 0 || Number.isNaN(baseMedian)) return null; + + const point = (median(b.slice(0, n)) - baseMedian) / baseMedian; + const rng = mulberry32(seed); + const deltas: number[] = []; + + for (let it = 0; it < iterations; it++) { + const sa: number[] = []; + const sb: number[] = []; + for (let i = 0; i < n; i++) { + const idx = Math.floor(rng() * n); + sa.push(a[idx] as number); + sb.push(b[idx] as number); + } + const ma = median(sa); + if (ma === 0) continue; + deltas.push((median(sb) - ma) / ma); + } + if (deltas.length < iterations / 2) return null; + + deltas.sort((x, y) => x - y); + const lo = deltas[Math.floor(deltas.length * 0.025)] as number; + const hi = deltas[Math.min(deltas.length - 1, Math.ceil(deltas.length * 0.975))] as number; + return { relativeDelta: point, ci: [lo, hi], n }; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..66c02b3 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,152 @@ +/** + * Core types for the Arena harness. + * + * Design constraints that shape these shapes: + * - Token fields are NORMALIZED: `input` never includes cache reads. Adapters + * are responsible for subtracting cached tokens when a CLI reports them + * combined, so cross-agent token comparisons are apples-to-apples. + * - Cost is dual-tracked: `computedUsd` comes from one shared pricing table + * applied to normalized tokens (null when the model has no pricing entry — + * never guessed), and `agentReportedUsd` is whatever the CLI claimed. + * - `outcome` distinguishes a real task failure from a harness/invocation + * failure ("agent-error"): a run where the agent binary rejected its flags + * must never be scored as the agent losing the task. + */ + +export type Outcome = "passed" | "failed" | "timeout" | "agent-error"; + +export interface TaskDef { + id: string; + name: string; + category: "bug-fix" | "feature" | "robustness" | "refactor"; + difficulty: "easy" | "medium" | "hard"; + /** Full behavior contract given to every agent. The held-out tests assert + * only behavior stated here. */ + prompt: string; + /** Seconds the verify step may take (default 60). */ + verifyTimeoutSeconds?: number; + tags: string[]; +} + +/** A task on disk: tasks//{task.json, workspace/, verify/, solution/}. */ +export interface LoadedTask extends TaskDef { + dir: string; +} + +export interface AgentSpec { + /** Adapter name, e.g. "claude-code". */ + adapter: string; + /** Arena-canonical model slug, e.g. "anthropic/claude-sonnet-5". */ + model: string; + /** Override for the CLI binary path. */ + bin?: string; +} + +export interface RunConfig { + agents: AgentSpec[]; + tasks: LoadedTask[]; + trials: number; + budgetUsd?: number; + timeoutSeconds: number; + seed: number; + outDir: string; +} + +export interface TokenCounts { + /** Uncached input tokens. */ + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + /** input + output + cacheRead + cacheWrite. */ + total: number; +} + +export interface ParsedEnvelope { + tokens: TokenCounts; + agentReportedUsd: number | null; + agentReportedSeconds: number | null; + toolCalls: number | null; + iterations: number | null; +} + +export interface TrialResult { + id: string; + taskId: string; + /** 1-based trial index. */ + trial: number; + agent: { + adapter: string; + model: string; + resolvedModel: string; + version: string; + bin: string; + }; + outcome: Outcome; + verify: { + passed: boolean; + /** Trimmed tail of the verify runner output (receipt). */ + output: string; + }; + timing: { + wallClockSeconds: number; + agentReportedSeconds: number | null; + }; + tokens: TokenCounts; + cost: { + computedUsd: number | null; + agentReportedUsd: number | null; + pricingModel: string | null; + }; + activity: { + toolCalls: number | null; + iterations: number | null; + filesTouched: number; + linesAdded: number; + linesRemoved: number; + diffBytes: number; + }; + provenance: { + runId: string; + startedAt: string; + finishedAt: string; + }; + /** Relative paths (within the run dir) to full receipts. */ + transcriptPath: string; + diffPath: string; + error?: string; +} + +export interface RunManifest { + runId: string; + createdAt: string; + harness: { name: string; version: string; gitSha: string }; + host: { platform: string; arch: string; node: string }; + seed: number; + trials: number; + budgetUsd: number | null; + timeoutSeconds: number; + agents: { + adapter: string; + model: string; + resolvedModel: string; + version: string; + bin: string; + }[]; + taskIds: string[]; + /** True when every agent runs the same canonical model slug. Reports must + * carry a prominent caveat when false. */ + matchedModels: boolean; + reproduceCommand: string; +} + +export interface PricingEntry { + /** USD per million tokens. */ + inputPerM: number; + outputPerM: number; + cacheReadPerM: number; + cacheWritePerM?: number; + note?: string; +} + +export type PricingTable = Record; diff --git a/src/workspace.ts b/src/workspace.ts new file mode 100644 index 0000000..4f05f19 --- /dev/null +++ b/src/workspace.ts @@ -0,0 +1,159 @@ +/** + * Workspace lifecycle: seed → agent runs → diff → held-out verify. + * + * The anti-self-grading invariant lives here: the `verify/` directory of a + * task is NEVER present while the agent runs. After the agent finishes, the + * harness deletes anything at `.arena-verify/` (so an agent cannot pre-plant + * its own grader there), copies the held-out tests in, and runs them with + * Node's built-in test runner. Success is decided only by those tests. + */ + +import { execFileSync } from "node:child_process"; +import { + cpSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +import type { LoadedTask, TaskDef } from "./types.js"; +import { isRecord } from "./parse.js"; + +const VERIFY_DIR = ".arena-verify"; + +/** Load tasks// directories under a task root. */ +export function loadTasks(taskRoot: string): LoadedTask[] { + const tasks: LoadedTask[] = []; + for (const entry of readdirSync(taskRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const dir = join(taskRoot, entry.name); + const defPath = join(dir, "task.json"); + if (!existsSync(defPath)) continue; + const def: unknown = JSON.parse(readFileSync(defPath, "utf8")); + if (!isRecord(def) || typeof def["id"] !== "string") { + throw new Error(`Malformed task.json in ${dir}`); + } + if (def["id"] !== entry.name) { + throw new Error( + `Task id "${String(def["id"])}" must match its directory name "${entry.name}"`, + ); + } + tasks.push({ ...(def as unknown as TaskDef), dir }); + } + return tasks.sort((a, b) => a.id.localeCompare(b.id)); +} + +/** The exact prompt every agent receives for a task. */ +export function buildPrompt(task: TaskDef): string { + return [ + `# ${task.name}`, + "", + task.prompt.trim(), + "", + "Work only inside the current directory. Verification is run by the", + "harness after you finish; it asserts exactly the behavior described", + "above. Do not create or modify anything under `.arena-verify/`.", + ].join("\n"); +} + +/** + * Materialize the starting workspace: a copy of the task's `workspace/` + * fixture plus TASK.md, committed to a fresh git repo so `git diff` isolates + * exactly the agent's changes. + */ +export function seedWorkspace(task: LoadedTask, workDir: string): void { + mkdirSync(workDir, { recursive: true }); + cpSync(join(task.dir, "workspace"), workDir, { recursive: true }); + writeFileSync(join(workDir, "TASK.md"), buildPrompt(task) + "\n"); + writeFileSync( + join(workDir, ".gitignore"), + ["node_modules/", "dist/", "coverage/", "*.log", `${VERIFY_DIR}/`].join("\n") + "\n", + ); + const git = (args: string[]): void => { + execFileSync("git", args, { cwd: workDir, stdio: "ignore" }); + }; + git(["init"]); + git(["config", "user.email", "arena@localhost"]); + git(["config", "user.name", "arena"]); + git(["add", "-A"]); + git(["commit", "-m", "arena: seed workspace", "--no-verify"]); +} + +/** Diff of the agent's changes (tracked + new files) against the seed commit. */ +export function collectDiff(workDir: string): string { + try { + execFileSync("git", ["add", "-A"], { cwd: workDir, stdio: "ignore" }); + return execFileSync("git", ["diff", "--cached", "HEAD"], { + cwd: workDir, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + } catch { + return ""; + } +} + +export interface VerifyResult { + passed: boolean; + output: string; +} + +/** + * Run the task's held-out tests inside the workspace with `node --test`. + * Any pre-existing `.arena-verify/` content (agent-planted or stale) is + * removed first. + */ +export function runVerification( + task: LoadedTask, + workDir: string, +): VerifyResult { + const target = join(workDir, VERIFY_DIR); + rmSync(target, { recursive: true, force: true }); + cpSync(join(task.dir, "verify"), target, { recursive: true }); + + const timeoutMs = (task.verifyTimeoutSeconds ?? 60) * 1000; + // node --test needs a glob, not a bare directory (a directory arg is + // treated as a module entry point and fails with MODULE_NOT_FOUND). + const env = { ...process.env, NO_COLOR: "1" }; + delete env.FORCE_COLOR; + try { + const output = execFileSync( + process.execPath, + ["--test", `${VERIFY_DIR}/**/*.test.mjs`], + { + cwd: workDir, + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + env, + }, + ); + return { passed: true, output: tail(output) }; + } catch (error) { + const e = error as { stdout?: string | Buffer; stderr?: string | Buffer; message?: string }; + const out = [ + e.stdout ? e.stdout.toString() : "", + e.stderr ? e.stderr.toString() : "", + e.message ?? "", + ] + .filter(Boolean) + .join("\n"); + return { passed: false, output: tail(out) }; + } finally { + rmSync(target, { recursive: true, force: true }); + } +} + +/** Copy the reference solution over a workspace (used by `arena verify`). */ +export function applySolution(task: LoadedTask, workDir: string): void { + cpSync(join(task.dir, "solution"), workDir, { recursive: true }); +} + +function tail(text: string, maxChars = 4000): string { + const trimmed = text.trim(); + return trimmed.length <= maxChars ? trimmed : "…" + trimmed.slice(-maxChars); +} diff --git a/tasks/bug-merge-intervals/solution/src/intervals.mjs b/tasks/bug-merge-intervals/solution/src/intervals.mjs new file mode 100644 index 0000000..c0a39be --- /dev/null +++ b/tasks/bug-merge-intervals/solution/src/intervals.mjs @@ -0,0 +1,18 @@ +/** + * Merge a list of [start, end] integer intervals. + */ +export function mergeIntervals(intervals) { + const sorted = [...intervals] + .map(([start, end]) => [start, end]) + .sort((a, b) => a[0] - b[0]); + const merged = []; + for (const [start, end] of sorted) { + const last = merged[merged.length - 1]; + if (last && start <= last[1]) { + last[1] = Math.max(last[1], end); + } else { + merged.push([start, end]); + } + } + return merged; +} diff --git a/tasks/bug-merge-intervals/task.json b/tasks/bug-merge-intervals/task.json new file mode 100644 index 0000000..68a1b87 --- /dev/null +++ b/tasks/bug-merge-intervals/task.json @@ -0,0 +1,8 @@ +{ + "id": "bug-merge-intervals", + "name": "Fix interval merging at touching boundaries", + "category": "bug-fix", + "difficulty": "easy", + "prompt": "The function `mergeIntervals` in `src/intervals.mjs` merges a list of `[start, end]` integer intervals, but intervals that merely TOUCH (e.g. `[1, 3]` and `[3, 5]`) are not merged, and they must be.\n\nFix `mergeIntervals` so that:\n- Overlapping intervals merge: `[[1,4],[2,6]]` → `[[1,6]]`.\n- Touching intervals merge: `[[1,3],[3,5]]` → `[[1,5]]`.\n- Disjoint intervals stay separate and the result is sorted by start: `[[5,6],[1,2]]` → `[[1,2],[5,6]]`.\n- Input order must not matter, the input array must not be mutated, and intervals fully contained in another collapse into it: `[[1,10],[2,3]]` → `[[1,10]]`.\n- An empty input returns `[]`.\n\nKeep the exported function name and signature unchanged.", + "tags": ["javascript", "algorithms"] +} diff --git a/tasks/bug-merge-intervals/verify/intervals.test.mjs b/tasks/bug-merge-intervals/verify/intervals.test.mjs new file mode 100644 index 0000000..33a72ac --- /dev/null +++ b/tasks/bug-merge-intervals/verify/intervals.test.mjs @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { mergeIntervals } from "../src/intervals.mjs"; + +test("merges overlapping intervals", () => { + assert.deepEqual(mergeIntervals([[1, 4], [2, 6]]), [[1, 6]]); +}); + +test("merges touching intervals", () => { + assert.deepEqual(mergeIntervals([[1, 3], [3, 5]]), [[1, 5]]); +}); + +test("keeps disjoint intervals separate, sorted by start", () => { + assert.deepEqual(mergeIntervals([[5, 6], [1, 2]]), [[1, 2], [5, 6]]); +}); + +test("collapses contained intervals", () => { + assert.deepEqual(mergeIntervals([[1, 10], [2, 3]]), [[1, 10]]); +}); + +test("handles unsorted mixed input", () => { + assert.deepEqual( + mergeIntervals([[8, 10], [1, 3], [2, 6], [15, 18], [10, 11]]), + [[1, 6], [8, 11], [15, 18]], + ); +}); + +test("does not mutate the input array", () => { + const input = [[3, 4], [1, 2]]; + const snapshot = JSON.stringify(input); + mergeIntervals(input); + assert.equal(JSON.stringify(input), snapshot); +}); + +test("empty input returns empty array", () => { + assert.deepEqual(mergeIntervals([]), []); +}); diff --git a/tasks/bug-merge-intervals/workspace/src/intervals.mjs b/tasks/bug-merge-intervals/workspace/src/intervals.mjs new file mode 100644 index 0000000..0e37c38 --- /dev/null +++ b/tasks/bug-merge-intervals/workspace/src/intervals.mjs @@ -0,0 +1,16 @@ +/** + * Merge a list of [start, end] integer intervals. + */ +export function mergeIntervals(intervals) { + const sorted = intervals.sort((a, b) => a[0] - b[0]); + const merged = []; + for (const [start, end] of sorted) { + const last = merged[merged.length - 1]; + if (last && start < last[1]) { + last[1] = Math.max(last[1], end); + } else { + merged.push([start, end]); + } + } + return merged; +} diff --git a/tasks/bug-paginate/solution/src/paginate.mjs b/tasks/bug-paginate/solution/src/paginate.mjs new file mode 100644 index 0000000..7c87b50 --- /dev/null +++ b/tasks/bug-paginate/solution/src/paginate.mjs @@ -0,0 +1,20 @@ +/** + * Pagination helpers. Pages are 1-based. + */ +function assertPageSize(pageSize) { + if (!Number.isInteger(pageSize) || pageSize < 1) { + throw new RangeError(`pageSize must be a positive integer, got ${pageSize}`); + } +} + +export function pageCount(totalItems, pageSize) { + assertPageSize(pageSize); + return Math.ceil(totalItems / pageSize); +} + +export function paginate(items, page, pageSize) { + assertPageSize(pageSize); + if (page < 1 || page > pageCount(items.length, pageSize)) return []; + const start = (page - 1) * pageSize; + return items.slice(start, start + pageSize); +} diff --git a/tasks/bug-paginate/task.json b/tasks/bug-paginate/task.json new file mode 100644 index 0000000..f5140fb --- /dev/null +++ b/tasks/bug-paginate/task.json @@ -0,0 +1,8 @@ +{ + "id": "bug-paginate", + "name": "Fix pagination page count and range guards", + "category": "bug-fix", + "difficulty": "easy", + "prompt": "`src/paginate.mjs` exports `pageCount(totalItems, pageSize)` and `paginate(items, page, pageSize)`. Two defects must be fixed:\n\n1. `pageCount` drops the final partial page: `pageCount(10, 3)` currently returns 3 but must return 4. The contract: `pageCount` returns the number of pages needed to hold `totalItems` items (`Math.ceil`), returns 0 when `totalItems` is 0, and throws a `RangeError` when `pageSize` is not a positive integer.\n2. `paginate` has no range guards. The contract: pages are 1-based; `paginate` returns the items for that page; it returns an empty array when `page < 1` or `page > pageCount(items.length, pageSize)`; it throws a `RangeError` when `pageSize` is not a positive integer.\n\nKeep both exported function names and signatures unchanged.", + "tags": ["javascript"] +} diff --git a/tasks/bug-paginate/verify/paginate.test.mjs b/tasks/bug-paginate/verify/paginate.test.mjs new file mode 100644 index 0000000..de626c7 --- /dev/null +++ b/tasks/bug-paginate/verify/paginate.test.mjs @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { pageCount, paginate } from "../src/paginate.mjs"; + +const items = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]; + +test("pageCount includes the final partial page", () => { + assert.equal(pageCount(10, 3), 4); + assert.equal(pageCount(9, 3), 3); + assert.equal(pageCount(1, 3), 1); +}); + +test("pageCount of zero items is zero", () => { + assert.equal(pageCount(0, 5), 0); +}); + +test("pageCount rejects non-positive or fractional pageSize", () => { + assert.throws(() => pageCount(10, 0), RangeError); + assert.throws(() => pageCount(10, -2), RangeError); + assert.throws(() => pageCount(10, 2.5), RangeError); +}); + +test("paginate returns the right slice", () => { + assert.deepEqual(paginate(items, 1, 3), ["a", "b", "c"]); + assert.deepEqual(paginate(items, 4, 3), ["j"]); +}); + +test("paginate returns [] for out-of-range pages", () => { + assert.deepEqual(paginate(items, 0, 3), []); + assert.deepEqual(paginate(items, -1, 3), []); + assert.deepEqual(paginate(items, 5, 3), []); +}); + +test("paginate rejects invalid pageSize", () => { + assert.throws(() => paginate(items, 1, 0), RangeError); +}); diff --git a/tasks/bug-paginate/workspace/src/paginate.mjs b/tasks/bug-paginate/workspace/src/paginate.mjs new file mode 100644 index 0000000..bdeb930 --- /dev/null +++ b/tasks/bug-paginate/workspace/src/paginate.mjs @@ -0,0 +1,11 @@ +/** + * Pagination helpers. Pages are 1-based. + */ +export function pageCount(totalItems, pageSize) { + return Math.floor(totalItems / pageSize); +} + +export function paginate(items, page, pageSize) { + const start = (page - 1) * pageSize; + return items.slice(start, start + pageSize); +} diff --git a/tasks/bug-query-string/solution/src/query-string.mjs b/tasks/bug-query-string/solution/src/query-string.mjs new file mode 100644 index 0000000..2f3c7c2 --- /dev/null +++ b/tasks/bug-query-string/solution/src/query-string.mjs @@ -0,0 +1,27 @@ +/** + * Parse a URL query string into a plain object. + */ +function decode(part) { + return decodeURIComponent(part.replace(/\+/g, " ")); +} + +export function parseQueryString(query) { + const stripped = query.startsWith("?") ? query.slice(1) : query; + const result = {}; + for (const pair of stripped.split("&")) { + if (pair === "") continue; + const eq = pair.indexOf("="); + const rawKey = eq === -1 ? pair : pair.slice(0, eq); + const rawValue = eq === -1 ? "" : pair.slice(eq + 1); + const key = decode(rawKey); + const value = decode(rawValue); + if (Object.prototype.hasOwnProperty.call(result, key)) { + const existing = result[key]; + if (Array.isArray(existing)) existing.push(value); + else result[key] = [existing, value]; + } else { + result[key] = value; + } + } + return result; +} diff --git a/tasks/bug-query-string/task.json b/tasks/bug-query-string/task.json new file mode 100644 index 0000000..93f0a84 --- /dev/null +++ b/tasks/bug-query-string/task.json @@ -0,0 +1,8 @@ +{ + "id": "bug-query-string", + "name": "Fix query-string parsing (decoding, repeated keys)", + "category": "bug-fix", + "difficulty": "medium", + "prompt": "`parseQueryString` in `src/query-string.mjs` parses a URL query string into an object, but it neither percent-decodes nor handles repeated keys.\n\nFix it to meet this contract exactly:\n- A leading `?` is ignored: `parseQueryString('?a=1')` → `{ a: '1' }`.\n- Keys and values are percent-decoded, and `+` decodes to a space: `'q=hello+w%C3%B6rld'` → `{ q: 'hello wörld' }`. Decoding applies to keys too.\n- A key that appears once maps to its string value. A key that appears multiple times maps to an ARRAY of its values in order of appearance: `'a=1&a=2&b=3'` → `{ a: ['1', '2'], b: '3' }`.\n- A key with no `=` maps to the empty string: `'flag&a=1'` → `{ flag: '', a: '1' }`. A key with a trailing `=` also maps to the empty string.\n- Values may contain `=`: `'a=b=c'` → `{ a: 'b=c' }` (split on the FIRST `=` only).\n- Empty segments are skipped: `'a=1&&b=2'` → `{ a: '1', b: '2' }`. The empty string and `'?'` both return `{}`.\n\nKeep the exported function name and signature unchanged.", + "tags": ["javascript", "parsing"] +} diff --git a/tasks/bug-query-string/verify/query-string.test.mjs b/tasks/bug-query-string/verify/query-string.test.mjs new file mode 100644 index 0000000..0c6eb49 --- /dev/null +++ b/tasks/bug-query-string/verify/query-string.test.mjs @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { parseQueryString } from "../src/query-string.mjs"; + +test("basic pairs", () => { + assert.deepEqual(parseQueryString("a=1&b=2"), { a: "1", b: "2" }); +}); + +test("leading ? is ignored", () => { + assert.deepEqual(parseQueryString("?a=1"), { a: "1" }); +}); + +test("percent-decodes keys and values, + is space", () => { + assert.deepEqual(parseQueryString("q=hello+w%C3%B6rld"), { q: "hello wörld" }); + assert.deepEqual(parseQueryString("my+key=v%20x"), { "my key": "v x" }); +}); + +test("repeated keys become arrays in order", () => { + assert.deepEqual(parseQueryString("a=1&a=2&b=3"), { a: ["1", "2"], b: "3" }); + assert.deepEqual(parseQueryString("a=1&a=2&a=3"), { a: ["1", "2", "3"] }); +}); + +test("key without = maps to empty string", () => { + assert.deepEqual(parseQueryString("flag&a=1"), { flag: "", a: "1" }); + assert.deepEqual(parseQueryString("a="), { a: "" }); +}); + +test("splits on first = only", () => { + assert.deepEqual(parseQueryString("a=b=c"), { a: "b=c" }); +}); + +test("empty segments skipped; empty inputs give {}", () => { + assert.deepEqual(parseQueryString("a=1&&b=2"), { a: "1", b: "2" }); + assert.deepEqual(parseQueryString(""), {}); + assert.deepEqual(parseQueryString("?"), {}); +}); diff --git a/tasks/bug-query-string/workspace/src/query-string.mjs b/tasks/bug-query-string/workspace/src/query-string.mjs new file mode 100644 index 0000000..975688b --- /dev/null +++ b/tasks/bug-query-string/workspace/src/query-string.mjs @@ -0,0 +1,11 @@ +/** + * Parse a URL query string into a plain object. + */ +export function parseQueryString(query) { + const result = {}; + for (const pair of query.split("&")) { + const [key, value] = pair.split("="); + result[key] = value ?? ""; + } + return result; +} diff --git a/tasks/feature-lru-cache/solution/src/lru-cache.mjs b/tasks/feature-lru-cache/solution/src/lru-cache.mjs new file mode 100644 index 0000000..a1caedf --- /dev/null +++ b/tasks/feature-lru-cache/solution/src/lru-cache.mjs @@ -0,0 +1,44 @@ +/** + * LRU cache factory. + * + * createLruCache(maxSize) -> { get, set, has, size, keys } + */ +export function createLruCache(maxSize) { + if (!Number.isInteger(maxSize) || maxSize < 1) { + throw new TypeError(`maxSize must be a positive integer, got ${maxSize}`); + } + + // Map iteration order is insertion order; re-inserting moves a key to the + // end, so the first key is always the least recently used. + const store = new Map(); + + function touch(key) { + const value = store.get(key); + store.delete(key); + store.set(key, value); + } + + return { + get(key) { + if (!store.has(key)) return undefined; + touch(key); + return store.get(key); + }, + set(key, value) { + if (store.has(key)) store.delete(key); + else if (store.size >= maxSize) { + store.delete(store.keys().next().value); + } + store.set(key, value); + }, + has(key) { + return store.has(key); + }, + size() { + return store.size; + }, + keys() { + return [...store.keys()]; + }, + }; +} diff --git a/tasks/feature-lru-cache/task.json b/tasks/feature-lru-cache/task.json new file mode 100644 index 0000000..d52c9cc --- /dev/null +++ b/tasks/feature-lru-cache/task.json @@ -0,0 +1,8 @@ +{ + "id": "feature-lru-cache", + "name": "Implement an LRU cache", + "category": "feature", + "difficulty": "medium", + "prompt": "Implement `createLruCache` in `src/lru-cache.mjs` (currently a stub that throws). Contract:\n\n`createLruCache(maxSize)` returns an object with `get(key)`, `set(key, value)`, `has(key)`, `size()`, and `keys()`.\n\n- `maxSize` must be a positive integer; otherwise `createLruCache` throws a `TypeError`.\n- `get(key)` returns the stored value, or `undefined` when absent. A hit marks the key as MOST recently used.\n- `set(key, value)` inserts or updates, marking the key most recently used. When inserting a NEW key would exceed `maxSize`, the LEAST recently used key is evicted first.\n- `has(key)` returns a boolean and does NOT affect recency.\n- `size()` returns the current entry count.\n- `keys()` returns an array of keys ordered from least recently used to most recently used.\n- Any value is storable, including `undefined` (`has` distinguishes a stored `undefined` from an absent key).\n\nKeep the exported function name and signature unchanged. Do not add new dependencies.", + "tags": ["javascript", "data-structures"] +} diff --git a/tasks/feature-lru-cache/verify/lru-cache.test.mjs b/tasks/feature-lru-cache/verify/lru-cache.test.mjs new file mode 100644 index 0000000..fbee494 --- /dev/null +++ b/tasks/feature-lru-cache/verify/lru-cache.test.mjs @@ -0,0 +1,74 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { createLruCache } from "../src/lru-cache.mjs"; + +test("stores and retrieves values", () => { + const cache = createLruCache(2); + cache.set("a", 1); + assert.equal(cache.get("a"), 1); + assert.equal(cache.get("missing"), undefined); +}); + +test("evicts the least recently used key on overflow", () => { + const cache = createLruCache(2); + cache.set("a", 1); + cache.set("b", 2); + cache.set("c", 3); + assert.equal(cache.has("a"), false); + assert.equal(cache.get("b"), 2); + assert.equal(cache.get("c"), 3); +}); + +test("get marks a key most recently used", () => { + const cache = createLruCache(2); + cache.set("a", 1); + cache.set("b", 2); + cache.get("a"); // now b is LRU + cache.set("c", 3); + assert.equal(cache.has("b"), false); + assert.equal(cache.has("a"), true); +}); + +test("set on an existing key updates value and recency without eviction", () => { + const cache = createLruCache(2); + cache.set("a", 1); + cache.set("b", 2); + cache.set("a", 10); // now b is LRU + cache.set("c", 3); + assert.equal(cache.get("a"), 10); + assert.equal(cache.has("b"), false); + assert.equal(cache.size(), 2); +}); + +test("has does not affect recency", () => { + const cache = createLruCache(2); + cache.set("a", 1); + cache.set("b", 2); + cache.has("a"); // must NOT promote a + cache.set("c", 3); + assert.equal(cache.has("a"), false, "a stayed LRU and was evicted"); +}); + +test("keys() orders least → most recently used", () => { + const cache = createLruCache(3); + cache.set("a", 1); + cache.set("b", 2); + cache.set("c", 3); + cache.get("a"); + assert.deepEqual(cache.keys(), ["b", "c", "a"]); +}); + +test("stored undefined is distinguishable from absent", () => { + const cache = createLruCache(2); + cache.set("u", undefined); + assert.equal(cache.has("u"), true); + assert.equal(cache.get("u"), undefined); +}); + +test("validates maxSize", () => { + assert.throws(() => createLruCache(0), TypeError); + assert.throws(() => createLruCache(-1), TypeError); + assert.throws(() => createLruCache(1.5), TypeError); + assert.throws(() => createLruCache("3"), TypeError); +}); diff --git a/tasks/feature-lru-cache/workspace/src/lru-cache.mjs b/tasks/feature-lru-cache/workspace/src/lru-cache.mjs new file mode 100644 index 0000000..9e816b0 --- /dev/null +++ b/tasks/feature-lru-cache/workspace/src/lru-cache.mjs @@ -0,0 +1,9 @@ +/** + * LRU cache factory. + * + * createLruCache(maxSize) -> { get, set, has, size, keys } + */ +export function createLruCache(maxSize) { + void maxSize; + throw new Error("not implemented"); +} diff --git a/tasks/feature-rate-limiter/solution/src/rate-limiter.mjs b/tasks/feature-rate-limiter/solution/src/rate-limiter.mjs new file mode 100644 index 0000000..c0ae421 --- /dev/null +++ b/tasks/feature-rate-limiter/solution/src/rate-limiter.mjs @@ -0,0 +1,40 @@ +/** + * Token-bucket rate limiter. + * + * createRateLimiter({ capacity, refillPerSecond, now }) -> { tryRemove(count) } + */ +export function createRateLimiter({ + capacity, + refillPerSecond, + now = () => Date.now() / 1000, +} = {}) { + if (!Number.isFinite(capacity) || capacity <= 0) { + throw new RangeError(`capacity must be a positive finite number, got ${capacity}`); + } + if (!Number.isFinite(refillPerSecond) || refillPerSecond < 0) { + throw new RangeError( + `refillPerSecond must be a non-negative finite number, got ${refillPerSecond}`, + ); + } + + let tokens = capacity; + let lastRefill = now(); + + function refill() { + const current = now(); + const elapsed = Math.max(0, current - lastRefill); + tokens = Math.min(capacity, tokens + elapsed * refillPerSecond); + lastRefill = current; + } + + return { + tryRemove(count = 1) { + refill(); + if (tokens >= count) { + tokens -= count; + return true; + } + return false; + }, + }; +} diff --git a/tasks/feature-rate-limiter/task.json b/tasks/feature-rate-limiter/task.json new file mode 100644 index 0000000..d89c06f --- /dev/null +++ b/tasks/feature-rate-limiter/task.json @@ -0,0 +1,8 @@ +{ + "id": "feature-rate-limiter", + "name": "Implement a token-bucket rate limiter", + "category": "feature", + "difficulty": "medium", + "prompt": "Implement `createRateLimiter` in `src/rate-limiter.mjs` (currently a stub that throws). Contract:\n\n`createRateLimiter({ capacity, refillPerSecond, now })` returns an object with a single method `tryRemove(count = 1)`.\n\n- `capacity` (positive number): maximum tokens the bucket holds. The bucket STARTS FULL.\n- `refillPerSecond` (non-negative number): tokens added per second, continuously (fractional accrual — after 0.5s at 2 tokens/sec, 1 token has accrued). The bucket never exceeds `capacity`.\n- `now` (function, defaults to `() => Date.now() / 1000`): returns the current time in SECONDS. All time arithmetic must go through it.\n- `tryRemove(count)`: if at least `count` tokens are currently available (after accruing refill since the last call), deduct them and return `true`; otherwise deduct NOTHING and return `false`. `tryRemove()` defaults to 1 token.\n- `createRateLimiter` throws a `RangeError` if `capacity` is not a positive finite number or `refillPerSecond` is negative or not finite.\n\nKeep the exported function name and signature unchanged. Do not add new dependencies.", + "tags": ["javascript", "feature"] +} diff --git a/tasks/feature-rate-limiter/verify/rate-limiter.test.mjs b/tasks/feature-rate-limiter/verify/rate-limiter.test.mjs new file mode 100644 index 0000000..c842ab4 --- /dev/null +++ b/tasks/feature-rate-limiter/verify/rate-limiter.test.mjs @@ -0,0 +1,60 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { createRateLimiter } from "../src/rate-limiter.mjs"; + +function fakeClock(start = 0) { + let t = start; + return { + now: () => t, + advance: (seconds) => { + t += seconds; + }, + }; +} + +test("bucket starts full", () => { + const clock = fakeClock(); + const limiter = createRateLimiter({ capacity: 3, refillPerSecond: 1, now: clock.now }); + assert.equal(limiter.tryRemove(3), true); + assert.equal(limiter.tryRemove(1), false); +}); + +test("failed tryRemove deducts nothing", () => { + const clock = fakeClock(); + const limiter = createRateLimiter({ capacity: 2, refillPerSecond: 0, now: clock.now }); + assert.equal(limiter.tryRemove(1), true); + assert.equal(limiter.tryRemove(2), false); + assert.equal(limiter.tryRemove(1), true, "the failed attempt must not have consumed the remaining token"); +}); + +test("refills continuously with fractional accrual", () => { + const clock = fakeClock(); + const limiter = createRateLimiter({ capacity: 2, refillPerSecond: 2, now: clock.now }); + assert.equal(limiter.tryRemove(2), true); + clock.advance(0.5); // accrues 1 token + assert.equal(limiter.tryRemove(1), true); + assert.equal(limiter.tryRemove(1), false); +}); + +test("never exceeds capacity", () => { + const clock = fakeClock(); + const limiter = createRateLimiter({ capacity: 2, refillPerSecond: 10, now: clock.now }); + clock.advance(100); + assert.equal(limiter.tryRemove(2), true); + assert.equal(limiter.tryRemove(1), false); +}); + +test("tryRemove defaults to 1", () => { + const clock = fakeClock(); + const limiter = createRateLimiter({ capacity: 1, refillPerSecond: 0, now: clock.now }); + assert.equal(limiter.tryRemove(), true); + assert.equal(limiter.tryRemove(), false); +}); + +test("validates options", () => { + assert.throws(() => createRateLimiter({ capacity: 0, refillPerSecond: 1 }), RangeError); + assert.throws(() => createRateLimiter({ capacity: -1, refillPerSecond: 1 }), RangeError); + assert.throws(() => createRateLimiter({ capacity: Infinity, refillPerSecond: 1 }), RangeError); + assert.throws(() => createRateLimiter({ capacity: 1, refillPerSecond: -1 }), RangeError); +}); diff --git a/tasks/feature-rate-limiter/workspace/src/rate-limiter.mjs b/tasks/feature-rate-limiter/workspace/src/rate-limiter.mjs new file mode 100644 index 0000000..3af17a3 --- /dev/null +++ b/tasks/feature-rate-limiter/workspace/src/rate-limiter.mjs @@ -0,0 +1,9 @@ +/** + * Token-bucket rate limiter. + * + * createRateLimiter({ capacity, refillPerSecond, now }) -> { tryRemove(count) } + */ +export function createRateLimiter(options) { + void options; + throw new Error("not implemented"); +} diff --git a/tasks/robustness-fetch-users/solution/src/users.mjs b/tasks/robustness-fetch-users/solution/src/users.mjs new file mode 100644 index 0000000..57e73ba --- /dev/null +++ b/tasks/robustness-fetch-users/solution/src/users.mjs @@ -0,0 +1,25 @@ +const BASE_URL = "https://api.example.com"; + +/** + * Fetch a user by id. Never rejects: all failures resolve to { ok: false }. + */ +export async function getUser(id, fetchImpl = fetch) { + try { + const response = await fetchImpl(`${BASE_URL}/users/${id}`); + if (!response.ok) { + return { + ok: false, + status: response.status, + error: `Request failed with status ${response.status}`, + }; + } + const data = await response.json(); + return { ok: true, status: response.status, data }; + } catch (error) { + return { + ok: false, + status: 0, + error: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/tasks/robustness-fetch-users/task.json b/tasks/robustness-fetch-users/task.json new file mode 100644 index 0000000..aa327b8 --- /dev/null +++ b/tasks/robustness-fetch-users/task.json @@ -0,0 +1,8 @@ +{ + "id": "robustness-fetch-users", + "name": "Add error handling to an unguarded API call", + "category": "robustness", + "difficulty": "medium", + "prompt": "`getUser` in `src/users.mjs` fetches a user from an API with NO error handling: a non-2xx response or a network/parse failure escapes as an unhandled rejection. Add proper error handling with this exact contract (`fetchImpl` stays an injectable parameter defaulting to global `fetch`):\n\n- Success (response.ok true, body parses as JSON): resolve to `{ ok: true, status: , data: }`.\n- HTTP error (response.ok false): resolve to `{ ok: false, status: , error: }`. The body must NOT be parsed in this case.\n- Thrown failure (fetch rejects, or the body fails to parse as JSON): resolve to `{ ok: false, status: 0, error: }`.\n- The function must never reject and never throw synchronously.\n\nKeep the exported function name and signature (`getUser(id, fetchImpl = fetch)`) unchanged. Do not add new dependencies.", + "tags": ["javascript", "error-handling"] +} diff --git a/tasks/robustness-fetch-users/verify/users.test.mjs b/tasks/robustness-fetch-users/verify/users.test.mjs new file mode 100644 index 0000000..a476228 --- /dev/null +++ b/tasks/robustness-fetch-users/verify/users.test.mjs @@ -0,0 +1,68 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { getUser } from "../src/users.mjs"; + +test("success resolves with ok, status, and parsed data", async () => { + const user = { id: 7, name: "Ada" }; + const fetchImpl = async () => ({ + ok: true, + status: 200, + json: async () => user, + }); + assert.deepEqual(await getUser(7, fetchImpl), { + ok: true, + status: 200, + data: user, + }); +}); + +test("HTTP error resolves with ok:false, real status, non-empty error; body not parsed", async () => { + let jsonCalled = false; + const fetchImpl = async () => ({ + ok: false, + status: 404, + json: async () => { + jsonCalled = true; + return {}; + }, + }); + const result = await getUser(999, fetchImpl); + assert.equal(result.ok, false); + assert.equal(result.status, 404); + assert.equal(typeof result.error, "string"); + assert.ok(result.error.length > 0); + assert.equal(jsonCalled, false, "body must not be parsed on HTTP error"); +}); + +test("network failure resolves with ok:false, status 0", async () => { + const fetchImpl = async () => { + throw new Error("connection reset"); + }; + const result = await getUser(1, fetchImpl); + assert.deepEqual( + { ok: result.ok, status: result.status }, + { ok: false, status: 0 }, + ); + assert.ok(result.error.length > 0); +}); + +test("JSON parse failure resolves with ok:false, status 0", async () => { + const fetchImpl = async () => ({ + ok: true, + status: 200, + json: async () => { + throw new SyntaxError("Unexpected token < in JSON"); + }, + }); + const result = await getUser(1, fetchImpl); + assert.equal(result.ok, false); + assert.equal(result.status, 0); +}); + +test("never rejects", async () => { + const fetchImpl = async () => { + throw new Error("boom"); + }; + await assert.doesNotReject(() => getUser(1, fetchImpl)); +}); diff --git a/tasks/robustness-fetch-users/workspace/src/users.mjs b/tasks/robustness-fetch-users/workspace/src/users.mjs new file mode 100644 index 0000000..2e5a037 --- /dev/null +++ b/tasks/robustness-fetch-users/workspace/src/users.mjs @@ -0,0 +1,10 @@ +const BASE_URL = "https://api.example.com"; + +/** + * Fetch a user by id. + */ +export async function getUser(id, fetchImpl = fetch) { + const response = await fetchImpl(`${BASE_URL}/users/${id}`); + const data = await response.json(); + return { ok: true, status: response.status, data }; +} diff --git a/test/adapters.test.ts b/test/adapters.test.ts new file mode 100644 index 0000000..6113856 --- /dev/null +++ b/test/adapters.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; + +import { ClaudeCodeAdapter } from "../src/adapters/claude-code.js"; +import { GeminiAdapter } from "../src/adapters/gemini.js"; +import { OxagenAdapter } from "../src/adapters/oxagen.js"; +import { StellaAdapter } from "../src/adapters/stella.js"; +import { adapterNames, createAdapter } from "../src/adapters/index.js"; +import type { AdapterRunArgs } from "../src/adapters/base.js"; + +const baseArgs: AdapterRunArgs = { + prompt: "fix the bug", + model: "anthropic/claude-sonnet-5", + budgetUsd: 5, + timeoutSeconds: 600, + workDir: "/tmp/x", +}; + +describe("registry", () => { + it("creates every registered adapter and rejects unknown ones", () => { + for (const name of adapterNames()) { + expect(createAdapter({ adapter: name, model: "m" }).name).toBe(name); + } + expect(() => createAdapter({ adapter: "nope", model: "m" })).toThrow(/Unknown adapter/); + }); +}); + +describe("claude-code", () => { + const adapter = new ClaudeCodeAdapter(); + + it("maps slugs to family aliases", () => { + expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe("sonnet"); + expect(adapter.resolveModel("anthropic/claude-opus-4.8")).toBe("opus"); + expect(adapter.resolveModel("weird-model")).toBe("weird-model"); + }); + + it("builds print-mode argv with budget", () => { + expect(adapter.buildArgs(baseArgs)).toEqual([ + "-p", + "--output-format", + "json", + "--model", + "sonnet", + "--permission-mode", + "acceptEdits", + "--max-budget-usd", + "5", + "fix the bug", + ]); + }); + + it("normalizes the result envelope (input already excludes cache reads)", () => { + const stdout = JSON.stringify({ + type: "result", + total_cost_usd: 0.12, + duration_ms: 30000, + num_turns: 6, + usage: { + input_tokens: 1000, + output_tokens: 400, + cache_read_input_tokens: 9000, + cache_creation_input_tokens: 500, + }, + }); + const parsed = adapter.parseEnvelope(stdout); + expect(parsed.tokens).toEqual({ + input: 1000, + output: 400, + cacheRead: 9000, + cacheWrite: 500, + total: 10900, + }); + expect(parsed.agentReportedUsd).toBe(0.12); + expect(parsed.agentReportedSeconds).toBe(30); + expect(parsed.iterations).toBe(6); + }); +}); + +describe("oxagen", () => { + const adapter = new OxagenAdapter(); + + it("builds one-shot argv with -- separator", () => { + expect(adapter.buildArgs(baseArgs)).toEqual([ + "--local", + "--output-format", + "json", + "--model", + "anthropic/claude-sonnet-5", + "--budget", + "5", + "--", + "fix the bug", + ]); + }); + + it("subtracts cached tokens from combined input (normalization)", () => { + const stdout = JSON.stringify({ + type: "result", + steps: 24, + durationMs: 94918, + usage: { inputTokens: 277032, outputTokens: 8432, cachedInputTokens: 244884 }, + commandsRun: ["a", "b", "c"], + }); + const parsed = adapter.parseEnvelope(stdout); + expect(parsed.tokens.input).toBe(277032 - 244884); + expect(parsed.tokens.cacheRead).toBe(244884); + expect(parsed.tokens.total).toBe(277032 + 8432); + expect(parsed.toolCalls).toBe(3); + expect(parsed.iterations).toBe(24); + }); +}); + +describe("stella", () => { + const adapter = new StellaAdapter(); + + it("prefixes bare model ids with zai/", () => { + expect(adapter.resolveModel("glm-5.2")).toBe("zai/glm-5.2"); + expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe( + "anthropic/claude-sonnet-5", + ); + }); + + it("passes config via env, prompt via `run`", () => { + expect(adapter.buildArgs(baseArgs)).toEqual(["run", "fix the bug"]); + const env = adapter.env(baseArgs); + expect(env.STELLA_OUTPUT_FORMAT).toBe("json"); + expect(env.STELLA_MODEL).toBe("anthropic/claude-sonnet-5"); + expect(env.STELLA_BUDGET).toBe("5"); + }); + + it("reads both camelCase and snake_case envelopes", () => { + const parsed = adapter.parseEnvelope( + JSON.stringify({ + type: "result", + usage: { input_tokens: 900, output_tokens: 100, cache_read_tokens: 400 }, + cost_usd: 0.05, + }), + ); + expect(parsed.tokens.input).toBe(500); + expect(parsed.tokens.cacheRead).toBe(400); + expect(parsed.agentReportedUsd).toBe(0.05); + }); +}); + +describe("gemini", () => { + const adapter = new GeminiAdapter(); + + it("strips the google/ prefix and uses auto_edit approval", () => { + const argv = adapter.buildArgs({ ...baseArgs, model: "google/gemini-2.5-pro" }); + expect(argv).toContain("gemini-2.5-pro"); + expect(argv).toContain("auto_edit"); + }); + + it("sums per-model token stats and subtracts cached from prompt", () => { + const stdout = JSON.stringify({ + response: "done", + stats: { + models: { + "gemini-2.5-pro": { tokens: { prompt: 5000, candidates: 700, cached: 2000, total: 5700 } }, + "gemini-2.5-flash": { tokens: { prompt: 100, candidates: 20, cached: 0, total: 120 } }, + }, + tools: { totalCalls: 9 }, + }, + }); + const parsed = adapter.parseEnvelope(stdout); + expect(parsed.tokens.input).toBe(5100 - 2000); + expect(parsed.tokens.cacheRead).toBe(2000); + expect(parsed.tokens.output).toBe(720); + expect(parsed.toolCalls).toBe(9); + }); +}); diff --git a/test/parse.test.ts b/test/parse.test.ts new file mode 100644 index 0000000..c68fce8 --- /dev/null +++ b/test/parse.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; + +import { countOf, diffStats, num, parseJsonEnvelope, sanitizeSegment } from "../src/parse.js"; + +describe("parseJsonEnvelope", () => { + it("returns null for empty/non-JSON output", () => { + expect(parseJsonEnvelope("")).toBeNull(); + expect(parseJsonEnvelope("plain text banner\nno json here")).toBeNull(); + }); + + it("prefers the last type:result object in JSONL", () => { + const stdout = [ + '{"type":"event","n":1}', + '{"type":"result","n":2}', + '{"type":"event","n":3}', + ].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", n: 2 }); + }); + + it("falls back to the last parseable object", () => { + const stdout = ['not json', '{"a":1}', '{"b":2}'].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ b: 2 }); + }); + + it("ignores log lines interleaved with JSON", () => { + const stdout = ['INFO starting', '{"type":"result","ok":true}', 'INFO done'].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", ok: true }); + }); +}); + +describe("num / countOf", () => { + it("num coerces safely", () => { + expect(num(5)).toBe(5); + expect(num(-1)).toBe(0); // negative counts rejected + expect(num("5")).toBe(0); + expect(num(NaN, 7)).toBe(7); + }); + + it("countOf handles arrays, numbers, and absence", () => { + expect(countOf([1, 2, 3])).toBe(3); + expect(countOf(4)).toBe(4); + expect(countOf(undefined)).toBeNull(); + }); +}); + +describe("sanitizeSegment", () => { + it("flattens model slugs into safe filenames", () => { + expect(sanitizeSegment("anthropic/claude-opus-4.8")).toBe( + "anthropic_claude-opus-4.8", + ); + }); +}); + +describe("diffStats", () => { + it("counts files and +/- lines, excluding headers", () => { + const diff = [ + "diff --git a/x.mjs b/x.mjs", + "--- a/x.mjs", + "+++ b/x.mjs", + "@@ -1,2 +1,3 @@", + "-old line", + "+new line", + "+another line", + " context", + ].join("\n"); + expect(diffStats(diff)).toEqual({ + filesTouched: 1, + linesAdded: 2, + linesRemoved: 1, + }); + }); +}); diff --git a/test/pipeline.test.ts b/test/pipeline.test.ts new file mode 100644 index 0000000..d13fe0c --- /dev/null +++ b/test/pipeline.test.ts @@ -0,0 +1,147 @@ +/** + * End-to-end pipeline test with the mock adapter: no API keys, no network. + * Exercises workspace seeding, held-out verification, agent-error detection, + * result persistence, and report generation. + */ + +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, describe, expect, it } from "vitest"; + +import { executeRun } from "../src/orchestrator.js"; +import { generateReport } from "../src/report.js"; +import { + applySolution, + loadTasks, + runVerification, + seedWorkspace, +} from "../src/workspace.js"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const TASK_ROOT = join(HERE, "..", "tasks"); + +const scratchDirs: string[] = []; +function scratch(): string { + const dir = mkdtempSync(join(tmpdir(), "arena-test-")); + scratchDirs.push(dir); + return dir; +} +afterAll(() => { + for (const dir of scratchDirs) rmSync(dir, { recursive: true, force: true }); +}); + +describe("task fixtures", () => { + const tasks = loadTasks(TASK_ROOT); + + it("loads all tasks with matching ids", () => { + expect(tasks.length).toBeGreaterThanOrEqual(6); + for (const task of tasks) { + expect(task.dir.endsWith(task.id)).toBe(true); + } + }); + + it("every task discriminates: pristine fails, solution passes", () => { + for (const task of tasks) { + const pristineDir = scratch(); + seedWorkspace(task, pristineDir); + const pristine = runVerification(task, pristineDir); + expect(pristine.passed, `${task.id}: held-out tests must FAIL on the pristine workspace`).toBe(false); + + const solvedDir = scratch(); + seedWorkspace(task, solvedDir); + applySolution(task, solvedDir); + const solved = runVerification(task, solvedDir); + expect(solved.passed, `${task.id}: reference solution must pass:\n${solved.output}`).toBe(true); + } + }, 300_000); + + it("verification removes agent-planted .arena-verify content", () => { + const task = loadTasks(TASK_ROOT)[0]!; + const dir = scratch(); + seedWorkspace(task, dir); + // Simulate an adversarial agent planting a trivially-green test suite. + const planted = join(dir, ".arena-verify"); + mkdirSync(planted, { recursive: true }); + writeFileSync( + join(planted, "fake.test.mjs"), + 'import { test } from "node:test"; test("ok", () => {});', + ); + const result = runVerification(task, dir); + expect(result.passed).toBe(false); + }); +}); + +describe("full run with mock agents", () => { + it("scores solve vs fail correctly and generates a defensible report", async () => { + const outDir = scratch(); + const tasks = loadTasks(TASK_ROOT).slice(0, 2); + + const { runDir, manifest, results } = await executeRun({ + agents: [ + { adapter: "mock", model: "solve" }, + { adapter: "mock", model: "fail" }, + ], + tasks, + trials: 2, + timeoutSeconds: 60, + seed: 42, + outDir, + }); + + expect(results).toHaveLength(2 * 2 * 2); // tasks × agents × trials + + const solved = results.filter((r) => r.agent.model === "solve"); + const failed = results.filter((r) => r.agent.model === "fail"); + expect(solved.every((r) => r.outcome === "passed")).toBe(true); + expect(failed.every((r) => r.outcome === "failed")).toBe(true); + + // Receipts exist for every trial. + for (const r of results) { + expect(existsSync(join(runDir, r.transcriptPath))).toBe(true); + expect(existsSync(join(runDir, r.diffPath))).toBe(true); + } + // The solving agent produced a real diff; the failing agent produced none. + expect(solved[0]!.activity.filesTouched).toBeGreaterThan(0); + expect(failed[0]!.activity.filesTouched).toBe(0); + + // Manifest flags the unmatched "models" (solve vs fail differ). + expect(manifest.matchedModels).toBe(false); + expect(manifest.reproduceCommand).toContain("--seed 42"); + + const report = generateReport(runDir); + expect(report).toContain("Unmatched models"); + expect(report).toContain("McNemar"); + expect(report).toContain(manifest.reproduceCommand); + + const persisted = JSON.parse(readFileSync(join(runDir, "results.json"), "utf8")); + expect(persisted.results).toHaveLength(results.length); + }, 300_000); + + it("marks an uninvocable agent as agent-error, not failed", async () => { + const outDir = scratch(); + const tasks = loadTasks(TASK_ROOT).slice(0, 1); + + // stella with a bin override pointing at a nonexistent binary would fail + // isAvailable(); instead simulate via a mock whose execute spawn errors by + // overriding bin to a missing path through the real spawn path. + const { StellaAdapter } = await import("../src/adapters/stella.js"); + const adapter = new StellaAdapter({ bin: "/nonexistent/definitely-missing" }); + const outcome = await adapter.execute({ + prompt: "x", + model: "zai/glm-5.2", + budgetUsd: undefined, + timeoutSeconds: 5, + workDir: scratch(), + }); + expect(outcome.spawnError).toBeDefined(); + }); +}); diff --git a/test/stats.test.ts b/test/stats.test.ts new file mode 100644 index 0000000..eda8c3d --- /dev/null +++ b/test/stats.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; + +import { + mcnemarExact, + median, + mulberry32, + pairedBootstrapDelta, + wilsonInterval, +} from "../src/stats.js"; + +describe("wilsonInterval", () => { + it("is [0,0] for n=0", () => { + expect(wilsonInterval(0, 0)).toEqual([0, 0]); + }); + + it("brackets the point estimate and stays in [0,1]", () => { + const [lo, hi] = wilsonInterval(8, 10); + expect(lo).toBeGreaterThan(0); + expect(lo).toBeLessThan(0.8); + expect(hi).toBeGreaterThan(0.8); + expect(hi).toBeLessThanOrEqual(1); + }); + + it("matches a known reference value (5/10 → ~[0.237, 0.763])", () => { + const [lo, hi] = wilsonInterval(5, 10); + expect(lo).toBeCloseTo(0.2366, 3); + expect(hi).toBeCloseTo(0.7634, 3); + }); + + it("narrows as n grows", () => { + const [lo10, hi10] = wilsonInterval(5, 10); + const [lo100, hi100] = wilsonInterval(50, 100); + expect(hi100 - lo100).toBeLessThan(hi10 - lo10); + }); +}); + +describe("mcnemarExact", () => { + it("is null with no discordant pairs", () => { + expect(mcnemarExact(0, 0)).toBeNull(); + }); + + it("is 1 for perfectly balanced discordance", () => { + // b=1, c=1: two-sided p = 2 * P(X<=1 | n=2) = 2 * 0.75 → capped at 1 + expect(mcnemarExact(1, 1)).toBe(1); + }); + + it("matches exact binomial for a lopsided split", () => { + // b=8, c=0: p = 2 * P(X<=0 | n=8, 0.5) = 2 * (1/256) = 1/128 + expect(mcnemarExact(8, 0)).toBeCloseTo(2 / 256, 10); + }); + + it("is symmetric in b and c", () => { + expect(mcnemarExact(6, 2)).toBeCloseTo(mcnemarExact(2, 6) as number, 12); + }); +}); + +describe("median", () => { + it("handles odd and even lengths", () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 2, 3])).toBe(2.5); + }); +}); + +describe("mulberry32", () => { + it("is deterministic for a given seed", () => { + const a = mulberry32(42); + const b = mulberry32(42); + expect([a(), a(), a()]).toEqual([b(), b(), b()]); + }); +}); + +describe("pairedBootstrapDelta", () => { + it("is null for too-small samples", () => { + expect(pairedBootstrapDelta([1, 2], [1, 2], 42)).toBeNull(); + }); + + it("finds a clear reduction with a CI excluding zero", () => { + const a = [100, 110, 95, 105, 102, 98, 107, 103]; + const b = a.map((x) => x * 0.7); // 30% less across the board + const delta = pairedBootstrapDelta(a, b, 42); + expect(delta).not.toBeNull(); + expect(delta!.relativeDelta).toBeCloseTo(-0.3, 5); + expect(delta!.ci[1]).toBeLessThan(0); + }); + + it("is deterministic given the same seed", () => { + const a = [10, 12, 9, 11, 10.5, 9.5]; + const b = [9, 13, 8, 12, 10, 9]; + expect(pairedBootstrapDelta(a, b, 7)).toEqual(pairedBootstrapDelta(a, b, 7)); + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..4ed7adb --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noEmit": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "esModuleInterop": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..e84f75e --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts"], + // Pipeline tests spawn git + node subprocesses per task workspace. + testTimeout: 120_000, + }, +}); From 68893b22ca16c4b1576b652e38bc623f80cfb63f Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 13:20:31 -0700 Subject: [PATCH 02/18] fix: type-safe FORCE_COLOR removal in verify env --- src/workspace.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/workspace.ts b/src/workspace.ts index 4f05f19..90c46c0 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -118,8 +118,9 @@ export function runVerification( const timeoutMs = (task.verifyTimeoutSeconds ?? 60) * 1000; // node --test needs a glob, not a bare directory (a directory arg is // treated as a module entry point and fails with MODULE_NOT_FOUND). - const env = { ...process.env, NO_COLOR: "1" }; - delete env.FORCE_COLOR; + // FORCE_COLOR (injected by pnpm) overrides NO_COLOR, so drop it. + const { FORCE_COLOR: _forceColor, ...baseEnv } = process.env; + const env = { ...baseEnv, NO_COLOR: "1" }; try { const output = execFileSync( process.execPath, From 9327127fcdedec5eb0d27d3c91b68b009aae187b Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 13:46:40 -0700 Subject: [PATCH 03/18] chore: rename to arena and point at oxageninc org Repo moved to github.com/oxageninc/arena. Updates the package name, repository URL, README title + clone command, harness self-name in the run manifest and CLI banner, and the Stella link (now oxageninc/stella). --- README.md | 8 ++++---- package.json | 4 ++-- src/cli.ts | 2 +- src/orchestrator.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 51c2dab..15dcb92 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# Agent Arena +# Arena **Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** -Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. +Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/oxageninc/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. ```bash -git clone https://github.com/macanderson/agent-arena -cd agent-arena && pnpm install +git clone https://github.com/oxageninc/arena +cd arena && pnpm install pnpm arena doctor # which agent CLIs are installed? pnpm arena list # available tasks diff --git a/package.json b/package.json index 6d85953..1ba88a5 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "name": "agent-arena", + "name": "arena", "version": "0.1.0", "description": "Head-to-head benchmark harness for agentic coding CLIs (Claude Code, Gemini CLI, Oxagen, Stella, and yours) with held-out verification, matched configs, paired statistics, and full receipts.", "license": "MIT", "type": "module", "repository": { "type": "git", - "url": "https://github.com/macanderson/agent-arena.git" + "url": "https://github.com/oxageninc/arena.git" }, "keywords": [ "benchmark", diff --git a/src/cli.ts b/src/cli.ts index db1653d..01dd32f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -52,7 +52,7 @@ switch (command) { default: console.log( [ - "agent-arena — head-to-head benchmarks for agentic coding CLIs", + "arena — head-to-head benchmarks for agentic coding CLIs", "", "Commands:", " list List available tasks", diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 888d615..1e2f16d 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -73,7 +73,7 @@ export async function executeRun( runId, createdAt: new Date().toISOString(), harness: { - name: "agent-arena", + name: "arena", version: HARNESS_VERSION, gitSha: gitSha(), }, From 9f66cfcc95ec0c5571de01c4cd2c00afd97eccca Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 14:06:15 -0700 Subject: [PATCH 04/18] feat(harbor): adapter to run any agent under Harbor's official verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds harbor/ — a Python Harbor adapter (BaseInstalledAgent) that runs your coding agent through Harbor's containerized SWE-bench / Terminal-Bench verifier head-to-head against the agents Harbor already ships (Claude Code, Gemini, Codex, Cursor, Aider, ...), all scored by the same repo-native test suite. - ByoAgent: wire up your own agent from a TOML/JSON spec — no Python. - OxagenAgent / StellaAgent: specs for agents Harbor lacks; worked examples. - Spec-driven install (binary upload or script) + one-shot run + best-effort token/cost parsing with the same cache-aware normalization as the TS side. - A non-zero agent exit never aborts scoring — the verifier decides pass/fail. - Pure logic (spec/metrics/command) is Harbor-free and unit-tested; agent tests run against real Harbor 0.6.1 (39 tests green) and verify Harbor's factory loads each agent by import path. - Docs + run.sh + a CI job (pip install -e .; pytest). Main README + METHODOLOGY now point at this adapter as the scale-up path. --- .github/workflows/ci.yml | 16 +++ .gitignore | 7 + METHODOLOGY.md | 4 +- README.md | 18 ++- harbor/README.md | 124 ++++++++++++++++ harbor/arena_harbor/__init__.py | 43 ++++++ harbor/arena_harbor/agents.py | 58 ++++++++ harbor/arena_harbor/base.py | 169 ++++++++++++++++++++++ harbor/arena_harbor/command.py | 85 +++++++++++ harbor/arena_harbor/metrics.py | 168 ++++++++++++++++++++++ harbor/arena_harbor/py.typed | 0 harbor/arena_harbor/spec.py | 206 +++++++++++++++++++++++++++ harbor/arena_harbor/specs_builtin.py | 87 +++++++++++ harbor/pyproject.toml | 43 ++++++ harbor/run.sh | 47 ++++++ harbor/specs/byo.example.toml | 76 ++++++++++ harbor/tests/__init__.py | 0 harbor/tests/test_agents.py | 135 ++++++++++++++++++ harbor/tests/test_command.py | 74 ++++++++++ harbor/tests/test_metrics.py | 102 +++++++++++++ harbor/tests/test_spec.py | 104 ++++++++++++++ 21 files changed, 1564 insertions(+), 2 deletions(-) create mode 100644 harbor/README.md create mode 100644 harbor/arena_harbor/__init__.py create mode 100644 harbor/arena_harbor/agents.py create mode 100644 harbor/arena_harbor/base.py create mode 100644 harbor/arena_harbor/command.py create mode 100644 harbor/arena_harbor/metrics.py create mode 100644 harbor/arena_harbor/py.typed create mode 100644 harbor/arena_harbor/spec.py create mode 100644 harbor/arena_harbor/specs_builtin.py create mode 100644 harbor/pyproject.toml create mode 100755 harbor/run.sh create mode 100644 harbor/specs/byo.example.toml create mode 100644 harbor/tests/__init__.py create mode 100644 harbor/tests/test_agents.py create mode 100644 harbor/tests/test_command.py create mode 100644 harbor/tests/test_metrics.py create mode 100644 harbor/tests/test_spec.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64c0d06..31cf3cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,19 @@ jobs: # Task audit: every task's held-out tests must FAIL on the pristine # workspace and PASS on the reference solution. - run: pnpm arena verify + + harbor-adapter: + runs-on: ubuntu-latest + defaults: + run: + working-directory: harbor + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - run: pip install -e '.[dev]' + # Spec/metrics/command logic is Harbor-free; the agent tests exercise the + # real Harbor BaseInstalledAgent subclasses (Harbor is a declared dep). + - run: pytest -q diff --git a/.gitignore b/.gitignore index 492dd79..8d38024 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,10 @@ results/ dist/ *.tsbuildinfo .DS_Store + +# Python (harbor adapter) +harbor/.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ +*.pyc diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 49e638d..25768c9 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -50,4 +50,6 @@ One run executes N agents × T tasks × K trials. Every trial: ## Scaling up -For headline resolve-rate claims, run this same protocol over [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) with a containerized runner such as [Harbor](https://github.com/laude-institute/harbor) and the **official** evaluator, with a pre-registered instance list (published random seed over the 500 verified instances, n ≥ 100). Arena's local suite is for harness development, smoke comparisons, and metric plumbing — the statistics and receipt discipline are identical either way. +For headline resolve-rate claims, run this same protocol over [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) with a containerized runner and the **official** evaluator, with a pre-registered instance list (published random seed over the 500 verified instances, n ≥ 100). Arena's local suite is for harness development, smoke comparisons, and metric plumbing — the statistics and receipt discipline are identical either way. + +The [`harbor/`](harbor/) adapter is how you do this: it plugs any agent into [Harbor](https://www.harborframework.com/), whose Docker verifier runs each task's own FAIL_TO_PASS / PASS_TO_PASS suite — the agent never sees the grader. Because Harbor already bundles the industry-leading agents (Claude Code, Gemini, Codex, Cursor, Aider, …), a matched-model run of your agent against those built-ins is a fair, official-scored, receipted comparison — exactly the bar this document sets, at research scale. The adapter's token/cost numbers are annotations; the resolve-rate from Harbor's verifier is the score. diff --git a/README.md b/README.md index 15dcb92..918bb8f 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,23 @@ Two invariants, enforced by `pnpm arena verify` in CI: The prompt states the complete behavior contract; the hidden tests assert only what the prompt states. Verification needs nothing but Node — no npm installs inside workspaces, no network. -The built-in suite is deliberately small, fast, and cheap — a calibration set, not a research benchmark. For publishable resolve-rate claims, pair Arena's protocol with [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) under a containerized runner (e.g. [Harbor](https://github.com/laude-institute/harbor)) and use the official evaluator; see [METHODOLOGY.md](METHODOLOGY.md#scaling-up). +The built-in suite is deliberately small, fast, and cheap — a calibration set, not a research benchmark. For publishable resolve-rate claims, run at scale on SWE-bench Verified with the **Harbor adapter** below. + +## Scale up: SWE-bench Verified via Harbor + +[`harbor/`](harbor/) is a [Harbor](https://www.harborframework.com/) adapter that runs **your** agent through Harbor's official, containerized SWE-bench / Terminal-Bench verifier — head-to-head against the industry-leading agents Harbor already ships (`claude-code`, `gemini-cli`, `codex`, `cursor-cli`, `aider`, …), all scored by the same repo-native test suite. + +Wire up your agent with one spec file — no Python: + +```bash +cd harbor && pip install -e . +export ARENA_AGENT_SPEC=$PWD/my-agent.toml # copy specs/byo.example.toml +harbor run --agent-import-path arena_harbor:ByoAgent \ + --dataset swe-bench/swe-bench-verified -m anthropic/claude-sonnet-5 -n 4 +# then the same run with --agent claude-code, and compare resolved counts. +``` + +Ships with `ByoAgent` (bring your own) plus `OxagenAgent` / `StellaAgent` specs. Same model, same dataset, same official verifier — the harness is the only variable. See [`harbor/README.md`](harbor/README.md) and [METHODOLOGY.md](METHODOLOGY.md#scaling-up). ## Reading the report diff --git a/harbor/README.md b/harbor/README.md new file mode 100644 index 0000000..b631e8e --- /dev/null +++ b/harbor/README.md @@ -0,0 +1,124 @@ +# Arena Harbor Adapter + +Run **your** coding agent through [Harbor](https://www.harborframework.com/)'s +official, containerized verifier — head-to-head against Claude Code, Gemini, +Codex, Cursor, Aider, and the rest — on SWE-bench Verified and Terminal-Bench. + +## The idea + +Harbor already ships adapters for the industry-leading agents +(`--agent claude-code`, `codex`, `gemini-cli`, `cursor-cli`, `aider`, …) and +scores every trial with the **same** repo-native test suite (FAIL_TO_PASS / +PASS_TO_PASS) inside a container. The agent never sees the verifier. + +This package adds the two things Harbor doesn't give you: + +1. **`ByoAgent`** — wire up your own agent from a small spec file. No Python. +2. Specs for agents Harbor lacks — **oxagen**, **stella** — that double as + worked examples. + +Your agent then competes against the built-ins under one scorer, so the +resolve-rate comparison is fair by construction. + +## Install + +```bash +pip install -e . # from this directory (arena/harbor) +# Harbor itself needs Docker running to execute tasks. +``` + +## Wire up your agent (the main path) + +1. Copy [`specs/byo.example.toml`](specs/byo.example.toml) and edit it for your + CLI — how to install it in the container, how to invoke it one-shot, which + env vars to forward. The placeholders (`{model}`, `{budget}`, `{instruction}`, + …) are documented in the file. + +2. Point Harbor at it. `--agent-import-path` loads any `module:Class`: + +```bash +export ARENA_AGENT_SPEC=$PWD/my-agent.toml +export ARENA_AGENT_NAME=my-agent # how results are labelled +export MY_AGENT_API_KEY=... # forwarded per your spec's env_keys + +harbor run \ + --agent-import-path arena_harbor:ByoAgent \ + --dataset swe-bench/swe-bench-verified \ + -m anthropic/claude-sonnet-5 \ + -n 4 -k 1 \ + -o results/my-agent +``` + +3. Run a built-in competitor on the **same** dataset and model, then compare + resolved counts: + +```bash +harbor run --agent claude-code -m anthropic/claude-sonnet-5 \ + --dataset swe-bench/swe-bench-verified -n 4 -o results/claude-code +``` + +Same model, same dataset, same verifier — the difference is the harness. + +## Built-in Arena agents + +```bash +# Stella — a single native binary, uploaded from the host. +# Build for the CONTAINER's arch (Linux x86-64), not your laptop: +# cargo build --release --target x86_64-unknown-linux-gnu -p stella-cli +export ARENA_STELLA_BIN=/path/to/linux/stella +harbor run --agent-import-path arena_harbor:StellaAgent \ + -m zai/glm-5.2 --dataset swe-bench/swe-bench-verified -n 4 + +# Oxagen — a Node CLI, so you supply its install command (no package name is +# hard-coded): +export ARENA_OXAGEN_INSTALL="npm install -g @your-scope/oxagen@X" +export AI_GATEWAY_API_KEY=... +harbor run --agent-import-path arena_harbor:OxagenAgent \ + -m anthropic/claude-sonnet-5 --dataset swe-bench/swe-bench-verified -n 4 +``` + +`run.sh` wraps this flow with sensible defaults — `AGENT=stella ./run.sh`. + +## How a run works + +For each task Harbor builds a container, then this adapter: + +1. **installs** your agent (`install.kind = "binary"` uploads a host executable; + `"script"` runs your shell snippet as root — npm/pip/curl/tarball); +2. **runs** it one-shot in the task repo with your `run_template`, forwarding the + host env vars your spec declares (plus every `ARENA_*`); +3. lets Harbor's **verifier** decide pass/fail from the repo's own tests — a + non-zero exit from your CLI never aborts scoring; +4. best-effort **parses** token/cost numbers from stdout for the trial metadata + (these annotate; they never score). + +Token normalization matches Arena's TS adapters: set `metrics.input_includes_cache` +when your reported input count already includes cache reads, so cross-agent +token totals stay comparable. + +## Configuration knobs + +| Env var | Purpose | +|---|---| +| `ARENA_AGENT_SPEC` | Path to your `ByoAgent` spec (`.toml`/`.json`). | +| `ARENA_AGENT_NAME` | Result label for `ByoAgent` (default `arena-byo`). | +| `ARENA_BUDGET` | Per-task USD cap (fills `{budget}`). | +| `ARENA_TIMEOUT` | Per-task seconds (fills `{timeout}`; default 1800). | +| `ARENA_STELLA_BIN` | Host path to the stella binary (StellaAgent). | +| `ARENA_OXAGEN_INSTALL` | Shell command that installs oxagen (OxagenAgent). | + +Provider keys (`ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `ZAI_API_KEY`, +`AI_GATEWAY_API_KEY`, …) are forwarded when named in the spec's `env_keys`. + +## Development + +```bash +pip install -e '.[dev]' +pytest -q # spec/metrics/command tests need no Harbor; agent tests use it +``` + +The pure logic (`spec.py`, `metrics.py`, `command.py`) imports nothing from +Harbor and is tested in isolation; `test_agents.py` exercises the real +`BaseInstalledAgent` subclasses and `importorskip`s when Harbor is absent. + +MIT © Oxagen diff --git a/harbor/arena_harbor/__init__.py b/harbor/arena_harbor/__init__.py new file mode 100644 index 0000000..b5423d2 --- /dev/null +++ b/harbor/arena_harbor/__init__.py @@ -0,0 +1,43 @@ +"""Arena Harbor adapter — run any coding CLI under Harbor's official verifier. + +Public surface: + +- Concrete agents: :class:`ByoAgent` (bring your own), :class:`OxagenAgent`, + :class:`StellaAgent` — pass to Harbor via ``--agent-import-path + arena_harbor:``. +- Spec building blocks: :class:`AgentSpec`, :class:`InstallSpec`, + :class:`MetricsSpec`, and :func:`load_spec_file`. + +The industry-leading agents you compare against (Claude Code, Gemini, Codex, +Cursor, Copilot, Aider, …) already ship inside Harbor — no adapter needed for +those. Arena adds the agents Harbor lacks and, above all, the bring-your-own +path. +""" + +from __future__ import annotations + +from .agents import ByoAgent, OxagenAgent, StellaAgent +from .base import ArenaInstalledAgent +from .spec import ( + AgentSpec, + InstallSpec, + MetricsSpec, + load_spec_file, + load_spec_from_env, + spec_from_dict, +) + +__all__ = [ + "ArenaInstalledAgent", + "ByoAgent", + "OxagenAgent", + "StellaAgent", + "AgentSpec", + "InstallSpec", + "MetricsSpec", + "load_spec_file", + "load_spec_from_env", + "spec_from_dict", +] + +__version__ = "0.1.0" diff --git a/harbor/arena_harbor/agents.py b/harbor/arena_harbor/agents.py new file mode 100644 index 0000000..353816f --- /dev/null +++ b/harbor/arena_harbor/agents.py @@ -0,0 +1,58 @@ +"""Concrete Harbor agents. + +Run any of these with Harbor's custom-agent import path, e.g.:: + + harbor run --agent-import-path arena_harbor:StellaAgent \\ + --dataset swe-bench/swe-bench-verified -m zai/glm-5.2 + +``ByoAgent`` is the flagship: it loads its spec from the ``ARENA_AGENT_SPEC`` +file, so wiring up *your* agent is a spec file plus one import path — no Python. +""" + +from __future__ import annotations + +import os + +from .base import ArenaInstalledAgent +from .spec import AgentSpec, load_spec_from_env +from .specs_builtin import OXAGEN_SPEC, STELLA_SPEC + + +class OxagenAgent(ArenaInstalledAgent): + @staticmethod + def name() -> str: + return "oxagen" + + def spec(self) -> AgentSpec: + return OXAGEN_SPEC + + +class StellaAgent(ArenaInstalledAgent): + @staticmethod + def name() -> str: + return "stella" + + def spec(self) -> AgentSpec: + return STELLA_SPEC + + +class ByoAgent(ArenaInstalledAgent): + """Bring-your-own agent, configured entirely from a spec file. + + Set ``ARENA_AGENT_SPEC`` to your ``.toml``/``.json`` spec and, optionally, + ``ARENA_AGENT_NAME`` to label results (default ``arena-byo``). The spec is + loaded once and cached on the instance. + """ + + @staticmethod + def name() -> str: + # Static so Harbor can label results without instantiating; the concrete + # run/install behavior comes from the spec file loaded per instance. + return os.environ.get("ARENA_AGENT_NAME", "arena-byo") + + def spec(self) -> AgentSpec: + cached = getattr(self, "_spec", None) + if cached is None: + cached = load_spec_from_env() + self._spec = cached + return cached diff --git a/harbor/arena_harbor/base.py b/harbor/arena_harbor/base.py new file mode 100644 index 0000000..bdf9c39 --- /dev/null +++ b/harbor/arena_harbor/base.py @@ -0,0 +1,169 @@ +"""``ArenaInstalledAgent`` — a spec-driven Harbor ``BaseInstalledAgent``. + +One class runs any coding CLI described by an :class:`~arena_harbor.spec.AgentSpec`: +it installs the binary, invokes it one-shot on the task, and (best-effort) reads +back token/cost numbers. Concrete agents (oxagen, stella, bring-your-own) are +thin subclasses that supply a spec and a ``name()``. + +Only this module and :mod:`arena_harbor.agents` import Harbor; the spec, +metrics, and command logic they lean on are Harbor-free and independently +tested. +""" + +from __future__ import annotations + +import os +import shlex +from pathlib import Path + +from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +from .command import build_command, forwarded_env, parse_version +from .metrics import extract_metrics +from .spec import AgentSpec + +_REMOTE_BIN_DIR = "/usr/local/bin" + +# apt / apk / yum, whichever the base image has. Best-effort: a missing package +# manager is a warning, not a hard failure (the image may already have the deps). +_SYSTEM_PKG_TEMPLATE = ( + "if command -v apt-get >/dev/null 2>&1; then " + " apt-get update && apt-get install -y {pkgs}; " + "elif command -v apk >/dev/null 2>&1; then " + " apk add --no-cache {pkgs}; " + "elif command -v yum >/dev/null 2>&1; then " + " yum install -y {pkgs}; " + "else " + ' echo "arena-harbor: no known package manager; assuming {pkgs} present" >&2; ' + "fi" +) + + +class ArenaInstalledAgent(BaseInstalledAgent): + """Base for Arena's Harbor agents. Subclasses set ``name()`` and ``spec()``.""" + + def spec(self) -> AgentSpec: + raise NotImplementedError("subclasses must implement spec()") + + # ── version ──────────────────────────────────────────────────────────── + def get_version_command(self) -> str | None: + return self.spec().version_command + + def parse_version(self, stdout: str) -> str: + return parse_version(stdout, self.spec().version_regex) + + # ── install ──────────────────────────────────────────────────────────── + async def install(self, environment: BaseEnvironment) -> None: + spec = self.spec() + install = spec.install + + if install.system_packages: + pkgs = " ".join(shlex.quote(p) for p in install.system_packages) + await self.exec_as_root( + environment, + command=_SYSTEM_PKG_TEMPLATE.format(pkgs=pkgs), + env={"DEBIAN_FRONTEND": "noninteractive"}, + timeout_sec=600, + ) + + if install.kind == "binary": + host_binary = self._resolve_host_binary(spec) + remote_tmp = f"/tmp/{spec.binary}" + dest = f"{_REMOTE_BIN_DIR}/{spec.binary}" + await environment.upload_file(str(host_binary), remote_tmp) + await self.exec_as_root( + environment, + command=( + f"cp {shlex.quote(remote_tmp)} {shlex.quote(dest)} && " + f"chmod +x {shlex.quote(dest)}" + ), + timeout_sec=120, + ) + elif install.kind == "script": + script = self._resolve_install_script(spec) + await self.exec_as_root(environment, command=script, timeout_sec=1800) + else: # pragma: no cover - spec loader rejects other kinds + raise ValueError(f"unknown install kind: {install.kind}") + + def _resolve_host_binary(self, spec: AgentSpec) -> Path: + env_var = spec.install.binary_env + raw = os.environ.get(env_var or "") + if not raw: + raise FileNotFoundError( + f"agent '{spec.name}': set {env_var} to the path of the " + f"{spec.binary} executable to upload (built for the container's " + f"OS/arch — Linux, usually x86-64)." + ) + path = Path(raw) + if not path.is_file(): + raise FileNotFoundError( + f"agent '{spec.name}': {env_var}={raw} is not a file." + ) + return path + + def _resolve_install_script(self, spec: AgentSpec) -> str: + install = spec.install + if install.script_env: + override = os.environ.get(install.script_env) + if override: + return override + if install.script: + return install.script + raise ValueError( + f"agent '{spec.name}': no install script " + f"(set {install.script_env} or provide install.script)." + ) + + # ── run ──────────────────────────────────────────────────────────────── + @with_prompt_template + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + spec = self.spec() + budget = os.environ.get("ARENA_BUDGET", spec.default_budget or "") + timeout = os.environ.get("ARENA_TIMEOUT", "1800") + command = build_command( + spec, self.model_name, instruction, budget=budget, timeout=timeout + ) + env = forwarded_env(spec) + + # Run directly (not exec_as_agent) so a non-zero agent exit does NOT + # abort the trial before Harbor's verifier gets to judge the workspace. + # The verifier — never the CLI's exit code — decides pass/fail. + try: + result = await environment.exec( + command=command, + env=env, + timeout_sec=int(timeout) if timeout.isdigit() else None, + ) + self._agent_output = "\n".join( + part + for part in ( + getattr(result, "stdout", "") or "", + getattr(result, "stderr", "") or "", + ) + if part + ) + except Exception as exc: # noqa: BLE001 - never let agent failure kill scoring + self._agent_output = f"[arena-harbor] agent execution error: {exc}" + self.logger.warning("arena-harbor: agent run raised: %s", exc) + + # ── metrics ──────────────────────────────────────────────────────────── + def populate_context_post_run(self, context: AgentContext) -> None: + spec = self.spec() + if spec.metrics is None: + return + output = getattr(self, "_agent_output", "") + if not output: + return + parsed = extract_metrics(output, spec.metrics) + if parsed.is_empty(): + return + if parsed.cost_usd is not None: + context.cost_usd = parsed.cost_usd + context.metadata = {**(context.metadata or {}), **parsed.as_metadata(spec.name)} diff --git a/harbor/arena_harbor/command.py b/harbor/arena_harbor/command.py new file mode 100644 index 0000000..d94fb73 --- /dev/null +++ b/harbor/arena_harbor/command.py @@ -0,0 +1,85 @@ +"""Command rendering and env forwarding — the pure core of an agent run. + +Kept Harbor-free so the exact shell command an agent will run, and the exact +set of secrets forwarded into the container, are unit-testable without spinning +up a container. +""" + +from __future__ import annotations + +import os +import re +import shlex +from typing import Mapping + +from .spec import AgentSpec + +_PLACEHOLDER = re.compile(r"\{(\w+)\}") + + +def render_template(template: str, mapping: Mapping[str, str]) -> str: + """Replace ``{key}`` tokens that appear in ``mapping``; leave every other + brace run (unknown ``{x}``, shell ``${VAR}``, awk ``{print}``) untouched.""" + + def repl(match: re.Match[str]) -> str: + key = match.group(1) + return mapping[key] if key in mapping else match.group(0) + + return _PLACEHOLDER.sub(repl, template) + + +def build_command( + spec: AgentSpec, + model_name: str | None, + instruction: str, + *, + budget: str | None = None, + timeout: str | None = None, +) -> str: + """Render ``spec.run_template`` into the concrete shell command. + + ``model_name`` is Arena-canonical ``provider/model``; ``{instruction}`` is + shell-quoted here so callers must not quote it themselves. + """ + model = model_name or spec.default_model or "" + provider, sep, model_id = model.partition("/") + if not sep: # bare model id, no provider prefix + provider, model_id = "", model + + mapping = { + "bin": spec.binary, + "model": model, + "provider": provider, + "model_id": model_id, + "budget": budget if budget is not None else (spec.default_budget or ""), + "timeout": timeout or "1800", + "instruction": shlex.quote(instruction), + } + return render_template(spec.run_template, mapping) + + +def forwarded_env(spec: AgentSpec, environ: Mapping[str, str] | None = None) -> dict[str, str]: + """Collect the host env vars to forward into the container: the spec's + declared ``env_keys`` (API keys etc.) plus every ``ARENA_*`` var.""" + src = os.environ if environ is None else environ + out: dict[str, str] = {} + for key in spec.env_keys: + if key in src: + out[key] = src[key] + for key, value in src.items(): + if key.startswith("ARENA_"): + out[key] = value + return out + + +def parse_version(stdout: str, version_regex: str | None) -> str: + """First regex capture group, else the first non-empty output line.""" + text = stdout.strip() + if version_regex: + m = re.search(version_regex, text) + if m: + return (m.group(1) if m.groups() else m.group(0)).strip() + for line in text.splitlines(): + if line.strip(): + return line.strip() + return text diff --git a/harbor/arena_harbor/metrics.py b/harbor/arena_harbor/metrics.py new file mode 100644 index 0000000..faea373 --- /dev/null +++ b/harbor/arena_harbor/metrics.py @@ -0,0 +1,168 @@ +"""Best-effort token/cost extraction from an agent's stdout. + +Secondary to Harbor's verifier by design: these numbers annotate a trial, they +never decide it. Normalization mirrors Arena's TypeScript adapters so token +counts are comparable across agents — most importantly, ``input`` never +includes cache reads. + +Harbor-free; unit-tested in isolation. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any + +from .spec import MetricsSpec + + +@dataclass +class ParsedMetrics: + input_tokens: int | None = None + output_tokens: int | None = None + cache_read_tokens: int | None = None + total_tokens: int | None = None + cost_usd: float | None = None + + def as_metadata(self, agent_name: str) -> dict[str, Any]: + prefix = f"arena_{agent_name.replace('-', '_')}" + out: dict[str, Any] = {} + for field_name in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "total_tokens", + "cost_usd", + ): + value = getattr(self, field_name) + if value is not None: + out[f"{prefix}_{field_name}"] = value + return out + + def is_empty(self) -> bool: + return all( + getattr(self, f) is None + for f in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "total_tokens", + "cost_usd", + ) + ) + + +def last_json_object(text: str) -> dict[str, Any] | None: + """Return the last ``{"type":"result"}`` JSON object in ``text`` (JSONL or a + single object), else the last parseable object. Non-JSON lines are ignored. + Mirrors the TS ``parseJsonEnvelope``. + """ + if not text: + return None + fallback: dict[str, Any] | None = None + for line in reversed([ln.strip() for ln in text.splitlines() if ln.strip().startswith("{")]): + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + if fallback is None: + fallback = obj + if obj.get("type") == "result": + return obj + return fallback + + +def dotted_get(obj: Any, path: str | None) -> Any: + """Resolve a dotted path (``a.b.c``) into nested dicts; None if absent.""" + if not path: + return None + current = obj + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _as_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)) and value >= 0: + return int(value) + return None + + +def _as_float(value: Any) -> float | None: + if isinstance(value, bool): + return None + if isinstance(value, (int, float)) and value >= 0: + return float(value) + return None + + +def extract_metrics(text: str, spec: MetricsSpec) -> ParsedMetrics: + """Extract normalized metrics from agent output per ``spec``. + + Returns an all-None :class:`ParsedMetrics` (``is_empty()``) when nothing + parseable is found — the caller then simply records no metrics. + """ + if spec.kind == "regex": + return _extract_regex(text, spec) + return _extract_json(text, spec) + + +def _extract_json(text: str, spec: MetricsSpec) -> ParsedMetrics: + env = last_json_object(text) + if env is None: + return ParsedMetrics() + + raw_input = _as_int(dotted_get(env, spec.input_path)) + output = _as_int(dotted_get(env, spec.output_path)) + cache_read = _as_int(dotted_get(env, spec.cache_read_path)) + cost = _as_float(dotted_get(env, spec.cost_path)) + + norm_input = raw_input + if spec.input_includes_cache and raw_input is not None and cache_read is not None: + norm_input = max(0, raw_input - cache_read) + + total = None + parts = [p for p in (norm_input, output, cache_read) if p is not None] + if parts: + total = sum(parts) + + return ParsedMetrics( + input_tokens=norm_input, + output_tokens=output, + cache_read_tokens=cache_read, + total_tokens=total, + cost_usd=cost, + ) + + +def _first_number(pattern: str | None, text: str) -> float | None: + if not pattern: + return None + m = re.search(pattern, text) + if not m or not m.groups(): + return None + try: + return float(m.group(1).replace(",", "")) + except (ValueError, IndexError): + return None + + +def _extract_regex(text: str, spec: MetricsSpec) -> ParsedMetrics: + cost = _first_number(spec.cost_regex, text) + input_tokens = _first_number(spec.input_regex, text) + output = _first_number(spec.output_regex, text) + total = _first_number(spec.total_regex, text) + return ParsedMetrics( + input_tokens=int(input_tokens) if input_tokens is not None else None, + output_tokens=int(output) if output is not None else None, + total_tokens=int(total) if total is not None else None, + cost_usd=cost, + ) diff --git a/harbor/arena_harbor/py.typed b/harbor/arena_harbor/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/harbor/arena_harbor/spec.py b/harbor/arena_harbor/spec.py new file mode 100644 index 0000000..5981d79 --- /dev/null +++ b/harbor/arena_harbor/spec.py @@ -0,0 +1,206 @@ +"""Agent specifications — the declarative contract that turns *any* coding CLI +into a Harbor agent. + +A spec says three things: how to get the agent's binary into the container +(``install``), how to invoke it one-shot on a task (``run_template``), and how +to read token/cost numbers back out of its output (``metrics``). Built-in +agents (oxagen, stella) ship as Python specs; a user wires up their own agent +by writing one of these as TOML/JSON and pointing ``ARENA_AGENT_SPEC`` at it — +no Python required. + +This module imports nothing from Harbor, so it (and its tests) run anywhere. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +InstallKind = Literal["binary", "script"] + + +@dataclass +class InstallSpec: + """How the agent binary gets into the (Linux) container. + + ``binary`` uploads a host executable — fast, but the host binary must be + built for the container's OS/arch (Linux, usually x86-64), so a macOS build + won't run. ``script`` runs an arbitrary shell snippet as root (npm/pip/curl + /tarball) and is the portable default. + """ + + kind: InstallKind = "script" + #: For ``binary``: host env var holding the path to the executable to upload. + binary_env: str | None = None + #: For ``script``: shell run as root; must leave ``AgentSpec.binary`` on PATH. + script: str | None = None + #: For ``script``: env var that, if set, overrides ``script`` at run time. + script_env: str | None = None + #: Best-effort OS packages ensured before install (apt/apk/yum). + system_packages: list[str] = field(default_factory=list) + + +@dataclass +class MetricsSpec: + """How to recover token/cost numbers from the agent's stdout. + + Best-effort and secondary — the headline resolve-rate always comes from + Harbor's official verifier, never from these numbers. ``json_tail`` reads + the last JSON object the agent printed and pulls values by dotted path; + ``regex`` scrapes a human summary line. + """ + + kind: Literal["json_tail", "regex"] = "json_tail" + # json_tail dotted paths (e.g. "usage.inputTokens") + input_path: str | None = None + output_path: str | None = None + cache_read_path: str | None = None + cost_path: str | None = None + #: True when the agent's reported input count already includes cache reads, + #: so normalized input = reported_input - cache_read (matches Arena's TS + #: adapters, keeping cross-agent token counts comparable). + input_includes_cache: bool = False + # regex alternative: each pattern's first capture group is the number + cost_regex: str | None = None + input_regex: str | None = None + output_regex: str | None = None + total_regex: str | None = None + + +@dataclass +class AgentSpec: + """A complete recipe for running one coding CLI under Harbor.""" + + name: str + #: The binary/command that must be on PATH after install. + binary: str + #: Shell template invoking the agent one-shot. Recognized placeholders: + #: ``{bin} {model} {provider} {model_id} {budget} {timeout} {instruction}``. + #: ``{instruction}`` is shell-quoted for you; unknown ``{...}`` are left + #: untouched so shell/awk braces survive. + run_template: str + install: InstallSpec = field(default_factory=InstallSpec) + version_command: str | None = None + #: Regex whose first group is the version (else the first output line). + version_regex: str | None = None + #: Host env vars (API keys etc.) forwarded into the container. + env_keys: list[str] = field(default_factory=list) + default_model: str | None = None + default_budget: str | None = None + metrics: MetricsSpec | None = None + + +def _as_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v) for v in value] + raise ValueError(f"expected a list, got {type(value).__name__}") + + +def spec_from_dict(data: dict[str, Any]) -> AgentSpec: + """Build an :class:`AgentSpec` from a plain dict (parsed TOML/JSON). + + Raises ``ValueError`` with a precise message on any missing/mistyped field, + so a broken bring-your-own spec fails loudly at load time, not mid-run. + """ + for required in ("name", "binary", "run_template"): + if not data.get(required): + raise ValueError(f"agent spec missing required field '{required}'") + + install_raw = data.get("install") or {} + if not isinstance(install_raw, dict): + raise ValueError("'install' must be a table/object") + kind = install_raw.get("kind", "script") + if kind not in ("binary", "script"): + raise ValueError(f"install.kind must be 'binary' or 'script', got '{kind}'") + install = InstallSpec( + kind=kind, + binary_env=install_raw.get("binary_env"), + script=install_raw.get("script"), + script_env=install_raw.get("script_env"), + system_packages=_as_list(install_raw.get("system_packages")), + ) + if kind == "binary" and not install.binary_env: + raise ValueError("install.kind='binary' requires 'binary_env'") + if kind == "script" and not (install.script or install.script_env): + raise ValueError( + "install.kind='script' requires 'script' or 'script_env'" + ) + + metrics = None + metrics_raw = data.get("metrics") + if metrics_raw is not None: + if not isinstance(metrics_raw, dict): + raise ValueError("'metrics' must be a table/object") + metrics = MetricsSpec( + kind=metrics_raw.get("kind", "json_tail"), + input_path=metrics_raw.get("input_path"), + output_path=metrics_raw.get("output_path"), + cache_read_path=metrics_raw.get("cache_read_path"), + cost_path=metrics_raw.get("cost_path"), + input_includes_cache=bool(metrics_raw.get("input_includes_cache", False)), + cost_regex=metrics_raw.get("cost_regex"), + input_regex=metrics_raw.get("input_regex"), + output_regex=metrics_raw.get("output_regex"), + total_regex=metrics_raw.get("total_regex"), + ) + + return AgentSpec( + name=str(data["name"]), + binary=str(data["binary"]), + run_template=str(data["run_template"]), + install=install, + version_command=data.get("version_command"), + version_regex=data.get("version_regex"), + env_keys=_as_list(data.get("env_keys")), + default_model=data.get("default_model"), + default_budget=( + str(data["default_budget"]) if data.get("default_budget") is not None else None + ), + metrics=metrics, + ) + + +def load_spec_file(path: str | Path) -> AgentSpec: + """Load an :class:`AgentSpec` from a ``.toml`` or ``.json`` file.""" + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"agent spec file not found: {p}") + text = p.read_text() + if p.suffix.lower() == ".json": + data = json.loads(text) + else: + data = _load_toml(text) + if not isinstance(data, dict): + raise ValueError(f"{p}: top level must be a table/object") + return spec_from_dict(data) + + +def load_spec_from_env(env_var: str = "ARENA_AGENT_SPEC") -> AgentSpec: + """Load the bring-your-own spec named by ``env_var`` (default + ``ARENA_AGENT_SPEC``).""" + path = os.environ.get(env_var) + if not path: + raise ValueError( + f"{env_var} is not set. Point it at your agent's spec file " + f"(see specs/byo.example.toml)." + ) + return load_spec_file(path) + + +def _load_toml(text: str) -> Any: + try: + import tomllib # Python 3.11+ + except ModuleNotFoundError: # pragma: no cover - exercised on <3.11 only + try: + import tomli as tomllib # type: ignore[no-redef] + except ModuleNotFoundError as exc: # pragma: no cover + raise RuntimeError( + "Parsing TOML specs on Python < 3.11 needs the 'tomli' package " + "(pip install tomli), or use a .json spec instead." + ) from exc + return tomllib.loads(text) diff --git a/harbor/arena_harbor/specs_builtin.py b/harbor/arena_harbor/specs_builtin.py new file mode 100644 index 0000000..6675ec3 --- /dev/null +++ b/harbor/arena_harbor/specs_builtin.py @@ -0,0 +1,87 @@ +"""Built-in agent specs for agents Harbor does not ship itself. + +Harbor 0.6+ already bundles Claude Code, Gemini CLI, Codex, Cursor, Copilot, +Aider, and more — run those with ``--agent claude-code`` etc. These specs cover +the gaps (oxagen, stella) and double as worked examples of the spec format. + +Harbor-free. +""" + +from __future__ import annotations + +from .spec import AgentSpec, InstallSpec, MetricsSpec + +# Provider API keys worth forwarding for most agents. +_COMMON_KEYS = [ + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "ZAI_API_KEY", + "DEEPSEEK_API_KEY", + "OPENROUTER_API_KEY", + "AI_GATEWAY_API_KEY", +] + + +#: Oxagen — a Node CLI (``#!/bin/sh`` launcher), so it needs a *script* install. +#: The install command is intentionally left to the operator via +#: ``ARENA_OXAGEN_INSTALL`` (e.g. ``npm i -g @oxagen/cli@X`` or a tarball URL) +#: rather than hard-coding a package name we can't verify. +OXAGEN_SPEC = AgentSpec( + name="oxagen", + binary="oxagen", + run_template=( + "{bin} --local --output-format json --model {model} " + "--budget {budget} -- {instruction}" + ), + install=InstallSpec( + kind="script", + script_env="ARENA_OXAGEN_INSTALL", + script=( + 'echo "Set ARENA_OXAGEN_INSTALL to the shell command that installs ' + 'oxagen in the container (e.g. an npm i -g or a tarball fetch)." >&2; ' + "exit 1" + ), + system_packages=["nodejs", "npm"], + ), + version_command="oxagen --version", + env_keys=[*_COMMON_KEYS, "OXAGEN_MODEL_SLUG", "OXAGEN_BUDGET"], + default_model="anthropic/claude-sonnet-5", + default_budget="5", + metrics=MetricsSpec( + kind="json_tail", + input_path="usage.inputTokens", + output_path="usage.outputTokens", + cache_read_path="usage.cachedInputTokens", + input_includes_cache=True, # oxagen's inputTokens includes cache reads + ), +) + + +#: Stella — a single native binary, so it can be *uploaded* from the host. The +#: host binary must match the container's OS/arch (Linux x86-64 for stock SWE- +#: bench images): build with ``cargo build --release --target x86_64-unknown- +#: linux-gnu`` and point ``ARENA_STELLA_BIN`` at it, or switch to a script +#: install that fetches a Linux release. +STELLA_SPEC = AgentSpec( + name="stella", + binary="stella", + run_template="{bin} --model {model} --output-format json --budget {budget} run {instruction}", + install=InstallSpec(kind="binary", binary_env="ARENA_STELLA_BIN"), + version_command="stella --version", + version_regex=r"(\d+\.\d+\.\d+)", + env_keys=[*_COMMON_KEYS, "STELLA_MODEL", "STELLA_BASE_URL", "STELLA_BUDGET"], + default_model="zai/glm-5.2", + default_budget="5", + metrics=MetricsSpec( + kind="json_tail", + input_path="usage.inputTokens", + output_path="usage.outputTokens", + cache_read_path="usage.cachedInputTokens", + cost_path="costUsd", + input_includes_cache=True, + ), +) + +BUILTIN_SPECS = {spec.name: spec for spec in (OXAGEN_SPEC, STELLA_SPEC)} diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml new file mode 100644 index 0000000..a20c953 --- /dev/null +++ b/harbor/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "arena-harbor" +version = "0.1.0" +description = "Harbor adapter to benchmark your coding agent against Claude Code, Gemini, Codex and others under the official verifier" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Oxagen", email = "dev@oxagen.ai" }] +keywords = ["benchmark", "harbor", "swe-bench", "ai-agents", "coding-agents"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "harbor>=0.6.0", + "tomli>=2.0; python_version < '3.11'", +] + +[project.optional-dependencies] +dev = ["pytest>=7.0", "pytest-asyncio>=0.21"] + +[project.urls] +Homepage = "https://github.com/oxageninc/arena" +Repository = "https://github.com/oxageninc/arena" + +[tool.setuptools] +packages = ["arena_harbor"] + +[tool.setuptools.package-data] +arena_harbor = ["py.typed"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/harbor/run.sh b/harbor/run.sh new file mode 100755 index 0000000..4b02f44 --- /dev/null +++ b/harbor/run.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Convenience wrapper: run an Arena Harbor agent on SWE-bench Verified. +# +# AGENT=stella ARENA_STELLA_BIN=/path/to/linux/stella ./run.sh +# AGENT=oxagen ARENA_OXAGEN_INSTALL="npm i -g @scope/oxagen" ./run.sh +# AGENT=byo ARENA_AGENT_SPEC=$PWD/my-agent.toml ARENA_AGENT_NAME=my ./run.sh +# AGENT=claude-code ./run.sh # a Harbor built-in competitor +# +# Prereqs: Docker running; the arena-harbor package installed (pip install -e .); +# a provider API key exported for your chosen model. +set -euo pipefail +cd "$(dirname "$0")" + +AGENT="${AGENT:-byo}" +MODEL="${MODEL:-anthropic/claude-sonnet-5}" +DATASET="${DATASET:-swe-bench/swe-bench-verified}" +N_CONCURRENT="${N_CONCURRENT:-4}" +N_ATTEMPTS="${N_ATTEMPTS:-1}" +JOBS_DIR="${JOBS_DIR:-results/$AGENT}" + +# Map the friendly AGENT name to a Harbor selector. Arena agents load by import +# path; anything else is treated as a Harbor built-in (claude-code, codex, ...). +case "$AGENT" in + byo) AGENT_SEL=(--agent-import-path arena_harbor:ByoAgent) ;; + oxagen) AGENT_SEL=(--agent-import-path arena_harbor:OxagenAgent) ;; + stella) AGENT_SEL=(--agent-import-path arena_harbor:StellaAgent) ;; + *) AGENT_SEL=(--agent "$AGENT") ;; +esac + +# Optional task filter: TASK_IDS="django__django-11099 sympy__sympy-1234" +FILTER=() +for t in ${TASK_IDS:-}; do FILTER+=(--include-task-name "*$t*"); done + +echo "== Arena/Harbor run ==" +echo "agent=$AGENT model=$MODEL dataset=$DATASET n=$N_CONCURRENT k=$N_ATTEMPTS" +echo "jobs-dir=$JOBS_DIR" + +exec harbor run \ + "${AGENT_SEL[@]}" \ + --dataset "$DATASET" \ + -m "$MODEL" \ + -n "$N_CONCURRENT" \ + -k "$N_ATTEMPTS" \ + -o "$JOBS_DIR" \ + "${FILTER[@]}" \ + ${HARBOR_EXTRA:-} diff --git a/harbor/specs/byo.example.toml b/harbor/specs/byo.example.toml new file mode 100644 index 0000000..08c1697 --- /dev/null +++ b/harbor/specs/byo.example.toml @@ -0,0 +1,76 @@ +# Bring-your-own-agent spec for the Arena Harbor adapter. +# +# Copy this file, edit it for your CLI, then run your agent under Harbor's +# official verifier against Claude Code / Gemini / Codex / etc.: +# +# export ARENA_AGENT_SPEC=$PWD/my-agent.toml +# export ARENA_AGENT_NAME=my-agent # how results are labelled +# export MY_AGENT_API_KEY=... +# harbor run \ +# --agent-import-path arena_harbor:ByoAgent \ +# --dataset swe-bench/swe-bench-verified \ +# -m anthropic/claude-sonnet-5 \ +# --n-concurrent 4 +# +# The headline number is the resolve-rate from Harbor's Docker verifier — the +# same scorer applied to every agent. The token/cost fields below are only +# annotations on each trial. + +# Harbor agent name is taken from ARENA_AGENT_NAME; `name` here documents intent. +name = "my-agent" + +# The command that must be on PATH in the container after `install`. +binary = "mycli" + +# One-shot invocation. Placeholders filled in for you: +# {bin} -> the `binary` above +# {model} -> full provider/model (e.g. anthropic/claude-sonnet-5) +# {provider} -> text before the first "/" (anthropic) +# {model_id} -> text after the first "/" (claude-sonnet-5) +# {budget} -> ARENA_BUDGET or default_budget +# {timeout} -> ARENA_TIMEOUT (seconds) or 1800 +# {instruction}-> the task prompt, ALREADY shell-quoted (don't add quotes) +# Unknown braces ({}, ${VAR}, awk {print}) are left untouched. +run_template = "{bin} solve --model {model} --max-usd {budget} --json {instruction}" + +# Provider keys / config to forward from the host into the container. +env_keys = ["MY_AGENT_API_KEY", "ANTHROPIC_API_KEY"] + +default_model = "anthropic/claude-sonnet-5" +default_budget = "5" + +version_command = "mycli --version" +# First capture group is the version; omit to use the first output line. +version_regex = "(\\d+\\.\\d+\\.\\d+)" + +[install] +# "script" (portable: npm/pip/curl/tarball) or "binary" (upload a host binary). +kind = "script" +# OS packages ensured first (best-effort apt/apk/yum). +system_packages = ["nodejs", "npm"] +# Shell run as root; must leave `binary` on PATH. Override at run time with the +# env var named in `script_env` if set. +script = "npm install -g my-agent-cli@latest" +script_env = "ARENA_MYCLI_INSTALL" + +# --- If your agent is a single native binary instead, use: ------------------ +# [install] +# kind = "binary" +# binary_env = "ARENA_MYCLI_BIN" # host path; must match container OS/arch + +# Best-effort token/cost extraction from the agent's stdout. Optional. +[metrics] +kind = "json_tail" # parse the last JSON object the agent prints +input_path = "usage.input_tokens" # dotted path into that object +output_path = "usage.output_tokens" +cache_read_path = "usage.cache_read_tokens" +cost_path = "total_cost_usd" +# Set true if `input_path` already counts cache reads (they'll be subtracted so +# token totals stay comparable across agents). +input_includes_cache = false + +# --- Regex alternative (for agents that print a human summary line): --------- +# [metrics] +# kind = "regex" +# cost_regex = "\\$([0-9.]+)" +# total_regex = "([0-9,]+) tokens" diff --git a/harbor/tests/__init__.py b/harbor/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/harbor/tests/test_agents.py b/harbor/tests/test_agents.py new file mode 100644 index 0000000..6f0a76e --- /dev/null +++ b/harbor/tests/test_agents.py @@ -0,0 +1,135 @@ +"""Harbor-integration tests: install/run/metrics against a mocked environment. + +Skipped automatically when Harbor isn't importable, so the pure logic tests +(spec/metrics/command) still run everywhere. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +pytest.importorskip("harbor") + +from arena_harbor import ByoAgent, OxagenAgent, StellaAgent # noqa: E402 + + +def _mk(agent_cls, tmp_path, model_name="anthropic/claude-sonnet-5"): + return agent_cls(logs_dir=tmp_path, model_name=model_name) + + +def _mock_env(stdout="", return_code=0): + env = MagicMock() + result = MagicMock(return_code=return_code, stdout=stdout, stderr="") + env.exec = AsyncMock(return_value=result) + env.upload_file = AsyncMock() + return env + + +def test_agent_names(): + assert OxagenAgent.name() == "oxagen" + assert StellaAgent.name() == "stella" + + +def test_import_path_format(): + # This is exactly what users pass to `harbor --agent-import-path`. + assert StellaAgent.import_path() == "arena_harbor.agents:StellaAgent" + + +def test_version_command_and_parse(tmp_path): + agent = _mk(StellaAgent, tmp_path) + assert agent.get_version_command() == "stella --version" + assert agent.parse_version("stella 1.4.2") == "1.4.2" + + +async def test_binary_install_uploads_host_binary(tmp_path, monkeypatch): + fake_bin = tmp_path / "stella" + fake_bin.write_text("#!/bin/sh\n") + monkeypatch.setenv("ARENA_STELLA_BIN", str(fake_bin)) + + agent = _mk(StellaAgent, tmp_path, model_name="zai/glm-5.2") + env = _mock_env() + await agent.install(env) + + env.upload_file.assert_awaited_once() + uploaded_from = env.upload_file.await_args.args[0] + assert uploaded_from == str(fake_bin) + # a cp/chmod into /usr/local/bin ran as root + root_cmds = [c.kwargs.get("command", "") for c in env.exec.await_args_list] + assert any("/usr/local/bin/stella" in c for c in root_cmds) + + +async def test_binary_install_missing_env_raises(tmp_path, monkeypatch): + monkeypatch.delenv("ARENA_STELLA_BIN", raising=False) + agent = _mk(StellaAgent, tmp_path, model_name="zai/glm-5.2") + with pytest.raises(FileNotFoundError, match="ARENA_STELLA_BIN"): + await agent.install(_mock_env()) + + +async def test_script_install_uses_override_env(tmp_path, monkeypatch): + monkeypatch.setenv("ARENA_OXAGEN_INSTALL", "npm i -g @acme/oxagen@1.2.3") + agent = _mk(OxagenAgent, tmp_path) + env = _mock_env() + await agent.install(env) + ran = [c.kwargs.get("command", "") for c in env.exec.await_args_list] + assert any("npm i -g @acme/oxagen@1.2.3" in c for c in ran) + + +async def test_run_invokes_agent_and_captures_output(tmp_path): + agent = _mk(OxagenAgent, tmp_path) + envelope = '{"type":"result","usage":{"inputTokens":1000,"outputTokens":200,"cachedInputTokens":400}}' + env = _mock_env(stdout=envelope) + context = MagicMock(metadata={}) + + await agent.run("Fix the failing test", env, context) + + # the built command carried the model, json flag, and quoted instruction + run_cmd = env.exec.await_args_list[-1].kwargs["command"] + assert "oxagen --local --output-format json --model anthropic/claude-sonnet-5" in run_cmd + assert "'Fix the failing test'" in run_cmd + assert agent._agent_output.strip().startswith("{") + + +async def test_run_does_not_raise_when_agent_errors(tmp_path): + # A crashing agent must NOT abort the trial before the verifier runs. + agent = _mk(OxagenAgent, tmp_path) + env = MagicMock() + env.exec = AsyncMock(side_effect=RuntimeError("boom")) + context = MagicMock(metadata={}) + await agent.run("do it", env, context) # should swallow the error + assert "agent execution error" in agent._agent_output + + +async def test_metrics_populate_context(tmp_path): + agent = _mk(StellaAgent, tmp_path, model_name="zai/glm-5.2") + agent._agent_output = ( + '{"type":"result","usage":{"inputTokens":5000,"outputTokens":800,' + '"cachedInputTokens":4000},"costUsd":0.12}' + ) + context = MagicMock(metadata={}, cost_usd=None) + agent.populate_context_post_run(context) + assert context.cost_usd == 0.12 + assert context.metadata["arena_stella_input_tokens"] == 1000 # 5000 - 4000 + assert context.metadata["arena_stella_cache_read_tokens"] == 4000 + + +def test_byo_agent_loads_spec_and_name(tmp_path, monkeypatch): + spec_file = tmp_path / "byo.toml" + spec_file.write_text( + 'name = "cool-agent"\n' + 'binary = "cool"\n' + 'run_template = "{bin} --model {model} {instruction}"\n' + "[install]\n" + 'kind = "script"\n' + 'script = "true"\n' + ) + monkeypatch.setenv("ARENA_AGENT_SPEC", str(spec_file)) + monkeypatch.setenv("ARENA_AGENT_NAME", "cool-agent") + + assert ByoAgent.name() == "cool-agent" + agent = _mk(ByoAgent, tmp_path) + assert agent.spec().binary == "cool" + + +def test_byo_agent_name_defaults(monkeypatch): + monkeypatch.delenv("ARENA_AGENT_NAME", raising=False) + assert ByoAgent.name() == "arena-byo" diff --git a/harbor/tests/test_command.py b/harbor/tests/test_command.py new file mode 100644 index 0000000..25b9db7 --- /dev/null +++ b/harbor/tests/test_command.py @@ -0,0 +1,74 @@ +"""Command rendering + env forwarding. Pure — no Harbor required.""" + +from arena_harbor.command import build_command, forwarded_env, parse_version, render_template +from arena_harbor.spec import AgentSpec, InstallSpec +from arena_harbor.specs_builtin import OXAGEN_SPEC, STELLA_SPEC + + +def _spec(run_template, **kw): + return AgentSpec( + name="a", + binary="mycli", + run_template=run_template, + install=InstallSpec(kind="script", script="true"), + **kw, + ) + + +def test_render_template_replaces_known_leaves_unknown(): + out = render_template("{bin} run {instruction} ${HOME} awk '{print $1}'", {"bin": "x", "instruction": "'go'"}) + assert out == "x run 'go' ${HOME} awk '{print $1}'" + + +def test_build_command_splits_provider_and_quotes_instruction(): + spec = _spec("{bin} --provider {provider} --model {model_id} {instruction}") + cmd = build_command(spec, "anthropic/claude-sonnet-5", "Fix the bug", budget="5", timeout="900") + assert "--provider anthropic" in cmd + assert "--model claude-sonnet-5" in cmd + # instruction is shell-quoted for us + assert "'Fix the bug'" in cmd + + +def test_build_command_bare_model_has_empty_provider(): + spec = _spec("{bin} -m {model} p={provider} i={model_id}") + cmd = build_command(spec, "glm-5.2", "x") + assert "-m glm-5.2" in cmd + assert "p= " in cmd # empty provider + assert "i=glm-5.2" in cmd + + +def test_build_command_falls_back_to_default_model_and_budget(): + spec = _spec("{bin} {model} {budget} {instruction}", default_model="d/m", default_budget="7") + cmd = build_command(spec, None, "task") + assert "d/m" in cmd + assert " 7 " in cmd + + +def test_instruction_with_shell_metachars_is_safe(): + spec = _spec("{bin} {instruction}") + cmd = build_command(spec, "d/m", "rm -rf / ; echo $(whoami) `id`") + # everything after {bin} is a single quoted token — no unquoted metachars + assert cmd.startswith("mycli ") + payload = cmd[len("mycli ") :] + assert payload.startswith("'") and payload.endswith("'") + + +def test_forwarded_env_includes_declared_keys_and_arena_prefix(): + spec = _spec("{bin} {instruction}", env_keys=["MY_KEY", "ABSENT_KEY"]) + environ = {"MY_KEY": "secret", "ARENA_BUDGET": "3", "UNRELATED": "x"} + env = forwarded_env(spec, environ) + assert env == {"MY_KEY": "secret", "ARENA_BUDGET": "3"} + + +def test_parse_version_regex_then_first_line(): + assert parse_version("mycli v1.2.3 (build)", r"(\d+\.\d+\.\d+)") == "1.2.3" + assert parse_version("mycli 9.9\nother", None) == "mycli 9.9" + + +def test_builtin_run_templates_render(): + oxa = build_command(OXAGEN_SPEC, "anthropic/claude-sonnet-5", "do it", budget="5") + assert oxa.startswith("oxagen --local --output-format json --model anthropic/claude-sonnet-5") + assert oxa.strip().endswith("'do it'") + + stella = build_command(STELLA_SPEC, "zai/glm-5.2", "do it", budget="5") + assert "stella --model zai/glm-5.2 --output-format json --budget 5 run 'do it'" == stella diff --git a/harbor/tests/test_metrics.py b/harbor/tests/test_metrics.py new file mode 100644 index 0000000..f97dc0a --- /dev/null +++ b/harbor/tests/test_metrics.py @@ -0,0 +1,102 @@ +"""Metrics extraction + token normalization. Pure — no Harbor required.""" + +import json + +from arena_harbor.metrics import extract_metrics, last_json_object +from arena_harbor.spec import MetricsSpec + + +def test_last_json_object_prefers_result_type(): + stdout = "\n".join( + [ + "starting banner", + json.dumps({"type": "event", "n": 1}), + json.dumps({"type": "result", "n": 2}), + "trailing log line", + ] + ) + assert last_json_object(stdout) == {"type": "result", "n": 2} + + +def test_last_json_object_falls_back_to_last_object(): + stdout = "\n".join(["not json", json.dumps({"a": 1}), json.dumps({"b": 2})]) + assert last_json_object(stdout) == {"b": 2} + + +def test_last_json_object_none_when_absent(): + assert last_json_object("just logs\nno json") is None + assert last_json_object("") is None + + +def test_json_metrics_subtract_cache_when_input_includes_it(): + # oxagen-shaped envelope: inputTokens includes the cached reads. + stdout = json.dumps( + { + "type": "result", + "usage": { + "inputTokens": 277032, + "outputTokens": 8432, + "cachedInputTokens": 244884, + }, + } + ) + spec = MetricsSpec( + kind="json_tail", + input_path="usage.inputTokens", + output_path="usage.outputTokens", + cache_read_path="usage.cachedInputTokens", + input_includes_cache=True, + ) + m = extract_metrics(stdout, spec) + assert m.input_tokens == 277032 - 244884 + assert m.cache_read_tokens == 244884 + assert m.output_tokens == 8432 + # total = normalized input + output + cache_read + assert m.total_tokens == (277032 - 244884) + 8432 + 244884 + + +def test_json_metrics_no_subtract_when_input_excludes_cache(): + stdout = json.dumps( + { + "usage": {"input_tokens": 1000, "output_tokens": 400, "cache_read_tokens": 9000}, + "total_cost_usd": 0.12, + } + ) + spec = MetricsSpec( + kind="json_tail", + input_path="usage.input_tokens", + output_path="usage.output_tokens", + cache_read_path="usage.cache_read_tokens", + cost_path="total_cost_usd", + input_includes_cache=False, + ) + m = extract_metrics(stdout, spec) + assert m.input_tokens == 1000 + assert m.cache_read_tokens == 9000 + assert m.cost_usd == 0.12 + + +def test_json_metrics_empty_when_unparseable(): + spec = MetricsSpec(kind="json_tail", input_path="usage.in") + assert extract_metrics("no json here", spec).is_empty() + + +def test_regex_metrics(): + text = "815.53s total · 83,086 tok · $0.2714 · 37 steps" + spec = MetricsSpec( + kind="regex", + cost_regex=r"\$([0-9.]+)", + total_regex=r"([0-9,]+)\s*tok", + ) + m = extract_metrics(text, spec) + assert m.cost_usd == 0.2714 + assert m.total_tokens == 83086 + + +def test_as_metadata_prefixes_and_skips_none(): + stdout = json.dumps({"usage": {"input_tokens": 10, "output_tokens": 5}}) + spec = MetricsSpec(input_path="usage.input_tokens", output_path="usage.output_tokens") + meta = extract_metrics(stdout, spec).as_metadata("my-agent") + assert meta["arena_my_agent_input_tokens"] == 10 + assert meta["arena_my_agent_output_tokens"] == 5 + assert "arena_my_agent_cost_usd" not in meta # None skipped diff --git a/harbor/tests/test_spec.py b/harbor/tests/test_spec.py new file mode 100644 index 0000000..6647e21 --- /dev/null +++ b/harbor/tests/test_spec.py @@ -0,0 +1,104 @@ +"""Spec loading + validation. Pure — no Harbor required.""" + +import json + +import pytest + +from arena_harbor.spec import ( + AgentSpec, + load_spec_file, + spec_from_dict, +) +from arena_harbor.specs_builtin import BUILTIN_SPECS, OXAGEN_SPEC, STELLA_SPEC + + +def _valid_dict(): + return { + "name": "my-agent", + "binary": "mycli", + "run_template": "{bin} solve --model {model} {instruction}", + "install": {"kind": "script", "script": "npm i -g mycli"}, + } + + +def test_spec_from_dict_minimal(): + spec = spec_from_dict(_valid_dict()) + assert isinstance(spec, AgentSpec) + assert spec.name == "my-agent" + assert spec.binary == "mycli" + assert spec.install.kind == "script" + + +@pytest.mark.parametrize("missing", ["name", "binary", "run_template"]) +def test_missing_required_field_raises(missing): + data = _valid_dict() + del data[missing] + with pytest.raises(ValueError, match=missing): + spec_from_dict(data) + + +def test_binary_install_requires_binary_env(): + data = _valid_dict() + data["install"] = {"kind": "binary"} + with pytest.raises(ValueError, match="binary_env"): + spec_from_dict(data) + + +def test_script_install_requires_script_or_env(): + data = _valid_dict() + data["install"] = {"kind": "script"} + with pytest.raises(ValueError, match="script"): + spec_from_dict(data) + + +def test_unknown_install_kind_rejected(): + data = _valid_dict() + data["install"] = {"kind": "docker"} + with pytest.raises(ValueError, match="binary.*script"): + spec_from_dict(data) + + +def test_load_json_spec(tmp_path): + p = tmp_path / "spec.json" + p.write_text(json.dumps(_valid_dict())) + spec = load_spec_file(p) + assert spec.name == "my-agent" + + +def test_load_toml_spec(tmp_path): + p = tmp_path / "spec.toml" + p.write_text( + 'name = "t"\n' + 'binary = "t"\n' + 'run_template = "{bin} {instruction}"\n' + "[install]\n" + 'kind = "script"\n' + 'script = "true"\n' + ) + spec = load_spec_file(p) + assert spec.name == "t" + assert spec.install.script == "true" + + +def test_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + load_spec_file(tmp_path / "nope.toml") + + +def test_example_spec_file_is_valid(): + # The shipped BYO template must always parse. + from pathlib import Path + + example = Path(__file__).resolve().parents[1] / "specs" / "byo.example.toml" + spec = load_spec_file(example) + assert spec.binary == "mycli" + assert spec.metrics is not None + + +def test_builtin_specs_are_wellformed(): + assert set(BUILTIN_SPECS) == {"oxagen", "stella"} + # oxagen reports combined input → must normalize by subtracting cache. + assert OXAGEN_SPEC.metrics.input_includes_cache is True + # stella uploads a native binary from a host env var. + assert STELLA_SPEC.install.kind == "binary" + assert STELLA_SPEC.install.binary_env == "ARENA_STELLA_BIN" From a94b3e51546da7d7cdac1e3d915ff7fc16b410d4 Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 14:08:24 -0700 Subject: [PATCH 05/18] ci(harbor): require Python 3.12 (Harbor's floor); pin harbor to 0.6.x Harbor requires Python >=3.12, so the CI job used 3.11 and could not resolve harbor. Bump CI to 3.12, set requires-python accordingly, and pin the dep to the tested 0.6.x line. --- .github/workflows/ci.yml | 2 +- harbor/pyproject.toml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31cf3cf..4893611 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" # Harbor requires >= 3.12 cache: pip - run: pip install -e '.[dev]' # Spec/metrics/command logic is Harbor-free; the agent tests exercise the diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index a20c953..d732ba8 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -7,7 +7,8 @@ name = "arena-harbor" version = "0.1.0" description = "Harbor adapter to benchmark your coding agent against Claude Code, Gemini, Codex and others under the official verifier" readme = "README.md" -requires-python = ">=3.10" +# Harbor itself requires Python >= 3.12, so this adapter does too. +requires-python = ">=3.12" license = { text = "MIT" } authors = [{ name = "Oxagen", email = "dev@oxagen.ai" }] keywords = ["benchmark", "harbor", "swe-bench", "ai-agents", "coding-agents"] @@ -16,13 +17,12 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] +# Pinned to the 0.6.x line the adapter is built and tested against. dependencies = [ - "harbor>=0.6.0", - "tomli>=2.0; python_version < '3.11'", + "harbor>=0.6,<0.7", ] [project.optional-dependencies] From b575e7aa394d14ba5745fd5db800d170dfc41cee Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 14:42:28 -0700 Subject: [PATCH 06/18] =?UTF-8?q?feat(gate):=20regression/drift=20gate=20?= =?UTF-8?q?=E2=80=94=20arena=20baseline=20+=20arena=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Snapshot a run's per-agent metrics as a committed baseline, then fail CI when a later run regresses: - arena baseline save [--agent A] -> arena-baseline.json - arena baseline show - arena gate [--require-significant] [--allow-task-mismatch] [...] exits non-zero on regression. Checks per agent: resolve-rate drop (strict, or only when it clears 95% CI noise with --require-significant), median token/cost increase past thresholds, wall-clock (reported; enforced with --speed-increase). Refuses to compare across different task sets unless --allow-task-mismatch. Thresholds via flags or arena-gate.json. src/summary.ts factors the per-agent aggregation into one source of truth shared by report + baseline + gate. 14 new unit tests (51 total); example CI workflow + gate config under examples/. README + METHODOLOGY document it. --- METHODOLOGY.md | 9 + README.md | 23 +++ examples/arena-gate.json | 21 +++ examples/regression-gate.yml | 60 ++++++ src/baseline.ts | 350 +++++++++++++++++++++++++++++++++++ src/cli.ts | 135 +++++++++++++- src/summary.ts | 84 +++++++++ test/baseline.test.ts | 240 ++++++++++++++++++++++++ 8 files changed, 919 insertions(+), 3 deletions(-) create mode 100644 examples/arena-gate.json create mode 100644 examples/regression-gate.yml create mode 100644 src/baseline.ts create mode 100644 src/summary.ts create mode 100644 test/baseline.test.ts diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 25768c9..42086a2 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -53,3 +53,12 @@ One run executes N agents × T tasks × K trials. Every trial: For headline resolve-rate claims, run this same protocol over [SWE-bench Verified](https://github.com/SWE-bench/SWE-bench) with a containerized runner and the **official** evaluator, with a pre-registered instance list (published random seed over the 500 verified instances, n ≥ 100). Arena's local suite is for harness development, smoke comparisons, and metric plumbing — the statistics and receipt discipline are identical either way. The [`harbor/`](harbor/) adapter is how you do this: it plugs any agent into [Harbor](https://www.harborframework.com/), whose Docker verifier runs each task's own FAIL_TO_PASS / PASS_TO_PASS suite — the agent never sees the grader. Because Harbor already bundles the industry-leading agents (Claude Code, Gemini, Codex, Cursor, Aider, …), a matched-model run of your agent against those built-ins is a fair, official-scored, receipted comparison — exactly the bar this document sets, at research scale. The adapter's token/cost numbers are annotations; the resolve-rate from Harbor's verifier is the score. + +## Tracking drift over time + +A comparison is a snapshot; a **baseline** turns it into a tripwire. `arena baseline save` records a run's per-agent resolve rate (with its Wilson interval), median tokens, cost, and wall clock; `arena gate` re-measures a later run and exits non-zero when it regresses past your thresholds. Two rules keep the gate honest, consistent with everything above: + +- **Compare like with like.** The gate refuses to run when the new task set differs from the baseline's — a resolve-rate diff across different problems is meaningless — unless you explicitly opt in. +- **Don't cry wolf on noise.** With `--require-significant`, an accuracy drop only fails once it clears the 95% CIs (baseline's lower bound above the new run's upper bound). Point-estimate mode is available for a strict "no drop at all" gate, but with small `n` that flakes; raise `--trials` or require significance. + +The baseline is a committed JSON artifact, so a regression shows up as a red check on the PR that caused it — the same day, not months later when someone re-benchmarks. diff --git a/README.md b/README.md index 918bb8f..2ca6acf 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,29 @@ Ships with `ByoAgent` (bring your own) plus `OxagenAgent` / `StellaAgent` specs. Before you publish a number, read [METHODOLOGY.md](METHODOLOGY.md). Short version: same model on every agent, pre-committed task set and trial count, p < 0.05 on the metric you're claiming, raw run directory published, and a conflict-of-interest note if you built one of the agents. +## Track drift: the regression gate + +Benchmarks aren't only for bragging — they keep your own agent from silently getting worse. Snapshot a run as a **baseline**, then fail CI when a later run regresses past your thresholds: + +```bash +# 1. Establish the baseline from a good run and commit it. +pnpm arena run --agents mine --model anthropic/claude-sonnet-5 --trials 5 -o results +pnpm arena baseline save results/run-… --agent mine # writes arena-baseline.json +git add arena-baseline.json && git commit -m "arena: baseline" + +# 2. In CI, re-run and gate. Non-zero exit = regression. +pnpm arena run --agents mine --model anthropic/claude-sonnet-5 --trials 5 -o results +pnpm arena gate results/run-… --require-significant +``` + +The gate compares the new run to the baseline per agent and fails on: + +- **accuracy** — resolve rate dropped past `--accuracy-drop` (default: any drop). With `--require-significant` a drop only fails once it clears 95% CI noise, so small-`n` jitter doesn't red-flag CI. +- **tokens / cost** — median rose past `--tokens-increase` (10%) / `--cost-increase` (15%). +- **speed** — median wall-clock, reported by default, enforced with `--speed-increase`. + +It refuses to compare across different task sets (a meaningless resolve-rate diff) unless you pass `--allow-task-mismatch`. Thresholds can also live in an `arena-gate.json` file. A ready-to-copy workflow is in [`examples/regression-gate.yml`](examples/regression-gate.yml). + ## Development ```bash diff --git a/examples/arena-gate.json b/examples/arena-gate.json new file mode 100644 index 0000000..fdaf3ef --- /dev/null +++ b/examples/arena-gate.json @@ -0,0 +1,21 @@ +{ + "//": "Regression-gate thresholds. Copy to your repo root as arena-gate.json (arena gate reads it by default) or pass --config. CLI flags override these.", + + "//accuracyMaxDropPoints": "Max allowed resolve-rate drop in points 0-1. 0 = any drop fails.", + "accuracyMaxDropPoints": 0, + + "//accuracyRequireSignificant": "Only fail accuracy once the drop clears 95% CI noise (baseline lower bound above the new run's upper bound). Avoids flaking CI on small samples.", + "accuracyRequireSignificant": true, + + "//tokensMaxIncreasePct": "Fail if median total tokens rose more than this %. null disables.", + "tokensMaxIncreasePct": 10, + + "//costMaxIncreasePct": "Fail if median computed cost rose more than this %. null disables.", + "costMaxIncreasePct": 15, + + "//speedMaxIncreasePct": "Fail if median wall-clock rose more than this %. null = report only (wall clock is environment-dependent).", + "speedMaxIncreasePct": null, + + "//allowTaskMismatch": "Permit gating a run over a different task set than the baseline. Off by default — comparing resolve rates across different tasks is meaningless.", + "allowTaskMismatch": false +} diff --git a/examples/regression-gate.yml b/examples/regression-gate.yml new file mode 100644 index 0000000..10f9216 --- /dev/null +++ b/examples/regression-gate.yml @@ -0,0 +1,60 @@ +# Example: fail CI when your agent regresses against a committed baseline. +# +# Copy to .github/workflows/ in your repo. Prerequisites: +# 1. Your agent has an Arena adapter (see src/adapters/ — ~80 lines). +# 2. You committed a baseline once: +# arena run --agents mine --model --trials 5 -o results +# arena baseline save results/run-… --agent mine # -> arena-baseline.json +# git add arena-baseline.json && git commit +# 3. Your provider key is a repo secret (here: MY_PROVIDER_KEY). +# +# Every PR then re-runs the benchmark and fails if the resolve rate drops +# (beyond CI noise) or tokens/cost blow past the thresholds. + +name: agent-regression-gate + +on: + pull_request: + workflow_dispatch: + +jobs: + gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + + # Re-run the same suite the baseline was measured on. Keep --trials high + # enough that --require-significant has the power to catch real drops. + - name: Benchmark the agent + env: + MY_PROVIDER_KEY: ${{ secrets.MY_PROVIDER_KEY }} + run: > + pnpm arena run + --agents mine + --model anthropic/claude-sonnet-5 + --trials 5 + --out results + + # Non-zero exit fails the job. --require-significant avoids flaking CI on + # small-sample jitter; drop it for a strict "no drop at all" gate. + - name: Gate against the committed baseline + run: > + pnpm arena gate results/run-* + --agent mine + --require-significant + --config arena-gate.json + + # Keep the run's receipts (transcripts, diffs, report) for inspection. + - uses: actions/upload-artifact@v4 + if: always() + with: + name: arena-run + path: results/ diff --git a/src/baseline.ts b/src/baseline.ts new file mode 100644 index 0000000..b83665d --- /dev/null +++ b/src/baseline.ts @@ -0,0 +1,350 @@ +/** + * Regression / drift gate. + * + * `arena baseline save` snapshots a run's per-agent headline metrics into a + * committed `arena-baseline.json`. Later, `arena gate` re-summarizes a fresh + * run and fails (exit 1) when an agent has drifted past the configured + * thresholds — a resolve-rate drop, or a token/cost/latency blow-up. This is + * how you keep your own agent honest over time: CI catches a regression the + * same day it lands. + * + * The gate compares like with like. A run over a different task set than the + * baseline is a hard error by default — you cannot compare resolve rates on + * different problems. And with `--require-significant`, an accuracy drop only + * fails when it clears statistical noise (baseline's lower 95% bound above the + * new run's upper bound), matching Arena's no-noise-theatre ethos. + */ + +import { existsSync, readFileSync, writeFileSync } from "node:fs"; + +import { isRecord } from "./parse.js"; +import { perAgentSummary, type AgentSummary } from "./summary.js"; +import type { RunManifest, TrialResult } from "./types.js"; + +export const BASELINE_SCHEMA = "arena-baseline/v1"; + +export interface BaselineAgent { + key: string; + adapter: string; + model: string; + scoredTrials: number; + passed: number; + successRate: number; + successCI: [number, number]; + medianTotalTokens: number | null; + medianWallClockSeconds: number | null; + medianComputedCost: number | null; +} + +export interface Baseline { + schema: typeof BASELINE_SCHEMA; + createdAt: string; + sourceRunId: string; + harnessVersion: string; + /** Sorted task ids the baseline was measured on. */ + taskIds: string[]; + trials: number; + agents: BaselineAgent[]; +} + +export interface GateThresholds { + /** Fail if success rate dropped by more than this many points (0–1). */ + accuracyMaxDropPoints: number; + /** Only fail accuracy when the drop is statistically real (CIs disjoint). */ + accuracyRequireSignificant: boolean; + /** Fail if median tokens rose more than this %, or null to not check. */ + tokensMaxIncreasePct: number | null; + /** Fail if median computed cost rose more than this %, or null. */ + costMaxIncreasePct: number | null; + /** Fail if median wall-clock rose more than this %, or null (default: report + * only — wall clock is environment-dependent and noisy). */ + speedMaxIncreasePct: number | null; + /** Allow a run whose task set differs from the baseline's. */ + allowTaskMismatch: boolean; +} + +export const DEFAULT_THRESHOLDS: GateThresholds = { + accuracyMaxDropPoints: 0, + accuracyRequireSignificant: false, + tokensMaxIncreasePct: 10, + costMaxIncreasePct: 15, + speedMaxIncreasePct: null, + allowTaskMismatch: false, +}; + +function toBaselineAgent(a: AgentSummary): BaselineAgent { + return { + key: a.key, + adapter: a.adapter, + model: a.model, + scoredTrials: a.scoredTrials, + passed: a.passed, + successRate: a.successRate, + successCI: a.successCI, + medianTotalTokens: a.medianTotalTokens, + medianWallClockSeconds: a.medianWallClockSeconds, + medianComputedCost: a.medianComputedCost, + }; +} + +/** + * Build a baseline from a run. `createdAt` is passed in (not read from the + * clock) so the function stays deterministic and unit-testable. + */ +export function buildBaseline( + manifest: RunManifest, + results: TrialResult[], + createdAt: string, + filterAdapter?: string, +): Baseline { + let agents = perAgentSummary(results); + if (filterAdapter) { + agents = agents.filter((a) => a.adapter === filterAdapter); + } + if (agents.length === 0) { + throw new Error( + filterAdapter + ? `no agent "${filterAdapter}" found in the run` + : "run has no agents to baseline", + ); + } + return { + schema: BASELINE_SCHEMA, + createdAt, + sourceRunId: manifest.runId, + harnessVersion: manifest.harness.version, + taskIds: [...manifest.taskIds].sort(), + trials: manifest.trials, + agents: agents.map(toBaselineAgent), + }; +} + +export function saveBaseline(path: string, baseline: Baseline): void { + writeFileSync(path, JSON.stringify(baseline, null, 2) + "\n"); +} + +export function loadBaseline(path: string): Baseline { + if (!existsSync(path)) { + throw new Error(`baseline not found at ${path} (run \`arena baseline save\` first)`); + } + const raw: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!isRecord(raw) || raw["schema"] !== BASELINE_SCHEMA || !Array.isArray(raw["agents"])) { + throw new Error(`${path} is not a valid ${BASELINE_SCHEMA} baseline`); + } + return raw as unknown as Baseline; +} + +/** Merge a partial JSON config file over the defaults. Unknown keys ignored. */ +export function loadGateConfig(path?: string): GateThresholds { + if (!path) return { ...DEFAULT_THRESHOLDS }; + if (!existsSync(path)) { + throw new Error(`gate config not found at ${path}`); + } + const raw: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!isRecord(raw)) throw new Error(`${path} is not a JSON object`); + const cfg: GateThresholds = { ...DEFAULT_THRESHOLDS }; + const numOrNull = (v: unknown, fallback: number | null): number | null => { + if (v === null) return null; + return typeof v === "number" && Number.isFinite(v) ? v : fallback; + }; + if ("accuracyMaxDropPoints" in raw) + cfg.accuracyMaxDropPoints = numOrNull(raw["accuracyMaxDropPoints"], cfg.accuracyMaxDropPoints) ?? 0; + if ("accuracyRequireSignificant" in raw) + cfg.accuracyRequireSignificant = raw["accuracyRequireSignificant"] === true; + if ("tokensMaxIncreasePct" in raw) + cfg.tokensMaxIncreasePct = numOrNull(raw["tokensMaxIncreasePct"], cfg.tokensMaxIncreasePct); + if ("costMaxIncreasePct" in raw) + cfg.costMaxIncreasePct = numOrNull(raw["costMaxIncreasePct"], cfg.costMaxIncreasePct); + if ("speedMaxIncreasePct" in raw) + cfg.speedMaxIncreasePct = numOrNull(raw["speedMaxIncreasePct"], cfg.speedMaxIncreasePct); + if ("allowTaskMismatch" in raw) + cfg.allowTaskMismatch = raw["allowTaskMismatch"] === true; + return cfg; +} + +export type CheckStatus = "pass" | "fail" | "warn" | "skip"; + +export interface GateCheck { + metric: "accuracy" | "tokens" | "cost" | "speed"; + status: CheckStatus; + baseline: number | null; + current: number | null; + detail: string; +} + +export interface GateAgentResult { + key: string; + regressed: boolean; + checks: GateCheck[]; +} + +export interface GateResult { + passed: boolean; + /** Non-null when the run's task set differs from the baseline's. */ + taskMismatch: { baselineOnly: string[]; runOnly: string[] } | null; + agents: GateAgentResult[]; + /** Baseline agents with no counterpart in the run (can't be checked). */ + unmatchedBaseline: string[]; + /** Fatal reasons unrelated to a specific metric (empty when clean). */ + blockers: string[]; +} + +function pctIncrease(base: number, current: number): number { + return ((current - base) / base) * 100; +} + +function checkIncrease( + metric: GateCheck["metric"], + base: number | null, + current: number | null, + maxPct: number | null, + unit: string, +): GateCheck { + if (base === null || current === null || base === 0) { + return { metric, status: "skip", baseline: base, current, detail: "no comparable data" }; + } + const delta = pctIncrease(base, current); + const sign = delta >= 0 ? "+" : ""; + const summary = `${base.toFixed(2)}${unit} → ${current.toFixed(2)}${unit} (${sign}${delta.toFixed(1)}%)`; + if (maxPct === null) { + return { metric, status: "warn", baseline: base, current, detail: `${summary} — not enforced` }; + } + if (delta > maxPct) { + return { metric, status: "fail", baseline: base, current, detail: `${summary} exceeds +${String(maxPct)}%` }; + } + return { metric, status: "pass", baseline: base, current, detail: `${summary} within +${String(maxPct)}%` }; +} + +function checkAccuracy( + base: BaselineAgent, + current: AgentSummary, + thresholds: GateThresholds, +): GateCheck { + const drop = base.successRate - current.successRate; + const pts = (x: number): string => `${(x * 100).toFixed(1)}%`; + const summary = `${pts(base.successRate)} → ${pts(current.successRate)}`; + if (drop <= thresholds.accuracyMaxDropPoints) { + const gained = current.successRate - base.successRate; + const detail = gained > 0 ? `${summary} (improved +${pts(gained)})` : `${summary} (within tolerance)`; + return { metric: "accuracy", status: "pass", baseline: base.successRate, current: current.successRate, detail }; + } + // Regressed beyond tolerance. Optionally demand statistical significance: + // baseline's lower 95% bound must sit above the new run's upper bound. + if (thresholds.accuracyRequireSignificant) { + const significant = base.successCI[0] > current.successCI[1]; + if (!significant) { + return { + metric: "accuracy", + status: "warn", + baseline: base.successRate, + current: current.successRate, + detail: `${summary} down ${pts(drop)} but within 95% CI noise — collect more trials`, + }; + } + } + return { + metric: "accuracy", + status: "fail", + baseline: base.successRate, + current: current.successRate, + detail: `${summary} — resolve rate dropped ${pts(drop)}`, + }; +} + +/** Compare a fresh run against the baseline and decide pass/fail. */ +export function evaluateGate( + baseline: Baseline, + manifest: RunManifest, + results: TrialResult[], + thresholds: GateThresholds, + filterAdapter?: string, +): GateResult { + const blockers: string[] = []; + + // Task-set parity: comparing resolve rates on different tasks is meaningless. + const runTasks = [...manifest.taskIds].sort(); + const baselineOnly = baseline.taskIds.filter((t) => !runTasks.includes(t)); + const runOnly = runTasks.filter((t) => !baseline.taskIds.includes(t)); + const taskMismatch = + baselineOnly.length > 0 || runOnly.length > 0 ? { baselineOnly, runOnly } : null; + if (taskMismatch && !thresholds.allowTaskMismatch) { + blockers.push( + "run task set differs from the baseline; resolve-rate comparison is invalid " + + "(pass --allow-task-mismatch to override)", + ); + } + + const current = perAgentSummary(results); + const currentByKey = new Map(current.map((a) => [a.key, a])); + + let baselineAgents = baseline.agents; + if (filterAdapter) { + baselineAgents = baselineAgents.filter((a) => a.adapter === filterAdapter); + if (baselineAgents.length === 0) { + blockers.push(`baseline has no agent "${filterAdapter}"`); + } + } + + const agents: GateAgentResult[] = []; + const unmatchedBaseline: string[] = []; + + for (const base of baselineAgents) { + const cur = currentByKey.get(base.key); + if (!cur) { + unmatchedBaseline.push(base.key); + continue; + } + const checks: GateCheck[] = [ + checkAccuracy(base, cur, thresholds), + checkIncrease("tokens", base.medianTotalTokens, cur.medianTotalTokens, thresholds.tokensMaxIncreasePct, ""), + checkIncrease("cost", base.medianComputedCost, cur.medianComputedCost, thresholds.costMaxIncreasePct, ""), + checkIncrease("speed", base.medianWallClockSeconds, cur.medianWallClockSeconds, thresholds.speedMaxIncreasePct, "s"), + ]; + agents.push({ key: base.key, regressed: checks.some((c) => c.status === "fail"), checks }); + } + + if (agents.length === 0 && blockers.length === 0) { + blockers.push("no baseline agent had a counterpart in the run — nothing to gate"); + } + + const passed = blockers.length === 0 && agents.every((a) => !a.regressed); + return { passed, taskMismatch, agents, unmatchedBaseline, blockers }; +} + +const ICON: Record = { pass: "✅", fail: "❌", warn: "⚠️", skip: "·" }; + +/** Render a gate result as a human-readable console report. */ +export function formatGateReport(result: GateResult, baseline: Baseline): string { + const lines: string[] = []; + lines.push( + `Regression gate vs baseline \`${baseline.sourceRunId}\` (${baseline.createdAt}, harness ${baseline.harnessVersion})`, + "", + ); + + for (const blocker of result.blockers) lines.push(`❌ ${blocker}`); + if (result.blockers.length) lines.push(""); + + if (result.taskMismatch) { + const { baselineOnly, runOnly } = result.taskMismatch; + lines.push("⚠️ Task set differs from baseline:"); + if (baselineOnly.length) lines.push(` only in baseline: ${baselineOnly.join(", ")}`); + if (runOnly.length) lines.push(` only in this run: ${runOnly.join(", ")}`); + lines.push(""); + } + + for (const agent of result.agents) { + lines.push(`${agent.regressed ? "❌" : "✅"} ${agent.key}`); + for (const c of agent.checks) { + lines.push(` ${ICON[c.status]} ${c.metric.padEnd(9)} ${c.detail}`); + } + lines.push(""); + } + + for (const key of result.unmatchedBaseline) { + lines.push(`⚠️ ${key}: in baseline but not in this run — not checked`); + } + if (result.unmatchedBaseline.length) lines.push(""); + + lines.push(result.passed ? "PASS — no regression beyond thresholds." : "FAIL — regression detected."); + return lines.join("\n"); +} diff --git a/src/cli.ts b/src/cli.ts index 01dd32f..19f3769 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,17 +9,30 @@ * arena verify [taskId…] — prove tasks discriminate: held-out * tests FAIL on the pristine workspace * and PASS on the reference solution + * arena baseline save — snapshot a run as the drift baseline + * arena baseline show — print the current baseline + * arena gate — fail (exit 1) if a run regressed vs + * the baseline (the CI drift gate) */ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { adapterNames, createAdapter } from "./adapters/index.js"; +import { + buildBaseline, + evaluateGate, + formatGateReport, + loadBaseline, + loadGateConfig, + saveBaseline, + type GateThresholds, +} from "./baseline.js"; import { executeRun } from "./orchestrator.js"; -import { generateReport } from "./report.js"; +import { generateReport, loadRun } from "./report.js"; import { applySolution, loadTasks, @@ -49,6 +62,12 @@ switch (command) { case "verify": cmdVerify(rest); break; + case "baseline": + cmdBaseline(rest); + break; + case "gate": + cmdGate(rest); + break; default: console.log( [ @@ -60,6 +79,9 @@ switch (command) { " run --agents [...] Run a benchmark (see below)", " report Regenerate report.md for a run", " verify [taskId...] Audit tasks: pristine must fail, solution must pass", + " baseline save Snapshot a run's metrics as the drift baseline", + " baseline show Print the current baseline", + " gate Fail (exit 1) if a run regressed vs the baseline", "", "Run options:", " --agents Comma list; each entry `adapter` or `adapter=model`", @@ -72,8 +94,21 @@ switch (command) { " --seed RNG seed for deterministic statistics (default: 42)", " --out Results root (default: ./results)", "", - "Example:", + "Gate options (fail CI when your agent drifts):", + " --baseline Baseline file (default: ./arena-baseline.json)", + " --config Gate thresholds JSON (default: ./arena-gate.json if present)", + " --agent Only gate this adapter", + " --accuracy-drop Max allowed resolve-rate drop, points 0-1 (default: 0)", + " --tokens-increase Max allowed median-token increase % (default: 10)", + " --cost-increase Max allowed median-cost increase % (default: 15)", + " --speed-increase Max allowed median wall-clock increase % (default: off)", + " --require-significant Only fail accuracy when the drop clears 95% CI noise", + " --allow-task-mismatch Permit a run over a different task set", + "", + "Examples:", " pnpm arena run --agents oxagen,claude-code --model anthropic/claude-sonnet-5 --trials 3", + " pnpm arena baseline save results/run-… --agent oxagen", + " pnpm arena gate results/run-… --require-significant # in CI", ].join("\n"), ); } @@ -209,3 +244,97 @@ function cmdVerify(argv: string[]): void { } console.log(`\nAll ${String(tasks.length)} task(s) discriminate correctly.`); } + +const DEFAULT_BASELINE_PATH = "arena-baseline.json"; + +function cmdBaseline(argv: string[]): void { + const [sub, ...subArgs] = argv; + if (sub === "save") { + const { values, positionals } = parseArgs({ + args: subArgs, + allowPositionals: true, + options: { out: { type: "string" }, agent: { type: "string" } }, + }); + const runDir = positionals[0]; + if (!runDir) { + console.error("Usage: arena baseline save [--out arena-baseline.json] [--agent ]"); + process.exit(1); + } + const { manifest, results } = loadRun(resolve(runDir)); + const baseline = buildBaseline( + manifest, + results, + new Date().toISOString(), + values.agent, + ); + const outPath = resolve(values.out ?? DEFAULT_BASELINE_PATH); + saveBaseline(outPath, baseline); + console.log( + `Baseline saved to ${outPath}\n` + + ` source run: ${baseline.sourceRunId} · tasks: ${String(baseline.taskIds.length)} · trials: ${String(baseline.trials)}\n` + + baseline.agents + .map( + (a) => + ` ${a.key}: ${(a.successRate * 100).toFixed(1)}% resolved` + + (a.medianTotalTokens !== null ? ` · ${Math.round(a.medianTotalTokens).toLocaleString()} tok` : "") + + (a.medianComputedCost !== null ? ` · $${a.medianComputedCost.toFixed(4)}` : ""), + ) + .join("\n"), + ); + return; + } + if (sub === "show") { + const { values } = parseArgs({ args: subArgs, options: { file: { type: "string" } } }); + const baseline = loadBaseline(resolve(values.file ?? DEFAULT_BASELINE_PATH)); + console.log(JSON.stringify(baseline, null, 2)); + return; + } + console.error("Usage: arena baseline …"); + process.exit(1); +} + +function cmdGate(argv: string[]): void { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + options: { + baseline: { type: "string" }, + config: { type: "string" }, + agent: { type: "string" }, + "accuracy-drop": { type: "string" }, + "tokens-increase": { type: "string" }, + "cost-increase": { type: "string" }, + "speed-increase": { type: "string" }, + "require-significant": { type: "boolean" }, + "allow-task-mismatch": { type: "boolean" }, + }, + }); + const runDir = positionals[0]; + if (!runDir) { + console.error("Usage: arena gate [--baseline arena-baseline.json] [--config arena-gate.json] [flags]"); + process.exit(1); + } + + const baseline = loadBaseline(resolve(values.baseline ?? DEFAULT_BASELINE_PATH)); + + // Config file (or built-in defaults), then CLI flag overrides on top. + const configPath = values.config ?? (existsSync(resolve("arena-gate.json")) ? "arena-gate.json" : undefined); + const thresholds: GateThresholds = loadGateConfig(configPath ? resolve(configPath) : undefined); + // A finite number, or null (a non-numeric flag value disables the check). + const num = (v: string | undefined): number | null => { + if (v === undefined) return null; + const n = parseFloat(v); + return Number.isFinite(n) ? n : null; + }; + if (values["accuracy-drop"] !== undefined) thresholds.accuracyMaxDropPoints = num(values["accuracy-drop"]) ?? 0; + if (values["tokens-increase"] !== undefined) thresholds.tokensMaxIncreasePct = num(values["tokens-increase"]); + if (values["cost-increase"] !== undefined) thresholds.costMaxIncreasePct = num(values["cost-increase"]); + if (values["speed-increase"] !== undefined) thresholds.speedMaxIncreasePct = num(values["speed-increase"]); + if (values["require-significant"]) thresholds.accuracyRequireSignificant = true; + if (values["allow-task-mismatch"]) thresholds.allowTaskMismatch = true; + + const { manifest, results } = loadRun(resolve(runDir)); + const result = evaluateGate(baseline, manifest, results, thresholds, values.agent); + console.log(formatGateReport(result, baseline)); + process.exit(result.passed ? 0 : 1); +} diff --git a/src/summary.ts b/src/summary.ts new file mode 100644 index 0000000..6cb1c6e --- /dev/null +++ b/src/summary.ts @@ -0,0 +1,84 @@ +/** + * Per-agent aggregation — the single source of truth for a run's headline + * numbers, shared by the report, the baseline snapshot, and the regression + * gate so all three agree exactly. + * + * `agent-error` trials (the CLI could not be invoked) are excluded from every + * figure, matching the report: a harness-side failure is never scored as the + * agent losing. + */ + +import { median, wilsonInterval } from "./stats.js"; +import type { TrialResult } from "./types.js"; + +export interface AgentSummary { + /** Stable identity: `"adapter (model)"`. */ + key: string; + adapter: string; + model: string; + /** Trials excluding `agent-error`. */ + scoredTrials: number; + errorTrials: number; + passed: number; + /** passed / scoredTrials (0 when nothing scored). */ + successRate: number; + /** 95% Wilson interval on the success rate. */ + successCI: [number, number]; + medianWallClockSeconds: number | null; + medianTotalTokens: number | null; + medianComputedCost: number | null; + medianAgentReportedCost: number | null; +} + +export function agentKey(r: TrialResult): string { + return `${r.agent.adapter} (${r.agent.model})`; +} + +function medianOrNull(values: number[]): number | null { + if (values.length === 0) return null; + const m = median(values); + return Number.isNaN(m) ? null : m; +} + +/** Aggregate raw trials into one summary row per agent, in first-seen order. */ +export function perAgentSummary(results: TrialResult[]): AgentSummary[] { + const order: string[] = []; + const byKey = new Map(); + for (const r of results) { + const key = agentKey(r); + if (!byKey.has(key)) { + byKey.set(key, []); + order.push(key); + } + (byKey.get(key) as TrialResult[]).push(r); + } + + return order.map((key) => { + const all = byKey.get(key) as TrialResult[]; + const scored = all.filter((r) => r.outcome !== "agent-error"); + const passed = scored.filter((r) => r.outcome === "passed").length; + const first = all[0] as TrialResult; + const costs = scored + .map((r) => r.cost.computedUsd) + .filter((c): c is number => c !== null); + const selfCosts = scored + .map((r) => r.cost.agentReportedUsd) + .filter((c): c is number => c !== null); + return { + key, + adapter: first.agent.adapter, + model: first.agent.model, + scoredTrials: scored.length, + errorTrials: all.length - scored.length, + passed, + successRate: scored.length ? passed / scored.length : 0, + successCI: wilsonInterval(passed, scored.length), + medianWallClockSeconds: medianOrNull( + scored.map((r) => r.timing.wallClockSeconds), + ), + medianTotalTokens: medianOrNull(scored.map((r) => r.tokens.total)), + medianComputedCost: medianOrNull(costs), + medianAgentReportedCost: medianOrNull(selfCosts), + }; + }); +} diff --git a/test/baseline.test.ts b/test/baseline.test.ts new file mode 100644 index 0000000..20d6570 --- /dev/null +++ b/test/baseline.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from "vitest"; + +import { + buildBaseline, + DEFAULT_THRESHOLDS, + evaluateGate, + loadGateConfig, + type Baseline, +} from "../src/baseline.js"; +import { perAgentSummary } from "../src/summary.js"; +import type { Outcome, RunManifest, TrialResult } from "../src/types.js"; + +// ── factories ──────────────────────────────────────────────────────────────── + +function trial( + taskId: string, + trialN: number, + adapter: string, + model: string, + outcome: Outcome, + over: { tokens?: number; wall?: number; cost?: number | null } = {}, +): TrialResult { + const tokens = over.tokens ?? 1000; + return { + id: `${taskId}-${adapter}-${model}-t${trialN}`, + taskId, + trial: trialN, + agent: { adapter, model, resolvedModel: model, version: "1", bin: adapter }, + outcome, + verify: { passed: outcome === "passed", output: "" }, + timing: { wallClockSeconds: over.wall ?? 10, agentReportedSeconds: null }, + tokens: { input: tokens, output: 0, cacheRead: 0, cacheWrite: 0, total: tokens }, + cost: { + computedUsd: over.cost === undefined ? 0.05 : over.cost, + agentReportedUsd: null, + pricingModel: model, + }, + activity: { + toolCalls: null, + iterations: null, + filesTouched: 1, + linesAdded: 1, + linesRemoved: 0, + diffBytes: 10, + }, + provenance: { runId: "run-x", startedAt: "", finishedAt: "" }, + transcriptPath: "t", + diffPath: "d", + }; +} + +/** N trials of one agent over `tasks`, the first `passCount` (per task) passing. */ +function agentRun( + adapter: string, + model: string, + tasks: string[], + passPerTask: number, + trialsPerTask: number, + over: { tokens?: number; wall?: number; cost?: number | null } = {}, +): TrialResult[] { + const out: TrialResult[] = []; + for (const task of tasks) { + for (let t = 1; t <= trialsPerTask; t++) { + out.push(trial(task, t, adapter, model, t <= passPerTask ? "passed" : "failed", over)); + } + } + return out; +} + +function manifest(taskIds: string[], trials: number): RunManifest { + return { + runId: "run-test", + createdAt: "2026-07-12T00:00:00.000Z", + harness: { name: "arena", version: "0.1.0", gitSha: "abc" }, + host: { platform: "linux", arch: "x64", node: "v24" }, + seed: 42, + trials, + budgetUsd: null, + timeoutSeconds: 600, + agents: [], + taskIds, + matchedModels: true, + reproduceCommand: "arena run …", + }; +} + +const CREATED = "2026-07-12T00:00:00.000Z"; + +// ── perAgentSummary ────────────────────────────────────────────────────────── + +describe("perAgentSummary", () => { + it("excludes agent-error trials from all figures", () => { + const results = [ + trial("t1", 1, "mine", "m", "passed"), + trial("t1", 2, "mine", "m", "agent-error"), + trial("t2", 1, "mine", "m", "failed"), + ]; + const row = perAgentSummary(results)[0]!; + expect(row.scoredTrials).toBe(2); + expect(row.errorTrials).toBe(1); + expect(row.passed).toBe(1); + expect(row.successRate).toBe(0.5); + }); + + it("returns null medians when there is no data", () => { + const row = perAgentSummary([trial("t", 1, "mine", "m", "agent-error")])[0]!; + expect(row.medianTotalTokens).toBeNull(); + expect(row.medianComputedCost).toBeNull(); + }); +}); + +// ── baseline build / config ────────────────────────────────────────────────── + +describe("buildBaseline", () => { + it("snapshots per-agent metrics and sorts task ids", () => { + const results = agentRun("mine", "m", ["t2", "t1"], 2, 2); + const base = buildBaseline(manifest(["t2", "t1"], 2), results, CREATED); + expect(base.taskIds).toEqual(["t1", "t2"]); + expect(base.agents).toHaveLength(1); + expect(base.agents[0]!.successRate).toBe(1); + }); + + it("can filter to a single adapter", () => { + const results = [ + ...agentRun("mine", "m", ["t1"], 1, 1), + ...agentRun("claude-code", "c", ["t1"], 1, 1), + ]; + const base = buildBaseline(manifest(["t1"], 1), results, CREATED, "mine"); + expect(base.agents.map((a) => a.adapter)).toEqual(["mine"]); + }); + + it("throws when the filtered adapter is absent", () => { + const results = agentRun("mine", "m", ["t1"], 1, 1); + expect(() => buildBaseline(manifest(["t1"], 1), results, CREATED, "ghost")).toThrow(/ghost/); + }); +}); + +describe("loadGateConfig", () => { + it("returns defaults when no path given", () => { + expect(loadGateConfig()).toEqual(DEFAULT_THRESHOLDS); + }); +}); + +// ── the gate ───────────────────────────────────────────────────────────────── + +function baselineFrom(results: TrialResult[], tasks: string[], trials: number): Baseline { + return buildBaseline(manifest(tasks, trials), results, CREATED); +} + +describe("evaluateGate", () => { + const tasks = ["t1", "t2"]; + + it("passes when the run matches the baseline", () => { + const base = baselineFrom(agentRun("mine", "m", tasks, 2, 2), tasks, 2); + const run = agentRun("mine", "m", tasks, 2, 2); + const res = evaluateGate(base, manifest(tasks, 2), run, DEFAULT_THRESHOLDS); + expect(res.passed).toBe(true); + expect(res.agents[0]!.regressed).toBe(false); + }); + + it("fails on a resolve-rate regression", () => { + const base = baselineFrom(agentRun("mine", "m", tasks, 10, 10), tasks, 10); + const run = agentRun("mine", "m", tasks, 0, 10); // 100% → 0% + const res = evaluateGate(base, manifest(tasks, 10), run, DEFAULT_THRESHOLDS); + expect(res.passed).toBe(false); + const acc = res.agents[0]!.checks.find((c) => c.metric === "accuracy"); + expect(acc?.status).toBe("fail"); + }); + + it("require-significant downgrades a noisy small-n drop to a warning", () => { + // 2/2 → 1/2: a drop, but the 95% CIs overlap heavily. + const base = baselineFrom(agentRun("mine", "m", ["t1"], 2, 2), ["t1"], 2); + const run = agentRun("mine", "m", ["t1"], 1, 2); + const strict = evaluateGate(base, manifest(["t1"], 2), run, DEFAULT_THRESHOLDS); + expect(strict.passed).toBe(false); // point-estimate drop fails by default + + const lenient = evaluateGate(base, manifest(["t1"], 2), run, { + ...DEFAULT_THRESHOLDS, + accuracyRequireSignificant: true, + }); + expect(lenient.passed).toBe(true); + expect(lenient.agents[0]!.checks.find((c) => c.metric === "accuracy")?.status).toBe("warn"); + }); + + it("fails on a token blow-up beyond the threshold", () => { + const base = baselineFrom(agentRun("mine", "m", tasks, 2, 2, { tokens: 1000 }), tasks, 2); + const run = agentRun("mine", "m", tasks, 2, 2, { tokens: 2000 }); // +100% + const res = evaluateGate(base, manifest(tasks, 2), run, DEFAULT_THRESHOLDS); + expect(res.passed).toBe(false); + expect(res.agents[0]!.checks.find((c) => c.metric === "tokens")?.status).toBe("fail"); + }); + + it("does not fail on a token increase within tolerance", () => { + const base = baselineFrom(agentRun("mine", "m", tasks, 2, 2, { tokens: 1000 }), tasks, 2); + const run = agentRun("mine", "m", tasks, 2, 2, { tokens: 1050 }); // +5% < 10% + const res = evaluateGate(base, manifest(tasks, 2), run, DEFAULT_THRESHOLDS); + expect(res.passed).toBe(true); + }); + + it("blocks on a task-set mismatch unless allowed", () => { + const base = baselineFrom(agentRun("mine", "m", ["t1", "t2"], 1, 1), ["t1", "t2"], 1); + const run = agentRun("mine", "m", ["t1", "t3"], 1, 1); + const strict = evaluateGate(base, manifest(["t1", "t3"], 1), run, DEFAULT_THRESHOLDS); + expect(strict.passed).toBe(false); + expect(strict.taskMismatch).not.toBeNull(); + expect(strict.blockers.length).toBeGreaterThan(0); + + const allowed = evaluateGate(base, manifest(["t1", "t3"], 1), run, { + ...DEFAULT_THRESHOLDS, + allowTaskMismatch: true, + }); + // task mismatch reported but not a blocker; accuracy on shared agent is fine + expect(allowed.blockers).toHaveLength(0); + }); + + it("blocks when no baseline agent appears in the run", () => { + const base = baselineFrom(agentRun("mine", "m", tasks, 2, 2), tasks, 2); + const run = agentRun("other", "x", tasks, 2, 2); + const res = evaluateGate(base, manifest(tasks, 2), run, DEFAULT_THRESHOLDS); + expect(res.passed).toBe(false); + expect(res.unmatchedBaseline).toContain("mine (m)"); + expect(res.blockers.length).toBeGreaterThan(0); + }); + + it("gates only the requested adapter", () => { + const baseResults = [ + ...agentRun("mine", "m", tasks, 2, 2), + ...agentRun("claude-code", "c", tasks, 2, 2), + ]; + const base = baselineFrom(baseResults, tasks, 2); + // claude-code regresses, mine holds steady; gating "mine" should pass. + const run = [ + ...agentRun("mine", "m", tasks, 2, 2), + ...agentRun("claude-code", "c", tasks, 0, 2), + ]; + const res = evaluateGate(base, manifest(tasks, 2), run, DEFAULT_THRESHOLDS, "mine"); + expect(res.passed).toBe(true); + expect(res.agents.map((a) => a.key)).toEqual(["mine (m)"]); + }); +}); From 71770d6c089d5a7230a6d2742c7ebc069b5437c0 Mon Sep 17 00:00:00 2001 From: macanderson Date: Sun, 12 Jul 2026 15:14:08 -0700 Subject: [PATCH 07/18] =?UTF-8?q?chore(security,quality):=20OSS=20hardenin?= =?UTF-8?q?g=20=E2=80=94=20Biome,=20Dependabot,=20Scorecard,=20dependency?= =?UTF-8?q?=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quality: - Add Biome (lint + format), pinned devDep, as a CI gate (pnpm lint). Normalized formatting across src/test; removed dead code Biome caught (unused pricing local in orchestrator; unused test locals). Supply chain / security (standard for popular OSS repos): - .github/dependabot.yml — weekly npm + pip + github-actions updates. - dependency-review workflow — blocks PRs adding high-severity vulns. - OpenSSF Scorecard workflow — security-posture scoring to the Security tab. - SECURITY.md — private vulnerability reporting policy. - CODEOWNERS — routes review to the maintainer. - Least-privilege 'permissions: contents: read' on CI. typecheck + 51 tests + biome ci all green. --- .github/CODEOWNERS | 3 + .github/dependabot.yml | 30 ++++++++ .github/workflows/ci.yml | 5 ++ .github/workflows/dependency-review.yml | 20 ++++++ .github/workflows/scorecard.yml | 37 ++++++++++ SECURITY.md | 21 ++++++ biome.json | 24 +++++++ package.json | 3 + pnpm-lock.yaml | 95 +++++++++++++++++++++++++ src/adapters/claude-code.ts | 10 +-- src/adapters/gemini.ts | 2 +- src/adapters/index.ts | 6 +- src/adapters/mock.ts | 3 +- src/adapters/oxagen.ts | 10 +-- src/adapters/stella.ts | 12 +--- src/baseline.ts | 65 +++++++++++++---- src/cli.ts | 49 +++++++------ src/orchestrator.ts | 29 +++----- src/parse.ts | 8 +-- src/pricing.ts | 5 +- src/report.ts | 38 +++++----- src/stats.ts | 11 +-- src/summary.ts | 8 +-- src/workspace.ts | 32 ++++----- test/adapters.test.ts | 13 ++-- test/baseline.test.ts | 2 +- test/parse.test.ts | 8 +-- test/pipeline.test.ts | 31 ++++---- 28 files changed, 395 insertions(+), 185 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/scorecard.yml create mode 100644 SECURITY.md create mode 100644 biome.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..104e85c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Default owner for everything in the repo. Routes review requests and, with +# branch protection's "require review from Code Owners", gates merges. +* @macanderson diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4444410 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,30 @@ +# Automated dependency + security updates. +# https://docs.github.com/code-security/dependabot +version: 2 +updates: + # JS/TS harness (root package.json, pnpm). + - package-ecosystem: npm + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + npm-dev: + dependency-type: development + update-types: [minor, patch] + + # Python Harbor adapter. + - package-ecosystem: pip + directory: /harbor + schedule: + interval: weekly + open-pull-requests-limit: 5 + + # The GitHub Actions we pin in workflows. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4893611..b56dfdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: branches: [main] pull_request: +# Least privilege by default (OpenSSF Scorecard checks for this). +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -19,6 +23,7 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm typecheck + - run: pnpm lint - run: pnpm test # Task audit: every task's held-out tests must FAIL on the pristine # workspace and PASS on the reference solution. diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..d474190 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,20 @@ +# Block PRs that introduce dependencies with known vulnerabilities or +# disallowed licenses. Free on public repos (uses the dependency graph). +name: dependency-review + +on: + pull_request: + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dependency Review + uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high + comment-summary-in-pr: on-failure diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..506b123 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,37 @@ +# OpenSSF Scorecard — measures the repo's security posture (branch protection, +# pinned actions, token permissions, etc.) and publishes results to the +# Security tab. This is the de-facto standard supply-chain health check for +# popular OSS repos. https://github.com/ossf/scorecard-action +name: scorecard + +on: + branch_protection_rule: + schedule: + - cron: "0 6 * * 1" # weekly, Monday 06:00 UTC + push: + branches: [main] + workflow_dispatch: + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write # upload SARIF to code scanning + id-token: write # publish results to the OpenSSF API + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Run analysis + uses: ossf/scorecard-action@v2.4.0 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results.sarif diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..9b3fe59 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Reporting a vulnerability + +Please **do not** open a public issue for security vulnerabilities. + +Report privately via GitHub's [**Report a vulnerability**](https://github.com/oxageninc/arena/security/advisories/new) button on the repository's Security tab (Security → Advisories → Report a vulnerability). This opens a private channel with the maintainers. + +We aim to acknowledge reports within 3 business days and to ship a fix or mitigation for confirmed high-severity issues as quickly as is practical. We'll coordinate a disclosure timeline with you and credit you in the advisory unless you prefer otherwise. + +## Scope + +Arena runs external agent CLIs and, via the Harbor adapter, untrusted task code inside containers. Reports we especially care about: + +- Command/argument injection in the harness or adapters (Arena passes argv arrays and shell-quotes task prompts specifically to avoid this — a bypass is in scope). +- A task or agent escaping the intended workspace/container boundary. +- Secrets (API keys) leaking into logs, transcripts, or committed results. + +## Supported versions + +This project is pre-1.0; only the latest `main` is supported. Please reproduce against the current `main` before reporting. diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..b7a00bb --- /dev/null +++ b/biome.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.5.2/schema.json", + "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, + "files": { "includes": ["src/**/*.ts", "test/**/*.ts"] }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "complexity": { + "useLiteralKeys": "off" + }, + "style": { + "noNonNullAssertion": "off" + } + } + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 100 + }, + "javascript": { "formatter": { "quoteStyle": "double" } } +} diff --git a/package.json b/package.json index 1ba88a5..8e23da1 100644 --- a/package.json +++ b/package.json @@ -24,9 +24,12 @@ "arena": "tsx src/cli.ts", "typecheck": "tsc --noEmit", "test": "vitest run", + "lint": "biome ci src test", + "format": "biome check --write src test", "verify:tasks": "tsx src/cli.ts verify" }, "devDependencies": { + "@biomejs/biome": "2.5.2", "@types/node": "^25.9.1", "tsx": "^4.19.2", "typescript": "^5.8.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d64c772..258c270 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + '@biomejs/biome': + specifier: 2.5.2 + version: 2.5.2 '@types/node': specifier: ^25.9.1 version: 25.9.5 @@ -23,6 +26,63 @@ importers: packages: + '@biomejs/biome@2.5.2': + resolution: {integrity: sha512-VQ3RCqr7JmDIX+w6stWYl+g/3bYofN3q2wDBHUKKc/c7i5QWrFKFBZYCYPWTE6agsUPMIZZe6/CMmVUfUAhkKA==} + engines: {node: '>=14.21.3'} + hasBin: true + + '@biomejs/cli-darwin-arm64@2.5.2': + resolution: {integrity: sha512-e7P3P7EkwFc/KiX2AHw4YDLIBOMfG9CPCAwy52k5Bp0dfhkozx9hf6wCmIr2QeXy2XeccJ3V/Sg+hDmzYEqxSg==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [darwin] + + '@biomejs/cli-darwin-x64@2.5.2': + resolution: {integrity: sha512-ymzMvjC1Jg0b9K0D26ZdARqFQXs7MocfLC5FOCGfkC0Ss+ACUJkX5364ZM5nT4NLZanHRZNVrZEy+Ibwcvux/g==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [darwin] + + '@biomejs/cli-linux-arm64-musl@2.5.2': + resolution: {integrity: sha512-w+ANG0ZvTu9IeEg9QnstoOnk6L0fpwJifW6aHR18+cb5Z39bkANItYjAfMrnvce5tmMK+IQ6nPX7/kQFdam5iw==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@biomejs/cli-linux-arm64@2.5.2': + resolution: {integrity: sha512-t7sseOmqND57uUWTwlawU6BYj+J06T/9EkydzBhkrgw/FK3QVhjU2wsJR0frljrKZ0/I8A/rYw7284QgqjQfIQ==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@biomejs/cli-linux-x64-musl@2.5.2': + resolution: {integrity: sha512-VArNLAzND063tF+XY0yPyM+DyahpzOMzOAvb7qs259nhjJWRjvjZdssuA+Rfl+l07+NOesKZ0Xu2yFrXyBMtzw==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@biomejs/cli-linux-x64@2.5.2': + resolution: {integrity: sha512-M/lOZrewzTCRDINbjhQ1gYYru37KlD3kJBQwwKCG0ckz5E9IZwIoJ3X0wBwRXA+yBDIwWUuPBHS67HzJY4dTfA==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@biomejs/cli-win32-arm64@2.5.2': + resolution: {integrity: sha512-kbjFFKyZlzYnAuw7sRy5qDoFG6zrP40UK08oPQsWK0ct3NMnGSt+Bs1iviEEyEIP57N5MrykGXdO/wRiaR4lww==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [win32] + + '@biomejs/cli-win32-x64@2.5.2': + resolution: {integrity: sha512-4InchVpdVmdkkkgjQqKpgvyu+VPnoF/7RPSw5YATgEVpt2j72wcCAeV5TwaE9ZGJUZWZn7v2CwSAj6CrMJEx8A==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [win32] + '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} engines: {node: '>=12'} @@ -692,6 +752,41 @@ packages: snapshots: + '@biomejs/biome@2.5.2': + optionalDependencies: + '@biomejs/cli-darwin-arm64': 2.5.2 + '@biomejs/cli-darwin-x64': 2.5.2 + '@biomejs/cli-linux-arm64': 2.5.2 + '@biomejs/cli-linux-arm64-musl': 2.5.2 + '@biomejs/cli-linux-x64': 2.5.2 + '@biomejs/cli-linux-x64-musl': 2.5.2 + '@biomejs/cli-win32-arm64': 2.5.2 + '@biomejs/cli-win32-x64': 2.5.2 + + '@biomejs/cli-darwin-arm64@2.5.2': + optional: true + + '@biomejs/cli-darwin-x64@2.5.2': + optional: true + + '@biomejs/cli-linux-arm64-musl@2.5.2': + optional: true + + '@biomejs/cli-linux-arm64@2.5.2': + optional: true + + '@biomejs/cli-linux-x64-musl@2.5.2': + optional: true + + '@biomejs/cli-linux-x64@2.5.2': + optional: true + + '@biomejs/cli-win32-arm64@2.5.2': + optional: true + + '@biomejs/cli-win32-x64@2.5.2': + optional: true + '@esbuild/aix-ppc64@0.21.5': optional: true diff --git a/src/adapters/claude-code.ts b/src/adapters/claude-code.ts index a06e95d..96e2b7e 100644 --- a/src/adapters/claude-code.ts +++ b/src/adapters/claude-code.ts @@ -13,9 +13,9 @@ * so counts map through directly. */ -import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; import { isRecord, num, parseJsonEnvelope } from "../parse.js"; import type { ParsedEnvelope } from "../types.js"; +import { Adapter, type AdapterRunArgs, emptyEnvelope, totalize } from "./base.js"; export class ClaudeCodeAdapter extends Adapter { readonly name = "claude-code"; @@ -63,13 +63,9 @@ export class ClaudeCodeAdapter extends Adapter { return { tokens, agentReportedUsd: - typeof reportedUsd === "number" && Number.isFinite(reportedUsd) - ? reportedUsd - : null, + typeof reportedUsd === "number" && Number.isFinite(reportedUsd) ? reportedUsd : null, agentReportedSeconds: - typeof durationMs === "number" && Number.isFinite(durationMs) - ? durationMs / 1000 - : null, + typeof durationMs === "number" && Number.isFinite(durationMs) ? durationMs / 1000 : null, toolCalls: null, // Claude's json envelope does not report tool calls. iterations: num(env["num_turns"], 0) || null, }; diff --git a/src/adapters/gemini.ts b/src/adapters/gemini.ts index b914a0b..2ae1f5f 100644 --- a/src/adapters/gemini.ts +++ b/src/adapters/gemini.ts @@ -17,9 +17,9 @@ * envelope. Re-verify against your installed version before publishing runs. */ -import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; import { isRecord, num, parseJsonEnvelope } from "../parse.js"; import type { ParsedEnvelope } from "../types.js"; +import { Adapter, type AdapterRunArgs, emptyEnvelope, totalize } from "./base.js"; export class GeminiAdapter extends Adapter { readonly name = "gemini"; diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 3cbb0da..6571e8c 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -1,5 +1,5 @@ import type { AgentSpec } from "../types.js"; -import { Adapter } from "./base.js"; +import type { Adapter } from "./base.js"; import { ClaudeCodeAdapter } from "./claude-code.js"; import { GeminiAdapter } from "./gemini.js"; import { MockAdapter } from "./mock.js"; @@ -23,9 +23,7 @@ export function adapterNames(): string[] { export function createAdapter(spec: AgentSpec): Adapter { const Ctor = registry[spec.adapter]; if (!Ctor) { - throw new Error( - `Unknown adapter "${spec.adapter}". Available: ${adapterNames().join(", ")}`, - ); + throw new Error(`Unknown adapter "${spec.adapter}". Available: ${adapterNames().join(", ")}`); } return new Ctor(spec); } diff --git a/src/adapters/mock.ts b/src/adapters/mock.ts index 87dc421..205dc22 100644 --- a/src/adapters/mock.ts +++ b/src/adapters/mock.ts @@ -11,9 +11,8 @@ import { cpSync, existsSync } from "node:fs"; import { join } from "node:path"; - -import { Adapter, totalize, type AdapterRunArgs, type ExecOutcome } from "./base.js"; import type { ParsedEnvelope } from "../types.js"; +import { Adapter, type AdapterRunArgs, type ExecOutcome, totalize } from "./base.js"; export class MockAdapter extends Adapter { readonly name = "mock"; diff --git a/src/adapters/oxagen.ts b/src/adapters/oxagen.ts index 114c529..b03201d 100644 --- a/src/adapters/oxagen.ts +++ b/src/adapters/oxagen.ts @@ -12,9 +12,9 @@ * inputTokens − cachedInputTokens. */ -import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; import { countOf, isRecord, num, parseJsonEnvelope } from "../parse.js"; import type { ParsedEnvelope } from "../types.js"; +import { Adapter, type AdapterRunArgs, emptyEnvelope, totalize } from "./base.js"; export class OxagenAdapter extends Adapter { readonly name = "oxagen"; @@ -30,9 +30,7 @@ export class OxagenAdapter extends Adapter { override env(args: AdapterRunArgs): Record { return { OXAGEN_MODEL_SLUG: args.model, - ...(args.budgetUsd !== undefined - ? { OXAGEN_BUDGET: String(args.budgetUsd) } - : {}), + ...(args.budgetUsd !== undefined ? { OXAGEN_BUDGET: String(args.budgetUsd) } : {}), }; } @@ -56,9 +54,7 @@ export class OxagenAdapter extends Adapter { tokens, agentReportedUsd: null, // envelope carries no cost agentReportedSeconds: - typeof durationMs === "number" && Number.isFinite(durationMs) - ? durationMs / 1000 - : null, + typeof durationMs === "number" && Number.isFinite(durationMs) ? durationMs / 1000 : null, toolCalls: countOf(env["commandsRun"]), iterations: num(env["steps"], 0) || null, }; diff --git a/src/adapters/stella.ts b/src/adapters/stella.ts index 22ad767..f6a93b5 100644 --- a/src/adapters/stella.ts +++ b/src/adapters/stella.ts @@ -12,9 +12,9 @@ * subtracted, mirroring the oxagen rule. */ -import { Adapter, emptyEnvelope, totalize, type AdapterRunArgs } from "./base.js"; import { countOf, isRecord, num, parseJsonEnvelope } from "../parse.js"; import type { ParsedEnvelope } from "../types.js"; +import { Adapter, type AdapterRunArgs, emptyEnvelope, totalize } from "./base.js"; export class StellaAdapter extends Adapter { readonly name = "stella"; @@ -33,9 +33,7 @@ export class StellaAdapter extends Adapter { return { STELLA_MODEL: this.resolveModel(args.model), STELLA_OUTPUT_FORMAT: "json", - ...(args.budgetUsd !== undefined - ? { STELLA_BUDGET: String(args.budgetUsd) } - : {}), + ...(args.budgetUsd !== undefined ? { STELLA_BUDGET: String(args.budgetUsd) } : {}), }; } @@ -46,11 +44,7 @@ export class StellaAdapter extends Adapter { const usage = isRecord(env["usage"]) ? env["usage"] : env; const rawInput = num(usage["inputTokens"] ?? usage["input_tokens"]); const cacheRead = Math.min( - num( - usage["cachedInputTokens"] ?? - usage["cacheReadTokens"] ?? - usage["cache_read_tokens"], - ), + num(usage["cachedInputTokens"] ?? usage["cacheReadTokens"] ?? usage["cache_read_tokens"]), rawInput, ); const tokens = totalize({ diff --git a/src/baseline.ts b/src/baseline.ts index b83665d..e11531d 100644 --- a/src/baseline.ts +++ b/src/baseline.ts @@ -18,7 +18,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { isRecord } from "./parse.js"; -import { perAgentSummary, type AgentSummary } from "./summary.js"; +import { type AgentSummary, perAgentSummary } from "./summary.js"; import type { RunManifest, TrialResult } from "./types.js"; export const BASELINE_SCHEMA = "arena-baseline/v1"; @@ -120,7 +120,7 @@ export function buildBaseline( } export function saveBaseline(path: string, baseline: Baseline): void { - writeFileSync(path, JSON.stringify(baseline, null, 2) + "\n"); + writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`); } export function loadBaseline(path: string): Baseline { @@ -148,7 +148,8 @@ export function loadGateConfig(path?: string): GateThresholds { return typeof v === "number" && Number.isFinite(v) ? v : fallback; }; if ("accuracyMaxDropPoints" in raw) - cfg.accuracyMaxDropPoints = numOrNull(raw["accuracyMaxDropPoints"], cfg.accuracyMaxDropPoints) ?? 0; + cfg.accuracyMaxDropPoints = + numOrNull(raw["accuracyMaxDropPoints"], cfg.accuracyMaxDropPoints) ?? 0; if ("accuracyRequireSignificant" in raw) cfg.accuracyRequireSignificant = raw["accuracyRequireSignificant"] === true; if ("tokensMaxIncreasePct" in raw) @@ -157,8 +158,7 @@ export function loadGateConfig(path?: string): GateThresholds { cfg.costMaxIncreasePct = numOrNull(raw["costMaxIncreasePct"], cfg.costMaxIncreasePct); if ("speedMaxIncreasePct" in raw) cfg.speedMaxIncreasePct = numOrNull(raw["speedMaxIncreasePct"], cfg.speedMaxIncreasePct); - if ("allowTaskMismatch" in raw) - cfg.allowTaskMismatch = raw["allowTaskMismatch"] === true; + if ("allowTaskMismatch" in raw) cfg.allowTaskMismatch = raw["allowTaskMismatch"] === true; return cfg; } @@ -210,9 +210,21 @@ function checkIncrease( return { metric, status: "warn", baseline: base, current, detail: `${summary} — not enforced` }; } if (delta > maxPct) { - return { metric, status: "fail", baseline: base, current, detail: `${summary} exceeds +${String(maxPct)}%` }; + return { + metric, + status: "fail", + baseline: base, + current, + detail: `${summary} exceeds +${String(maxPct)}%`, + }; } - return { metric, status: "pass", baseline: base, current, detail: `${summary} within +${String(maxPct)}%` }; + return { + metric, + status: "pass", + baseline: base, + current, + detail: `${summary} within +${String(maxPct)}%`, + }; } function checkAccuracy( @@ -225,8 +237,15 @@ function checkAccuracy( const summary = `${pts(base.successRate)} → ${pts(current.successRate)}`; if (drop <= thresholds.accuracyMaxDropPoints) { const gained = current.successRate - base.successRate; - const detail = gained > 0 ? `${summary} (improved +${pts(gained)})` : `${summary} (within tolerance)`; - return { metric: "accuracy", status: "pass", baseline: base.successRate, current: current.successRate, detail }; + const detail = + gained > 0 ? `${summary} (improved +${pts(gained)})` : `${summary} (within tolerance)`; + return { + metric: "accuracy", + status: "pass", + baseline: base.successRate, + current: current.successRate, + detail, + }; } // Regressed beyond tolerance. Optionally demand statistical significance: // baseline's lower 95% bound must sit above the new run's upper bound. @@ -296,9 +315,27 @@ export function evaluateGate( } const checks: GateCheck[] = [ checkAccuracy(base, cur, thresholds), - checkIncrease("tokens", base.medianTotalTokens, cur.medianTotalTokens, thresholds.tokensMaxIncreasePct, ""), - checkIncrease("cost", base.medianComputedCost, cur.medianComputedCost, thresholds.costMaxIncreasePct, ""), - checkIncrease("speed", base.medianWallClockSeconds, cur.medianWallClockSeconds, thresholds.speedMaxIncreasePct, "s"), + checkIncrease( + "tokens", + base.medianTotalTokens, + cur.medianTotalTokens, + thresholds.tokensMaxIncreasePct, + "", + ), + checkIncrease( + "cost", + base.medianComputedCost, + cur.medianComputedCost, + thresholds.costMaxIncreasePct, + "", + ), + checkIncrease( + "speed", + base.medianWallClockSeconds, + cur.medianWallClockSeconds, + thresholds.speedMaxIncreasePct, + "s", + ), ]; agents.push({ key: base.key, regressed: checks.some((c) => c.status === "fail"), checks }); } @@ -345,6 +382,8 @@ export function formatGateReport(result: GateResult, baseline: Baseline): string } if (result.unmatchedBaseline.length) lines.push(""); - lines.push(result.passed ? "PASS — no regression beyond thresholds." : "FAIL — regression detected."); + lines.push( + result.passed ? "PASS — no regression beyond thresholds." : "FAIL — regression detected.", + ); return lines.join("\n"); } diff --git a/src/cli.ts b/src/cli.ts index 19f3769..aa01a57 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -17,7 +17,7 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, dirname, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; @@ -26,20 +26,15 @@ import { buildBaseline, evaluateGate, formatGateReport, + type GateThresholds, loadBaseline, loadGateConfig, saveBaseline, - type GateThresholds, } from "./baseline.js"; import { executeRun } from "./orchestrator.js"; import { generateReport, loadRun } from "./report.js"; -import { - applySolution, - loadTasks, - runVerification, - seedWorkspace, -} from "./workspace.js"; import type { AgentSpec, LoadedTask } from "./types.js"; +import { applySolution, loadTasks, runVerification, seedWorkspace } from "./workspace.js"; const HERE = dirname(fileURLToPath(import.meta.url)); const TASK_ROOT = join(HERE, "..", "tasks"); @@ -156,9 +151,7 @@ async function cmdRun(argv: string[]): Promise { const [adapter, model] = entry.split("=") as [string, string | undefined]; const resolved = model ?? defaultModel; if (!resolved) { - console.error( - `No model for agent "${adapter}". Pass --model or use ${adapter}=.`, - ); + console.error(`No model for agent "${adapter}". Pass --model or use ${adapter}=.`); process.exit(1); } return { adapter: adapter.trim(), model: resolved.trim() }; @@ -257,16 +250,13 @@ function cmdBaseline(argv: string[]): void { }); const runDir = positionals[0]; if (!runDir) { - console.error("Usage: arena baseline save [--out arena-baseline.json] [--agent ]"); + console.error( + "Usage: arena baseline save [--out arena-baseline.json] [--agent ]", + ); process.exit(1); } const { manifest, results } = loadRun(resolve(runDir)); - const baseline = buildBaseline( - manifest, - results, - new Date().toISOString(), - values.agent, - ); + const baseline = buildBaseline(manifest, results, new Date().toISOString(), values.agent); const outPath = resolve(values.out ?? DEFAULT_BASELINE_PATH); saveBaseline(outPath, baseline); console.log( @@ -276,7 +266,9 @@ function cmdBaseline(argv: string[]): void { .map( (a) => ` ${a.key}: ${(a.successRate * 100).toFixed(1)}% resolved` + - (a.medianTotalTokens !== null ? ` · ${Math.round(a.medianTotalTokens).toLocaleString()} tok` : "") + + (a.medianTotalTokens !== null + ? ` · ${Math.round(a.medianTotalTokens).toLocaleString()} tok` + : "") + (a.medianComputedCost !== null ? ` · $${a.medianComputedCost.toFixed(4)}` : ""), ) .join("\n"), @@ -311,14 +303,17 @@ function cmdGate(argv: string[]): void { }); const runDir = positionals[0]; if (!runDir) { - console.error("Usage: arena gate [--baseline arena-baseline.json] [--config arena-gate.json] [flags]"); + console.error( + "Usage: arena gate [--baseline arena-baseline.json] [--config arena-gate.json] [flags]", + ); process.exit(1); } const baseline = loadBaseline(resolve(values.baseline ?? DEFAULT_BASELINE_PATH)); // Config file (or built-in defaults), then CLI flag overrides on top. - const configPath = values.config ?? (existsSync(resolve("arena-gate.json")) ? "arena-gate.json" : undefined); + const configPath = + values.config ?? (existsSync(resolve("arena-gate.json")) ? "arena-gate.json" : undefined); const thresholds: GateThresholds = loadGateConfig(configPath ? resolve(configPath) : undefined); // A finite number, or null (a non-numeric flag value disables the check). const num = (v: string | undefined): number | null => { @@ -326,10 +321,14 @@ function cmdGate(argv: string[]): void { const n = parseFloat(v); return Number.isFinite(n) ? n : null; }; - if (values["accuracy-drop"] !== undefined) thresholds.accuracyMaxDropPoints = num(values["accuracy-drop"]) ?? 0; - if (values["tokens-increase"] !== undefined) thresholds.tokensMaxIncreasePct = num(values["tokens-increase"]); - if (values["cost-increase"] !== undefined) thresholds.costMaxIncreasePct = num(values["cost-increase"]); - if (values["speed-increase"] !== undefined) thresholds.speedMaxIncreasePct = num(values["speed-increase"]); + if (values["accuracy-drop"] !== undefined) + thresholds.accuracyMaxDropPoints = num(values["accuracy-drop"]) ?? 0; + if (values["tokens-increase"] !== undefined) + thresholds.tokensMaxIncreasePct = num(values["tokens-increase"]); + if (values["cost-increase"] !== undefined) + thresholds.costMaxIncreasePct = num(values["cost-increase"]); + if (values["speed-increase"] !== undefined) + thresholds.speedMaxIncreasePct = num(values["speed-increase"]); if (values["require-significant"]) thresholds.accuracyRequireSignificant = true; if (values["allow-task-mismatch"]) thresholds.allowTaskMismatch = true; diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 1e2f16d..ce675f4 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -14,21 +14,15 @@ */ import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { mkdirSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir, platform, arch } from "node:os"; +import { arch, platform, tmpdir } from "node:os"; import { join } from "node:path"; -import { randomUUID } from "node:crypto"; -import { createAdapter, type Adapter } from "./adapters/index.js"; +import { type Adapter, createAdapter } from "./adapters/index.js"; import { MockAdapter } from "./adapters/mock.js"; import { diffStats, sanitizeSegment } from "./parse.js"; import { computeCost, loadPricing } from "./pricing.js"; -import { - buildPrompt, - collectDiff, - runVerification, - seedWorkspace, -} from "./workspace.js"; import type { AgentSpec, LoadedTask, @@ -37,18 +31,17 @@ import type { RunManifest, TrialResult, } from "./types.js"; +import { buildPrompt, collectDiff, runVerification, seedWorkspace } from "./workspace.js"; export const HARNESS_VERSION = "0.1.0"; -export interface RunProgress { - (message: string): void; -} +export type RunProgress = (message: string) => void; export async function executeRun( config: RunConfig, log: RunProgress = () => {}, ): Promise<{ runDir: string; manifest: RunManifest; results: TrialResult[] }> { - const pricing = loadPricing(); + const _pricing = loadPricing(); const runId = `run-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}-${randomUUID().slice(0, 8)}`; const runDir = join(config.outDir, runId); mkdirSync(join(runDir, "trials"), { recursive: true }); @@ -105,10 +98,7 @@ export async function executeRun( log(`▶ trial ${trial}/${config.trials} · ${task.id} · ${spec.adapter} (${spec.model})`); const result = await runOne(task, spec, adapter, trial, config, runId, runDir); results.push(result); - writeFileSync( - join(runDir, "trials", `${result.id}.json`), - JSON.stringify(result, null, 2), - ); + writeFileSync(join(runDir, "trials", `${result.id}.json`), JSON.stringify(result, null, 2)); const mark = result.outcome === "passed" ? "✅" : result.outcome === "agent-error" ? "⚠️" : "❌"; log( @@ -119,10 +109,7 @@ export async function executeRun( } } - writeFileSync( - join(runDir, "results.json"), - JSON.stringify({ manifest, results }, null, 2), - ); + writeFileSync(join(runDir, "results.json"), JSON.stringify({ manifest, results }, null, 2)); return { runDir, manifest, results }; } diff --git a/src/parse.ts b/src/parse.ts index b8901c7..1827144 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -5,9 +5,7 @@ /** Coerce an unknown JSON value into a finite non-negative number, else fallback. */ export function num(value: unknown, fallback = 0): number { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? value - : fallback; + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback; } /** A count that may arrive as an array (length) or a number. Null when absent. */ @@ -28,9 +26,7 @@ export function isRecord(value: unknown): value is Record { * per line). We want the last `type: "result"` object, falling back to the * last parseable object. Banners and log lines are ignored. */ -export function parseJsonEnvelope( - stdout: string, -): Record | null { +export function parseJsonEnvelope(stdout: string): Record | null { if (!stdout) return null; const lines = stdout diff --git a/src/pricing.ts b/src/pricing.ts index 9f1e183..b351de2 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -9,11 +9,10 @@ */ import { readFileSync } from "node:fs"; -import { join, dirname } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; - -import type { PricingTable, TokenCounts } from "./types.js"; import { isRecord, num } from "./parse.js"; +import type { PricingTable, TokenCounts } from "./types.js"; const HERE = dirname(fileURLToPath(import.meta.url)); diff --git a/src/report.ts b/src/report.ts index c8e2ab6..ab17ae6 100644 --- a/src/report.ts +++ b/src/report.ts @@ -14,23 +14,15 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; - -import { - mcnemarExact, - median, - pairedBootstrapDelta, - wilsonInterval, -} from "./stats.js"; import { isRecord } from "./parse.js"; +import { mcnemarExact, median, pairedBootstrapDelta, wilsonInterval } from "./stats.js"; import type { RunManifest, TrialResult } from "./types.js"; export function loadRun(runDir: string): { manifest: RunManifest; results: TrialResult[]; } { - const raw: unknown = JSON.parse( - readFileSync(join(runDir, "results.json"), "utf8"), - ); + const raw: unknown = JSON.parse(readFileSync(join(runDir, "results.json"), "utf8")); if (!isRecord(raw) || !isRecord(raw["manifest"]) || !Array.isArray(raw["results"])) { throw new Error(`Malformed results.json in ${runDir}`); } @@ -67,15 +59,21 @@ export function generateReport(runDir: string): string { const lines: string[] = []; lines.push(`# Arena run report — \`${manifest.runId}\``, ""); - lines.push(`- Harness: ${manifest.harness.name} v${manifest.harness.version} (git ${manifest.harness.gitSha})`); + lines.push( + `- Harness: ${manifest.harness.name} v${manifest.harness.version} (git ${manifest.harness.gitSha})`, + ); lines.push(`- Host: ${manifest.host.platform}/${manifest.host.arch}, node ${manifest.host.node}`); lines.push(`- Created: ${manifest.createdAt}`); - lines.push(`- Trials per (task, agent): ${String(manifest.trials)} · timeout ${String(manifest.timeoutSeconds)}s · budget ${manifest.budgetUsd === null ? "none" : `$${String(manifest.budgetUsd)}`} · seed ${String(manifest.seed)}`); + lines.push( + `- Trials per (task, agent): ${String(manifest.trials)} · timeout ${String(manifest.timeoutSeconds)}s · budget ${manifest.budgetUsd === null ? "none" : `$${String(manifest.budgetUsd)}`} · seed ${String(manifest.seed)}`, + ); lines.push(`- Tasks (${String(manifest.taskIds.length)}): ${manifest.taskIds.join(", ")}`); lines.push(""); lines.push("Agents:"); for (const a of manifest.agents) { - lines.push(`- **${a.adapter}** · model \`${a.model}\` (resolved \`${a.resolvedModel}\`) · version \`${a.version}\``); + lines.push( + `- **${a.adapter}** · model \`${a.model}\` (resolved \`${a.resolvedModel}\`) · version \`${a.version}\``, + ); } lines.push(""); @@ -111,12 +109,8 @@ export function generateReport(runDir: string): string { const [lo, hi] = wilsonInterval(passed, scored.length); const wall = median(scored.map((r) => r.timing.wallClockSeconds)); const toks = median(scored.map((r) => r.tokens.total)); - const costs = scored - .map((r) => r.cost.computedUsd) - .filter((c): c is number => c !== null); - const self = scored - .map((r) => r.cost.agentReportedUsd) - .filter((c): c is number => c !== null); + const costs = scored.map((r) => r.cost.computedUsd).filter((c): c is number => c !== null); + const self = scored.map((r) => r.cost.agentReportedUsd).filter((c): c is number => c !== null); lines.push( `| ${key} | ${String(scored.length)} | ${String(passed)} | ${pct(scored.length ? passed / scored.length : 0)} (${pct(lo)}–${pct(hi)}) | ${wall.toFixed(1)}s | ${Math.round(toks).toLocaleString()} | ${fmtUsd(costs.length ? median(costs) : null)} | ${fmtUsd(self.length ? median(self) : null)} |`, ); @@ -161,7 +155,11 @@ export function generateReport(runDir: string): string { }); lines.push(`| ${taskId} | ${cells.join(" | ")} |`); } - lines.push("", "One symbol per trial: ✅ passed · ❌ failed · ⏱ timeout · ⚠️ agent-error (excluded).", ""); + lines.push( + "", + "One symbol per trial: ✅ passed · ❌ failed · ⏱ timeout · ⚠️ agent-error (excluded).", + "", + ); // ── Excluded trials ── if (errorTrials.length > 0) { diff --git a/src/stats.ts b/src/stats.ts index 776f372..9bc2729 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -6,20 +6,13 @@ */ /** 95% Wilson score interval for a binomial proportion. */ -export function wilsonInterval( - successes: number, - n: number, - z = 1.96, -): [number, number] { +export function wilsonInterval(successes: number, n: number, z = 1.96): [number, number] { if (n === 0) return [0, 0]; const p = successes / n; const denom = 1 + (z * z) / n; const center = p + (z * z) / (2 * n); const margin = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n)); - return [ - Math.max(0, (center - margin) / denom), - Math.min(1, (center + margin) / denom), - ]; + return [Math.max(0, (center - margin) / denom), Math.min(1, (center + margin) / denom)]; } /** diff --git a/src/summary.ts b/src/summary.ts index 6cb1c6e..6c8508c 100644 --- a/src/summary.ts +++ b/src/summary.ts @@ -58,9 +58,7 @@ export function perAgentSummary(results: TrialResult[]): AgentSummary[] { const scored = all.filter((r) => r.outcome !== "agent-error"); const passed = scored.filter((r) => r.outcome === "passed").length; const first = all[0] as TrialResult; - const costs = scored - .map((r) => r.cost.computedUsd) - .filter((c): c is number => c !== null); + const costs = scored.map((r) => r.cost.computedUsd).filter((c): c is number => c !== null); const selfCosts = scored .map((r) => r.cost.agentReportedUsd) .filter((c): c is number => c !== null); @@ -73,9 +71,7 @@ export function perAgentSummary(results: TrialResult[]): AgentSummary[] { passed, successRate: scored.length ? passed / scored.length : 0, successCI: wilsonInterval(passed, scored.length), - medianWallClockSeconds: medianOrNull( - scored.map((r) => r.timing.wallClockSeconds), - ), + medianWallClockSeconds: medianOrNull(scored.map((r) => r.timing.wallClockSeconds)), medianTotalTokens: medianOrNull(scored.map((r) => r.tokens.total)), medianComputedCost: medianOrNull(costs), medianAgentReportedCost: medianOrNull(selfCosts), diff --git a/src/workspace.ts b/src/workspace.ts index 90c46c0..e5062bd 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -19,9 +19,8 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; - -import type { LoadedTask, TaskDef } from "./types.js"; import { isRecord } from "./parse.js"; +import type { LoadedTask, TaskDef } from "./types.js"; const VERIFY_DIR = ".arena-verify"; @@ -68,10 +67,10 @@ export function buildPrompt(task: TaskDef): string { export function seedWorkspace(task: LoadedTask, workDir: string): void { mkdirSync(workDir, { recursive: true }); cpSync(join(task.dir, "workspace"), workDir, { recursive: true }); - writeFileSync(join(workDir, "TASK.md"), buildPrompt(task) + "\n"); + writeFileSync(join(workDir, "TASK.md"), `${buildPrompt(task)}\n`); writeFileSync( join(workDir, ".gitignore"), - ["node_modules/", "dist/", "coverage/", "*.log", `${VERIFY_DIR}/`].join("\n") + "\n", + `${["node_modules/", "dist/", "coverage/", "*.log", `${VERIFY_DIR}/`].join("\n")}\n`, ); const git = (args: string[]): void => { execFileSync("git", args, { cwd: workDir, stdio: "ignore" }); @@ -107,10 +106,7 @@ export interface VerifyResult { * Any pre-existing `.arena-verify/` content (agent-planted or stale) is * removed first. */ -export function runVerification( - task: LoadedTask, - workDir: string, -): VerifyResult { +export function runVerification(task: LoadedTask, workDir: string): VerifyResult { const target = join(workDir, VERIFY_DIR); rmSync(target, { recursive: true, force: true }); cpSync(join(task.dir, "verify"), target, { recursive: true }); @@ -122,17 +118,13 @@ export function runVerification( const { FORCE_COLOR: _forceColor, ...baseEnv } = process.env; const env = { ...baseEnv, NO_COLOR: "1" }; try { - const output = execFileSync( - process.execPath, - ["--test", `${VERIFY_DIR}/**/*.test.mjs`], - { - cwd: workDir, - encoding: "utf8", - timeout: timeoutMs, - maxBuffer: 16 * 1024 * 1024, - env, - }, - ); + const output = execFileSync(process.execPath, ["--test", `${VERIFY_DIR}/**/*.test.mjs`], { + cwd: workDir, + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + env, + }); return { passed: true, output: tail(output) }; } catch (error) { const e = error as { stdout?: string | Buffer; stderr?: string | Buffer; message?: string }; @@ -156,5 +148,5 @@ export function applySolution(task: LoadedTask, workDir: string): void { function tail(text: string, maxChars = 4000): string { const trimmed = text.trim(); - return trimmed.length <= maxChars ? trimmed : "…" + trimmed.slice(-maxChars); + return trimmed.length <= maxChars ? trimmed : `…${trimmed.slice(-maxChars)}`; } diff --git a/test/adapters.test.ts b/test/adapters.test.ts index 6113856..88c834f 100644 --- a/test/adapters.test.ts +++ b/test/adapters.test.ts @@ -1,11 +1,10 @@ import { describe, expect, it } from "vitest"; - +import type { AdapterRunArgs } from "../src/adapters/base.js"; import { ClaudeCodeAdapter } from "../src/adapters/claude-code.js"; import { GeminiAdapter } from "../src/adapters/gemini.js"; +import { adapterNames, createAdapter } from "../src/adapters/index.js"; import { OxagenAdapter } from "../src/adapters/oxagen.js"; import { StellaAdapter } from "../src/adapters/stella.js"; -import { adapterNames, createAdapter } from "../src/adapters/index.js"; -import type { AdapterRunArgs } from "../src/adapters/base.js"; const baseArgs: AdapterRunArgs = { prompt: "fix the bug", @@ -114,9 +113,7 @@ describe("stella", () => { it("prefixes bare model ids with zai/", () => { expect(adapter.resolveModel("glm-5.2")).toBe("zai/glm-5.2"); - expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe( - "anthropic/claude-sonnet-5", - ); + expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe("anthropic/claude-sonnet-5"); }); it("passes config via env, prompt via `run`", () => { @@ -155,7 +152,9 @@ describe("gemini", () => { response: "done", stats: { models: { - "gemini-2.5-pro": { tokens: { prompt: 5000, candidates: 700, cached: 2000, total: 5700 } }, + "gemini-2.5-pro": { + tokens: { prompt: 5000, candidates: 700, cached: 2000, total: 5700 }, + }, "gemini-2.5-flash": { tokens: { prompt: 100, candidates: 20, cached: 0, total: 120 } }, }, tools: { totalCalls: 9 }, diff --git a/test/baseline.test.ts b/test/baseline.test.ts index 20d6570..f703ac6 100644 --- a/test/baseline.test.ts +++ b/test/baseline.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from "vitest"; import { + type Baseline, buildBaseline, DEFAULT_THRESHOLDS, evaluateGate, loadGateConfig, - type Baseline, } from "../src/baseline.js"; import { perAgentSummary } from "../src/summary.js"; import type { Outcome, RunManifest, TrialResult } from "../src/types.js"; diff --git a/test/parse.test.ts b/test/parse.test.ts index c68fce8..88129ba 100644 --- a/test/parse.test.ts +++ b/test/parse.test.ts @@ -18,12 +18,12 @@ describe("parseJsonEnvelope", () => { }); it("falls back to the last parseable object", () => { - const stdout = ['not json', '{"a":1}', '{"b":2}'].join("\n"); + const stdout = ["not json", '{"a":1}', '{"b":2}'].join("\n"); expect(parseJsonEnvelope(stdout)).toEqual({ b: 2 }); }); it("ignores log lines interleaved with JSON", () => { - const stdout = ['INFO starting', '{"type":"result","ok":true}', 'INFO done'].join("\n"); + const stdout = ["INFO starting", '{"type":"result","ok":true}', "INFO done"].join("\n"); expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", ok: true }); }); }); @@ -45,9 +45,7 @@ describe("num / countOf", () => { describe("sanitizeSegment", () => { it("flattens model slugs into safe filenames", () => { - expect(sanitizeSegment("anthropic/claude-opus-4.8")).toBe( - "anthropic_claude-opus-4.8", - ); + expect(sanitizeSegment("anthropic/claude-opus-4.8")).toBe("anthropic_claude-opus-4.8"); }); }); diff --git a/test/pipeline.test.ts b/test/pipeline.test.ts index d13fe0c..91fe864 100644 --- a/test/pipeline.test.ts +++ b/test/pipeline.test.ts @@ -4,27 +4,15 @@ * result persistence, and report generation. */ -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, dirname } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterAll, describe, expect, it } from "vitest"; import { executeRun } from "../src/orchestrator.js"; import { generateReport } from "../src/report.js"; -import { - applySolution, - loadTasks, - runVerification, - seedWorkspace, -} from "../src/workspace.js"; +import { applySolution, loadTasks, runVerification, seedWorkspace } from "../src/workspace.js"; const HERE = dirname(fileURLToPath(import.meta.url)); const TASK_ROOT = join(HERE, "..", "tasks"); @@ -54,13 +42,18 @@ describe("task fixtures", () => { const pristineDir = scratch(); seedWorkspace(task, pristineDir); const pristine = runVerification(task, pristineDir); - expect(pristine.passed, `${task.id}: held-out tests must FAIL on the pristine workspace`).toBe(false); + expect( + pristine.passed, + `${task.id}: held-out tests must FAIL on the pristine workspace`, + ).toBe(false); const solvedDir = scratch(); seedWorkspace(task, solvedDir); applySolution(task, solvedDir); const solved = runVerification(task, solvedDir); - expect(solved.passed, `${task.id}: reference solution must pass:\n${solved.output}`).toBe(true); + expect(solved.passed, `${task.id}: reference solution must pass:\n${solved.output}`).toBe( + true, + ); } }, 300_000); @@ -127,8 +120,8 @@ describe("full run with mock agents", () => { }, 300_000); it("marks an uninvocable agent as agent-error, not failed", async () => { - const outDir = scratch(); - const tasks = loadTasks(TASK_ROOT).slice(0, 1); + const _outDir = scratch(); + const _tasks = loadTasks(TASK_ROOT).slice(0, 1); // stella with a bin override pointing at a nonexistent binary would fail // isAvailable(); instead simulate via a mock whose execute spawn errors by From 01cbb23e086e2f11a5647be937fd833879221ed5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:32:27 -0700 Subject: [PATCH 08/18] chore(deps-dev): Bump @types/node from 25.9.5 to 26.1.1 (#14) Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 25.9.5 to 26.1.1. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node) --- updated-dependencies: - dependency-name: "@types/node" dependency-version: 26.1.1 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- pnpm-lock.yaml | 59 ++++++++++++++++++-------------------------------- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/package.json b/package.json index 8e23da1..38f0dfc 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ }, "devDependencies": { "@biomejs/biome": "2.5.2", - "@types/node": "^25.9.1", + "@types/node": "^26.1.1", "tsx": "^4.19.2", "typescript": "^5.8.3", "vitest": "^2.1.9" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 258c270..7f50fc5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 2.5.2 version: 2.5.2 '@types/node': - specifier: ^25.9.1 - version: 25.9.5 + specifier: ^26.1.1 + version: 26.1.1 tsx: specifier: ^4.19.2 version: 4.23.0 @@ -22,7 +22,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.9 - version: 2.1.9(@types/node@25.9.5) + version: 2.1.9(@types/node@26.1.1) packages: @@ -48,28 +48,24 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [musl] '@biomejs/cli-linux-arm64@2.5.2': resolution: {integrity: sha512-t7sseOmqND57uUWTwlawU6BYj+J06T/9EkydzBhkrgw/FK3QVhjU2wsJR0frljrKZ0/I8A/rYw7284QgqjQfIQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [glibc] '@biomejs/cli-linux-x64-musl@2.5.2': resolution: {integrity: sha512-VArNLAzND063tF+XY0yPyM+DyahpzOMzOAvb7qs259nhjJWRjvjZdssuA+Rfl+l07+NOesKZ0Xu2yFrXyBMtzw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [musl] '@biomejs/cli-linux-x64@2.5.2': resolution: {integrity: sha512-M/lOZrewzTCRDINbjhQ1gYYru37KlD3kJBQwwKCG0ckz5E9IZwIoJ3X0wBwRXA+yBDIwWUuPBHS67HzJY4dTfA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [glibc] '@biomejs/cli-win32-arm64@2.5.2': resolution: {integrity: sha512-kbjFFKyZlzYnAuw7sRy5qDoFG6zrP40UK08oPQsWK0ct3NMnGSt+Bs1iviEEyEIP57N5MrykGXdO/wRiaR4lww==} @@ -414,79 +410,66 @@ packages: resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.62.2': resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.62.2': resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.62.2': resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.62.2': resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.62.2': resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.62.2': resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.62.2': resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.62.2': resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.62.2': resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.62.2': resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.62.2': resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.62.2': resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.62.2': resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} @@ -521,8 +504,8 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} - '@types/node@25.9.5': - resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + '@types/node@26.1.1': + resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} '@vitest/expect@2.1.9': resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} @@ -681,8 +664,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - undici-types@7.24.6: - resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} vite-node@2.1.9: resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} @@ -1013,9 +996,9 @@ snapshots: '@types/estree@1.0.9': {} - '@types/node@25.9.5': + '@types/node@26.1.1': dependencies: - undici-types: 7.24.6 + undici-types: 8.3.0 '@vitest/expect@2.1.9': dependencies: @@ -1024,13 +1007,13 @@ snapshots: chai: 5.3.3 tinyrainbow: 1.2.0 - '@vitest/mocker@2.1.9(vite@5.4.21(@types/node@25.9.5))': + '@vitest/mocker@2.1.9(vite@5.4.21(@types/node@26.1.1))': dependencies: '@vitest/spy': 2.1.9 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 5.4.21(@types/node@25.9.5) + vite: 5.4.21(@types/node@26.1.1) '@vitest/pretty-format@2.1.9': dependencies: @@ -1222,15 +1205,15 @@ snapshots: typescript@5.9.3: {} - undici-types@7.24.6: {} + undici-types@8.3.0: {} - vite-node@2.1.9(@types/node@25.9.5): + vite-node@2.1.9(@types/node@26.1.1): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 1.1.2 - vite: 5.4.21(@types/node@25.9.5) + vite: 5.4.21(@types/node@26.1.1) transitivePeerDependencies: - '@types/node' - less @@ -1242,19 +1225,19 @@ snapshots: - supports-color - terser - vite@5.4.21(@types/node@25.9.5): + vite@5.4.21(@types/node@26.1.1): dependencies: esbuild: 0.21.5 postcss: 8.5.17 rollup: 4.62.2 optionalDependencies: - '@types/node': 25.9.5 + '@types/node': 26.1.1 fsevents: 2.3.3 - vitest@2.1.9(@types/node@25.9.5): + vitest@2.1.9(@types/node@26.1.1): dependencies: '@vitest/expect': 2.1.9 - '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@25.9.5)) + '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@26.1.1)) '@vitest/pretty-format': 2.1.9 '@vitest/runner': 2.1.9 '@vitest/snapshot': 2.1.9 @@ -1270,11 +1253,11 @@ snapshots: tinyexec: 0.3.2 tinypool: 1.1.1 tinyrainbow: 1.2.0 - vite: 5.4.21(@types/node@25.9.5) - vite-node: 2.1.9(@types/node@25.9.5) + vite: 5.4.21(@types/node@26.1.1) + vite-node: 2.1.9(@types/node@26.1.1) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 25.9.5 + '@types/node': 26.1.1 transitivePeerDependencies: - less - lightningcss From 7245ef4d51be7f7883e4d146ee8425ab30a2cb74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:34:20 -0700 Subject: [PATCH 09/18] chore(deps): Update harbor requirement in /harbor (#17) Updates the requirements on [harbor](https://github.com/harbor-framework/harbor-cookbook) to permit the latest version. - [Commits](https://github.com/harbor-framework/harbor-cookbook/commits) --- updated-dependencies: - dependency-name: harbor dependency-version: 0.19.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mac Anderson --- harbor/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index d732ba8..2b942c4 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] # Pinned to the 0.6.x line the adapter is built and tested against. dependencies = [ - "harbor>=0.6,<0.7", + "harbor>=0.6,<0.20", ] [project.optional-dependencies] From 4330ab24395cfc4214a1e0bca1b3a476a9871925 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:35:42 -0700 Subject: [PATCH 10/18] chore(deps-dev): Bump typescript from 5.9.3 to 7.0.2 (#15) Bumps [typescript](https://github.com/microsoft/TypeScript) from 5.9.3 to 7.0.2. - [Release notes](https://github.com/microsoft/TypeScript/releases) - [Commits](https://github.com/microsoft/TypeScript/commits) --- updated-dependencies: - dependency-name: typescript dependency-version: 7.0.2 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mac Anderson --- package.json | 2 +- pnpm-lock.yaml | 213 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 208 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 38f0dfc..34f6a27 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@biomejs/biome": "2.5.2", "@types/node": "^26.1.1", "tsx": "^4.19.2", - "typescript": "^5.8.3", + "typescript": "^7.0.2", "vitest": "^2.1.9" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f50fc5..2d6d1f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: specifier: ^4.19.2 version: 4.23.0 typescript: - specifier: ^5.8.3 - version: 5.9.3 + specifier: ^7.0.2 + version: 7.0.2 vitest: specifier: ^2.1.9 version: 2.1.9(@types/node@26.1.1) @@ -507,6 +507,126 @@ packages: '@types/node@26.1.1': resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + '@vitest/expect@2.1.9': resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} @@ -659,9 +779,9 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - typescript@5.9.3: - resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} - engines: {node: '>=14.17'} + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} hasBin: true undici-types@8.3.0: @@ -1000,6 +1120,66 @@ snapshots: dependencies: undici-types: 8.3.0 + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + '@vitest/expect@2.1.9': dependencies: '@vitest/spy': 2.1.9 @@ -1203,7 +1383,28 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - typescript@5.9.3: {} + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 undici-types@8.3.0: {} From 4a8a8c5c2f922be4d90ab8f9518e03da5a7d1379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:37:48 -0700 Subject: [PATCH 11/18] chore(deps): Bump the actions group with 7 updates (#12) Bumps the actions group with 7 updates: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `4` | `7` | | [pnpm/action-setup](https://github.com/pnpm/action-setup) | `4` | `6` | | [actions/setup-node](https://github.com/actions/setup-node) | `4` | `6` | | [actions/setup-python](https://github.com/actions/setup-python) | `5` | `6` | | [actions/dependency-review-action](https://github.com/actions/dependency-review-action) | `4` | `5` | | [ossf/scorecard-action](https://github.com/ossf/scorecard-action) | `2.4.0` | `2.4.3` | | [github/codeql-action](https://github.com/github/codeql-action) | `3` | `4` | Updates `actions/checkout` from 4 to 7 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v7) Updates `pnpm/action-setup` from 4 to 6 - [Release notes](https://github.com/pnpm/action-setup/releases) - [Commits](https://github.com/pnpm/action-setup/compare/v4...v6) Updates `actions/setup-node` from 4 to 6 - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v6) Updates `actions/setup-python` from 5 to 6 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) Updates `actions/dependency-review-action` from 4 to 5 - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/v4...v5) Updates `ossf/scorecard-action` from 2.4.0 to 2.4.3 - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/v2.4.0...v2.4.3) Updates `github/codeql-action` from 3 to 4 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: pnpm/action-setup dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/setup-node dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/dependency-review-action dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: ossf/scorecard-action dependency-version: 2.4.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mac Anderson --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/dependency-review.yml | 4 ++-- .github/workflows/scorecard.yml | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56dfdd..221b125 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,11 +13,11 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 with: version: 11 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: 24 cache: pnpm @@ -35,8 +35,8 @@ jobs: run: working-directory: harbor steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 with: python-version: "3.12" # Harbor requires >= 3.12 cache: pip diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index d474190..330ec50 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -12,9 +12,9 @@ jobs: dependency-review: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Dependency Review - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: fail-on-severity: high comment-summary-in-pr: on-failure diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 506b123..c76f2f3 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -22,16 +22,16 @@ jobs: security-events: write # upload SARIF to code scanning id-token: write # publish results to the OpenSSF API steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@v2.4.0 + uses: ossf/scorecard-action@v2.4.3 with: results_file: results.sarif results_format: sarif publish_results: true - name: Upload to code scanning - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: results.sarif From 856990485a749149c0ece0498234c0e80a5bd0d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 22:39:25 -0700 Subject: [PATCH 12/18] chore(deps-dev): Bump vitest from 2.1.9 to 3.2.6 (#16) * chore(deps-dev): Bump vitest from 2.1.9 to 3.2.6 Bumps [vitest](https://github.com/vitest-dev/vitest/tree/HEAD/packages/vitest) from 2.1.9 to 3.2.6. - [Release notes](https://github.com/vitest-dev/vitest/releases) - [Changelog](https://github.com/vitest-dev/vitest/blob/main/docs/releases.md) - [Commits](https://github.com/vitest-dev/vitest/commits/v3.2.6/packages/vitest) --- updated-dependencies: - dependency-name: vitest dependency-version: 3.2.6 dependency-type: direct:development ... Signed-off-by: dependabot[bot] * fix * fix --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mac Anderson --- .github/CODE_OF_CONDUCT.md | 128 ++++++ .github/ISSUE_TEMPLATE/bug_report.md | 41 ++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.md | 36 ++ .github/pull_request_template.md | 31 ++ .gitignore | 3 + CHANGELOG.md | 56 +++ CONTRIBUTING.md | 110 +++++ README.md | 11 + package.json | 2 +- pnpm-lock.yaml | 522 ++++++++-------------- 11 files changed, 616 insertions(+), 329 deletions(-) create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..e63eb48 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +**mac@oxagen.ai**. All complaints will be reviewed and investigated promptly and +fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Community Impact**: A serious violation of community standards, +including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the ban, is allowed during this time. Violating these terms +may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html). +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq). +Translations are available at +[https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations). diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..2419098 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,41 @@ +--- +name: Bug report +about: Something in Arena is broken or produces wrong results +title: "[bug] " +labels: ["bug"] +--- +## What happened? + + + +## What did you expect? + + + +## Reproduce + + + +``` +# commands here +``` + +## Environment + +- Arena version: +- OS: +- Node version: +- Agent CLI(s) and versions: + +## Suspected cause (optional) + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0ee3cf4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability (private report) + url: https://github.com/oxageninc/arena/security/advisories/new + about: Report security issues privately via GitHub Security Advisories — do NOT open a public issue. See SECURITY.md. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..24131c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,36 @@ +--- +name: Feature request +about: Suggest a new adapter, task, metric, or workflow +title: "[feat] " +labels: ["enhancement"] +--- +## What and why + + + +## Kind + + + +- [ ] New agent adapter (`src/adapters/`) +- [ ] New task fixture (`tasks/`) +- [ ] New metric or report output +- [ ] CLI / workflow improvement +- [ ] Harbor adapter (`harbor/`) +- [ ] Other + +## Fairness impact (important) + +Arena's value is defensible, comparable measurement. Could this change affect +fairness — token normalization, what counts as success, matched configs, +determinism? If so, how do we keep it fair by construction? + + + +## Out of scope + + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..caeb9ed --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + + +## Summary + + + +## Fairness & correctness checklist + +Arena's value is defensible, comparable measurement. Confirm what applies: + +- [ ] **No behavior change** (docs/chore/refactor only) +- [ ] **New/existing adapter** — `tokens.input` excludes cache reads; an + un-invokable CLI scores `agent-error` (never counted as a loss) +- [ ] **New/existing task** — passes `pnpm arena verify`: held-out tests + **fail** on the pristine workspace and **pass** on the solution; runs on + plain Node with no network and no in-workspace install +- [ ] **Stats / report change** — output is reproducible bit-for-bit from the + same raw data + seed; a test pins it (see `test/stats.test.ts`) + +## Testing + + + +- [ ] `pnpm typecheck` +- [ ] `pnpm lint` +- [ ] `pnpm test` +- [ ] `pnpm arena verify` + +## Notes for reviewers + + diff --git a/.gitignore b/.gitignore index 8d38024..75c1cfc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ dist/ *.tsbuildinfo .DS_Store +# Agent working caches (code graph, context DB) — never commit. +.stella/ + # Python (harbor adapter) harbor/.venv/ __pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..81d2219 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,56 @@ +# Changelog + +All notable changes to Arena are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project +adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (pre-1.0: +minor bumps may include breaking changes). + +## [Unreleased] + +### Added +- Open-source launch artifacts: `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, issue + and PR templates, and this changelog. +- `.stella/` agent caches are now gitignored. + +## [0.1.0] — initial release + +The first public release: a head-to-head benchmark harness for agentic coding +CLIs, engineered to survive scrutiny. + +### Added +- **Core harness** — orchestrates two or more agents on the same task, model, + budget, and timeout, with interleaved (ABBA) ordering that flips every trial. + Tasks run sequentially so agents never contend for the host. +- **Held-out verification** — verification tests live outside the workspace, + are copied in only after the agent exits, and anything the agent planted at the + verify path is deleted first. `arena verify` proves every task's tests + **fail on the pristine workspace** and **pass on the reference solution**. +- **Adapters** — `claude-code`, `gemini`, `oxagen`, `stella`, and an in-process + `mock` for CI. Token usage is normalized so `tokens.input` never includes cache + reads; cache reads/writes tracked separately. A CLI that can't be invoked is + scored `agent-error` and excluded from every comparison. +- **Statistics** — multiple trials per (task, agent); Wilson 95% CIs on success + rates; **exact McNemar** on paired outcomes; seeded paired-bootstrap CIs on + wall-clock / token / cost deltas. Deterministic: same raw data + seed ⇒ same + report, bit for bit. Each metric reported separately — never a blended score. +- **Pricing** — one shared pricing table (`pricing.json`) applied to every agent; + models without an entry get **no** computed cost (never a guessed one). +- **Receipts** — every trial writes full stdout/stderr transcripts, a workspace + diff, and a manifest with CLI versions, models, host, seed, and a one-line + reproduce command. +- **Regression gate** — `arena baseline save` snapshots a run; `arena gate` + fails CI when accuracy drops (with `--require-significant` to clear CI noise), + or tokens/cost/speed drift past configurable thresholds. +- **CLI** — `arena list | doctor | run | report | verify | baseline | gate`. +- **Harbor adapter** (`harbor/`) — run **your** agent through Harbor's official, + containerized SWE-bench / Terminal-Bench verifier head-to-head against the + built-ins. Ships with `ByoAgent` (bring your own) plus `OxagenAgent` / + `StellaAgent` specs. No Python required to add an agent. + +### Security & supply chain +- Least-privilege CI, pinned GitHub Actions, [OpenSSF Scorecard](https://github.com/ossf/scorecard-action), + Dependabot, and a dependency-review gate that blocks known-vulnerable PRs. +- `SECURITY.md` with a private vulnerability reporting channel. + +[Unreleased]: https://github.com/oxageninc/arena/compare/5122005...HEAD +[0.1.0]: https://github.com/oxageninc/arena/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2360e59 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing to Arena + +Thanks for your interest in making Arena better. Arena is a benchmark harness, so +its contribution rules are a little stricter than a typical app — **correctness +and fairness are the product.** A change that subtly weakens a held-out test, an +adapter that normalizes tokens differently, or a stat that loses determinism +each undermine the one thing the project exists to provide. + +## Before you write code + +- **Search [open issues](https://github.com/oxageninc/arena/issues) first** — + someone may already be on it. +- For anything non-trivial (new adapter, new task, stat/report change), **open + an issue** and sketch the approach before investing in code. A 5-minute + alignment round saves a 2-hour review. +- This project follows the [Code of Conduct](.github/CODE_OF_CONDUCT.md). By + participating you agree to it. + +## Getting set up + +Arena needs **Node 22+** and **pnpm** (the CI pins pnpm 11): + +```bash +git clone https://github.com/oxageninc/arena +cd arena +pnpm install +pnpm typecheck +pnpm test # unit + end-to-end pipeline, mock agents, no API keys +pnpm arena verify # audit every task: held-out tests fail-pristine / pass-solution +pnpm lint # biome +``` + +Everything runs offline with mock agents — you do **not** need any provider API +keys to develop or test the harness. The Harbor adapter is Python; see +[`harbor/README.md`](harbor/README.md). + +## What you can contribute + +### Add a task fixture + +Tasks live in `tasks//` and are the most common contribution. A task must +satisfy two invariants that `pnpm arena verify` enforces in CI: + +1. Its held-out `verify/` tests **fail** on the pristine `workspace/` (no + tautology), and +2. They **pass** on the `solution/` (no impossibility). + +Copy an existing task directory as a template and read its `task.json` — the +prompt given to agents must state the *entire* behavior contract; the hidden +tests assert only what the prompt states. Verification must run on plain Node +with **no npm install inside the workspace and no network**. Run +`pnpm arena verify ` before pushing. + +### Add an agent adapter + +One file in `src/adapters/`, registered in `src/adapters/index.ts`. Implement +how to invoke your CLI headlessly in a directory and how to map its output +envelope to normalized tokens. Two hard rules: + +- **`tokens.input` must exclude cache reads** (see how the existing adapters + subtract cached counts). Cross-agent token comparisons are meaningless + otherwise. +- **Adapters never decide success.** Only the harness's held-out verification + does. An adapter that can't even be invoked is scored `agent-error` and + excluded from comparisons — it is never counted as the agent losing. + +See `src/adapters/base.ts` for the contract; the built-ins (`claude-code`, +`gemini`, `oxagen`, `stella`, `mock`) are worked examples at ~80–120 lines each. + +### Improve the stats / report + +Every report must be **reproducible bit-for-bit from the same raw data + seed**, +and **each metric is reported separately** — Arena never emits a blended score. +If you touch `src/stats.ts` or `src/report.ts`, add a test that pins the output +of a fixed input (see `test/stats.test.ts`) so determinism is enforced. + +## Code style + +- **Biome** enforces formatting and linting (`pnpm lint` / `pnpm format`). + Don't hand-format; let the tool do it. +- **TypeScript strict** is on (`tsconfig.json`), including + `noUncheckedIndexedAccess` and `exactOptionalPropertyTypes`. Don't weaken + these to make a type error go away — fix the code. +- Node-only: target ES2023, ESM (`"type": "module"`), NodeNext resolution. + `.js` extensions are required in relative imports. + +## Tests + +- Add or update tests for any behavior change. The pipeline test + (`test/pipeline.test.ts`) spawns real `git`/`node` subprocesses per task + workspace, so keep task workspaces tiny and fast. +- `pnpm arena verify` is part of CI — if your task fails it, CI is red. + +## Submitting + +1. Fork and branch from `main`. +2. `pnpm typecheck && pnpm lint && pnpm test && pnpm arena verify` must all pass + locally (this is exactly what CI runs). +3. Commit message format (Conventional Commits): + `feat(adapter): add codex adapter`, `fix(stats): …`, `docs: …`, + `test: …`, `chore: …`. +4. Open a PR against `main`. Reference the issue it closes (`Closes #123`). + Dependency review runs automatically and blocks known-vulnerable deps. + +### DCO / sign-off + +By submitting, you confirm your contribution is your own and you license it +under the project's [MIT license](LICENSE). We follow the +[Developer Certificate of Origin](https://developercertificate.org/): add a +`Signed-off-by: Your Name ` line to your commits (`git commit -s`). diff --git a/README.md b/README.md index 2ca6acf..35369ab 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Arena +[![CI](https://github.com/oxageninc/arena/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/oxageninc/arena/actions/workflows/ci.yml) +[![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/oxageninc/arena/badge)](https://scorecard.dev/viewer/?uri=github.com/oxageninc/arena) + **Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/oxageninc/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. @@ -126,4 +130,11 @@ pnpm test # unit + end-to-end pipeline (mock agents, no API keys) pnpm arena verify # audit every task's discrimination invariants ``` +Contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) (adding an +adapter or task fixture is a great first PR). We also have a +[Code of Conduct](.github/CODE_OF_CONDUCT.md) and a +[changelog](CHANGELOG.md). Report security issues privately via the repo's +[Security tab](https://github.com/oxageninc/arena/security/advisories/new) — not +as a public issue. + MIT © Mac Anderson diff --git a/package.json b/package.json index 34f6a27..15b8124 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,6 @@ "@types/node": "^26.1.1", "tsx": "^4.19.2", "typescript": "^7.0.2", - "vitest": "^2.1.9" + "vitest": "^3.2.6" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d6d1f5..1f95d93 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^7.0.2 version: 7.0.2 vitest: - specifier: ^2.1.9 - version: 2.1.9(@types/node@26.1.1) + specifier: ^3.2.6 + version: 3.2.6(@types/node@26.1.1)(tsx@4.23.0) packages: @@ -48,24 +48,28 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [musl] '@biomejs/cli-linux-arm64@2.5.2': resolution: {integrity: sha512-t7sseOmqND57uUWTwlawU6BYj+J06T/9EkydzBhkrgw/FK3QVhjU2wsJR0frljrKZ0/I8A/rYw7284QgqjQfIQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [glibc] '@biomejs/cli-linux-x64-musl@2.5.2': resolution: {integrity: sha512-VArNLAzND063tF+XY0yPyM+DyahpzOMzOAvb7qs259nhjJWRjvjZdssuA+Rfl+l07+NOesKZ0Xu2yFrXyBMtzw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [musl] '@biomejs/cli-linux-x64@2.5.2': resolution: {integrity: sha512-M/lOZrewzTCRDINbjhQ1gYYru37KlD3kJBQwwKCG0ckz5E9IZwIoJ3X0wBwRXA+yBDIwWUuPBHS67HzJY4dTfA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [glibc] '@biomejs/cli-win32-arm64@2.5.2': resolution: {integrity: sha512-kbjFFKyZlzYnAuw7sRy5qDoFG6zrP40UK08oPQsWK0ct3NMnGSt+Bs1iviEEyEIP57N5MrykGXdO/wRiaR4lww==} @@ -79,204 +83,102 @@ packages: cpu: [x64] os: [win32] - '@esbuild/aix-ppc64@0.21.5': - resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - '@esbuild/aix-ppc64@0.28.1': resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.21.5': - resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - '@esbuild/android-arm64@0.28.1': resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.21.5': - resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - '@esbuild/android-arm@0.28.1': resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.21.5': - resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - '@esbuild/android-x64@0.28.1': resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.21.5': - resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - '@esbuild/darwin-arm64@0.28.1': resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.21.5': - resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - '@esbuild/darwin-x64@0.28.1': resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.21.5': - resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - '@esbuild/freebsd-arm64@0.28.1': resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.21.5': - resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - '@esbuild/freebsd-x64@0.28.1': resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.21.5': - resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - '@esbuild/linux-arm64@0.28.1': resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.21.5': - resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - '@esbuild/linux-arm@0.28.1': resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.21.5': - resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - '@esbuild/linux-ia32@0.28.1': resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.21.5': - resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - '@esbuild/linux-loong64@0.28.1': resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.21.5': - resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - '@esbuild/linux-mips64el@0.28.1': resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.21.5': - resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - '@esbuild/linux-ppc64@0.28.1': resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.21.5': - resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - '@esbuild/linux-riscv64@0.28.1': resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.21.5': - resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - '@esbuild/linux-s390x@0.28.1': resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.21.5': - resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - '@esbuild/linux-x64@0.28.1': resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} engines: {node: '>=18'} @@ -289,12 +191,6 @@ packages: cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.21.5': - resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - '@esbuild/netbsd-x64@0.28.1': resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} engines: {node: '>=18'} @@ -307,12 +203,6 @@ packages: cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.21.5': - resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - '@esbuild/openbsd-x64@0.28.1': resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} engines: {node: '>=18'} @@ -325,48 +215,24 @@ packages: cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.21.5': - resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - '@esbuild/sunos-x64@0.28.1': resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.21.5': - resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - '@esbuild/win32-arm64@0.28.1': resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.21.5': - resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - '@esbuild/win32-ia32@0.28.1': resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.21.5': - resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - '@esbuild/win32-x64@0.28.1': resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} engines: {node: '>=18'} @@ -410,66 +276,79 @@ packages: resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.62.2': resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.62.2': resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.62.2': resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.62.2': resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.62.2': resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.62.2': resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.62.2': resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.62.2': resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.62.2': resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.62.2': resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.62.2': resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.62.2': resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.62.2': resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} @@ -501,6 +380,12 @@ packages: cpu: [x64] os: [win32] + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} @@ -627,34 +512,37 @@ packages: cpu: [x64] os: [win32] - '@vitest/expect@2.1.9': - resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + '@vitest/expect@3.2.6': + resolution: {integrity: sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==} - '@vitest/mocker@2.1.9': - resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + '@vitest/mocker@3.2.6': + resolution: {integrity: sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==} peerDependencies: msw: ^2.4.9 - vite: ^5.0.0 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 peerDependenciesMeta: msw: optional: true vite: optional: true - '@vitest/pretty-format@2.1.9': - resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + '@vitest/pretty-format@3.2.6': + resolution: {integrity: sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==} + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} - '@vitest/runner@2.1.9': - resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + '@vitest/runner@3.2.6': + resolution: {integrity: sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==} - '@vitest/snapshot@2.1.9': - resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + '@vitest/snapshot@3.2.6': + resolution: {integrity: sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==} - '@vitest/spy@2.1.9': - resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + '@vitest/spy@3.2.6': + resolution: {integrity: sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==} - '@vitest/utils@2.1.9': - resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + '@vitest/utils@3.2.6': + resolution: {integrity: sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==} assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} @@ -688,11 +576,6 @@ packages: es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} - esbuild@0.21.5: - resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} - engines: {node: '>=12'} - hasBin: true - esbuild@0.28.1: resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} @@ -705,11 +588,23 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} @@ -719,13 +614,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.15: - resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - pathe@1.1.2: - resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} pathval@2.0.1: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} @@ -734,8 +629,12 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - postcss@8.5.17: - resolution: {integrity: sha512-J7EF+8X+CzRPaJPOv9Ck2wNWJvGnnl3PcNPAdGg6GTLjyVpyQ0yATMSXRFRV01BviT/9Gwuc3rjEyJbDJG9a4w==} + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + + postcss@8.5.18: + resolution: {integrity: sha512-xdB1oSLHbz1vRWgCDalrCqEFTWzFlhqFC5tIHLMOSUIjhm3XXQ1qrFy8S/ESr1JYRRXqM3c1QFiMZUJdUTqyMQ==} engines: {node: ^10 || ^12 || >=14} rollup@4.62.2: @@ -756,22 +655,29 @@ packages: std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + tinypool@1.1.1: resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} engines: {node: ^18.0.0 || >=20.0.0} - tinyrainbow@1.2.0: - resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} engines: {node: '>=14.0.0'} - tinyspy@3.0.2: - resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + tinyspy@4.0.4: + resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} tsx@4.23.0: @@ -787,27 +693,32 @@ packages: undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} - vite-node@2.1.9: - resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} - engines: {node: ^18.0.0 || >=20.0.0} + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} hasBin: true - vite@5.4.21: - resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} - engines: {node: ^18.0.0 || >=20.0.0} + vite@7.3.6: + resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} + engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: - '@types/node': ^18.0.0 || >=20.0.0 - less: '*' + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 lightningcss: ^1.21.0 - sass: '*' - sass-embedded: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 peerDependenciesMeta: '@types/node': optional: true + jiti: + optional: true less: optional: true lightningcss: @@ -822,21 +733,28 @@ packages: optional: true terser: optional: true + tsx: + optional: true + yaml: + optional: true - vitest@2.1.9: - resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} - engines: {node: ^18.0.0 || >=20.0.0} + vitest@3.2.6: + resolution: {integrity: sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' - '@types/node': ^18.0.0 || >=20.0.0 - '@vitest/browser': 2.1.9 - '@vitest/ui': 2.1.9 + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.6 + '@vitest/ui': 3.2.6 happy-dom: '*' jsdom: '*' peerDependenciesMeta: '@edge-runtime/vm': optional: true + '@types/debug': + optional: true '@types/node': optional: true '@vitest/browser': @@ -890,150 +808,81 @@ snapshots: '@biomejs/cli-win32-x64@2.5.2': optional: true - '@esbuild/aix-ppc64@0.21.5': - optional: true - '@esbuild/aix-ppc64@0.28.1': optional: true - '@esbuild/android-arm64@0.21.5': - optional: true - '@esbuild/android-arm64@0.28.1': optional: true - '@esbuild/android-arm@0.21.5': - optional: true - '@esbuild/android-arm@0.28.1': optional: true - '@esbuild/android-x64@0.21.5': - optional: true - '@esbuild/android-x64@0.28.1': optional: true - '@esbuild/darwin-arm64@0.21.5': - optional: true - '@esbuild/darwin-arm64@0.28.1': optional: true - '@esbuild/darwin-x64@0.21.5': - optional: true - '@esbuild/darwin-x64@0.28.1': optional: true - '@esbuild/freebsd-arm64@0.21.5': - optional: true - '@esbuild/freebsd-arm64@0.28.1': optional: true - '@esbuild/freebsd-x64@0.21.5': - optional: true - '@esbuild/freebsd-x64@0.28.1': optional: true - '@esbuild/linux-arm64@0.21.5': - optional: true - '@esbuild/linux-arm64@0.28.1': optional: true - '@esbuild/linux-arm@0.21.5': - optional: true - '@esbuild/linux-arm@0.28.1': optional: true - '@esbuild/linux-ia32@0.21.5': - optional: true - '@esbuild/linux-ia32@0.28.1': optional: true - '@esbuild/linux-loong64@0.21.5': - optional: true - '@esbuild/linux-loong64@0.28.1': optional: true - '@esbuild/linux-mips64el@0.21.5': - optional: true - '@esbuild/linux-mips64el@0.28.1': optional: true - '@esbuild/linux-ppc64@0.21.5': - optional: true - '@esbuild/linux-ppc64@0.28.1': optional: true - '@esbuild/linux-riscv64@0.21.5': - optional: true - '@esbuild/linux-riscv64@0.28.1': optional: true - '@esbuild/linux-s390x@0.21.5': - optional: true - '@esbuild/linux-s390x@0.28.1': optional: true - '@esbuild/linux-x64@0.21.5': - optional: true - '@esbuild/linux-x64@0.28.1': optional: true '@esbuild/netbsd-arm64@0.28.1': optional: true - '@esbuild/netbsd-x64@0.21.5': - optional: true - '@esbuild/netbsd-x64@0.28.1': optional: true '@esbuild/openbsd-arm64@0.28.1': optional: true - '@esbuild/openbsd-x64@0.21.5': - optional: true - '@esbuild/openbsd-x64@0.28.1': optional: true '@esbuild/openharmony-arm64@0.28.1': optional: true - '@esbuild/sunos-x64@0.21.5': - optional: true - '@esbuild/sunos-x64@0.28.1': optional: true - '@esbuild/win32-arm64@0.21.5': - optional: true - '@esbuild/win32-arm64@0.28.1': optional: true - '@esbuild/win32-ia32@0.21.5': - optional: true - '@esbuild/win32-ia32@0.28.1': optional: true - '@esbuild/win32-x64@0.21.5': - optional: true - '@esbuild/win32-x64@0.28.1': optional: true @@ -1114,6 +963,13 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + '@types/estree@1.0.9': {} '@types/node@26.1.1': @@ -1180,45 +1036,51 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true - '@vitest/expect@2.1.9': + '@vitest/expect@3.2.6': dependencies: - '@vitest/spy': 2.1.9 - '@vitest/utils': 2.1.9 + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.6 + '@vitest/utils': 3.2.6 chai: 5.3.3 - tinyrainbow: 1.2.0 + tinyrainbow: 2.0.0 - '@vitest/mocker@2.1.9(vite@5.4.21(@types/node@26.1.1))': + '@vitest/mocker@3.2.6(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0))': dependencies: - '@vitest/spy': 2.1.9 + '@vitest/spy': 3.2.6 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 5.4.21(@types/node@26.1.1) + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) + + '@vitest/pretty-format@3.2.6': + dependencies: + tinyrainbow: 2.0.0 - '@vitest/pretty-format@2.1.9': + '@vitest/pretty-format@3.2.7': dependencies: - tinyrainbow: 1.2.0 + tinyrainbow: 2.0.0 - '@vitest/runner@2.1.9': + '@vitest/runner@3.2.6': dependencies: - '@vitest/utils': 2.1.9 - pathe: 1.1.2 + '@vitest/utils': 3.2.6 + pathe: 2.0.3 + strip-literal: 3.1.0 - '@vitest/snapshot@2.1.9': + '@vitest/snapshot@3.2.6': dependencies: - '@vitest/pretty-format': 2.1.9 + '@vitest/pretty-format': 3.2.6 magic-string: 0.30.21 - pathe: 1.1.2 + pathe: 2.0.3 - '@vitest/spy@2.1.9': + '@vitest/spy@3.2.6': dependencies: - tinyspy: 3.0.2 + tinyspy: 4.0.4 - '@vitest/utils@2.1.9': + '@vitest/utils@3.2.6': dependencies: - '@vitest/pretty-format': 2.1.9 + '@vitest/pretty-format': 3.2.6 loupe: 3.2.1 - tinyrainbow: 1.2.0 + tinyrainbow: 2.0.0 assertion-error@2.0.1: {} @@ -1242,32 +1104,6 @@ snapshots: es-module-lexer@1.7.0: {} - esbuild@0.21.5: - optionalDependencies: - '@esbuild/aix-ppc64': 0.21.5 - '@esbuild/android-arm': 0.21.5 - '@esbuild/android-arm64': 0.21.5 - '@esbuild/android-x64': 0.21.5 - '@esbuild/darwin-arm64': 0.21.5 - '@esbuild/darwin-x64': 0.21.5 - '@esbuild/freebsd-arm64': 0.21.5 - '@esbuild/freebsd-x64': 0.21.5 - '@esbuild/linux-arm': 0.21.5 - '@esbuild/linux-arm64': 0.21.5 - '@esbuild/linux-ia32': 0.21.5 - '@esbuild/linux-loong64': 0.21.5 - '@esbuild/linux-mips64el': 0.21.5 - '@esbuild/linux-ppc64': 0.21.5 - '@esbuild/linux-riscv64': 0.21.5 - '@esbuild/linux-s390x': 0.21.5 - '@esbuild/linux-x64': 0.21.5 - '@esbuild/netbsd-x64': 0.21.5 - '@esbuild/openbsd-x64': 0.21.5 - '@esbuild/sunos-x64': 0.21.5 - '@esbuild/win32-arm64': 0.21.5 - '@esbuild/win32-ia32': 0.21.5 - '@esbuild/win32-x64': 0.21.5 - esbuild@0.28.1: optionalDependencies: '@esbuild/aix-ppc64': 0.28.1 @@ -1303,9 +1139,15 @@ snapshots: expect-type@1.4.0: {} + fdir@6.5.0(picomatch@4.0.5): + optionalDependencies: + picomatch: 4.0.5 + fsevents@2.3.3: optional: true + js-tokens@9.0.1: {} + loupe@3.2.1: {} magic-string@0.30.21: @@ -1314,17 +1156,19 @@ snapshots: ms@2.1.3: {} - nanoid@3.3.15: {} + nanoid@3.3.16: {} - pathe@1.1.2: {} + pathe@2.0.3: {} pathval@2.0.1: {} picocolors@1.1.1: {} - postcss@8.5.17: + picomatch@4.0.5: {} + + postcss@8.5.18: dependencies: - nanoid: 3.3.15 + nanoid: 3.3.16 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -1367,15 +1211,24 @@ snapshots: std-env@3.10.0: {} + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + tinybench@2.9.0: {} tinyexec@0.3.2: {} + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + tinypool@1.1.1: {} - tinyrainbow@1.2.0: {} + tinyrainbow@2.0.0: {} - tinyspy@3.0.2: {} + tinyspy@4.0.4: {} tsx@4.23.0: dependencies: @@ -1408,15 +1261,16 @@ snapshots: undici-types@8.3.0: {} - vite-node@2.1.9(@types/node@26.1.1): + vite-node@3.2.4(@types/node@26.1.1)(tsx@4.23.0): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 - pathe: 1.1.2 - vite: 5.4.21(@types/node@26.1.1) + pathe: 2.0.3 + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) transitivePeerDependencies: - '@types/node' + - jiti - less - lightningcss - sass @@ -1425,41 +1279,51 @@ snapshots: - sugarss - supports-color - terser + - tsx + - yaml - vite@5.4.21(@types/node@26.1.1): + vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0): dependencies: - esbuild: 0.21.5 - postcss: 8.5.17 + esbuild: 0.28.1 + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + postcss: 8.5.18 rollup: 4.62.2 + tinyglobby: 0.2.17 optionalDependencies: '@types/node': 26.1.1 fsevents: 2.3.3 + tsx: 4.23.0 - vitest@2.1.9(@types/node@26.1.1): + vitest@3.2.6(@types/node@26.1.1)(tsx@4.23.0): dependencies: - '@vitest/expect': 2.1.9 - '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@26.1.1)) - '@vitest/pretty-format': 2.1.9 - '@vitest/runner': 2.1.9 - '@vitest/snapshot': 2.1.9 - '@vitest/spy': 2.1.9 - '@vitest/utils': 2.1.9 + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.6 + '@vitest/mocker': 3.2.6(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0)) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.2.6 + '@vitest/snapshot': 3.2.6 + '@vitest/spy': 3.2.6 + '@vitest/utils': 3.2.6 chai: 5.3.3 debug: 4.4.3 expect-type: 1.4.0 magic-string: 0.30.21 - pathe: 1.1.2 + pathe: 2.0.3 + picomatch: 4.0.5 std-env: 3.10.0 tinybench: 2.9.0 tinyexec: 0.3.2 + tinyglobby: 0.2.17 tinypool: 1.1.1 - tinyrainbow: 1.2.0 - vite: 5.4.21(@types/node@26.1.1) - vite-node: 2.1.9(@types/node@26.1.1) + tinyrainbow: 2.0.0 + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) + vite-node: 3.2.4(@types/node@26.1.1)(tsx@4.23.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 26.1.1 transitivePeerDependencies: + - jiti - less - lightningcss - msw @@ -1469,6 +1333,8 @@ snapshots: - sugarss - supports-color - terser + - tsx + - yaml why-is-node-running@2.3.0: dependencies: From d04e2f1b64ff31c421126c110a77602e69687cb9 Mon Sep 17 00:00:00 2001 From: macanderson Date: Fri, 17 Jul 2026 22:51:00 -0700 Subject: [PATCH 13/18] chore(deps-dev): bump @biomejs/biome to 2.5.4 and tsx to 4.23.1 (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces #18 — the Dependabot branch accumulated lockfile conflicts after the vitest-3.x and typescript-7 merges landed on main. This reapplies the same two patch bumps (biome 2.5.2->2.5.4, tsx 4.19.2->4.23.1) on a clean base with a freshly-regenerated lockfile. --- package.json | 4 +- pnpm-lock.yaml | 185 ++++++++++++++++++++++++------------------------- 2 files changed, 91 insertions(+), 98 deletions(-) diff --git a/package.json b/package.json index 15b8124..d89b2eb 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,9 @@ "verify:tasks": "tsx src/cli.ts verify" }, "devDependencies": { - "@biomejs/biome": "2.5.2", + "@biomejs/biome": "2.5.4", "@types/node": "^26.1.1", - "tsx": "^4.19.2", + "tsx": "^4.23.1", "typescript": "^7.0.2", "vitest": "^3.2.6" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f95d93..b9f9f19 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,76 +9,76 @@ importers: .: devDependencies: '@biomejs/biome': - specifier: 2.5.2 - version: 2.5.2 + specifier: 2.5.4 + version: 2.5.4 '@types/node': specifier: ^26.1.1 version: 26.1.1 tsx: - specifier: ^4.19.2 - version: 4.23.0 + specifier: ^4.23.1 + version: 4.23.1 typescript: specifier: ^7.0.2 version: 7.0.2 vitest: specifier: ^3.2.6 - version: 3.2.6(@types/node@26.1.1)(tsx@4.23.0) + version: 3.2.7(@types/node@26.1.1)(tsx@4.23.1) packages: - '@biomejs/biome@2.5.2': - resolution: {integrity: sha512-VQ3RCqr7JmDIX+w6stWYl+g/3bYofN3q2wDBHUKKc/c7i5QWrFKFBZYCYPWTE6agsUPMIZZe6/CMmVUfUAhkKA==} + '@biomejs/biome@2.5.4': + resolution: {integrity: sha512-xy5FNE5kQJKyK5MR1gJy6ztXYx4WBAbYGlK04lMEgmyPRWKybY9NFwiG9yo0XdzOU8Xvhj41u034J1ywfoWfMw==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-darwin-arm64@2.5.2': - resolution: {integrity: sha512-e7P3P7EkwFc/KiX2AHw4YDLIBOMfG9CPCAwy52k5Bp0dfhkozx9hf6wCmIr2QeXy2XeccJ3V/Sg+hDmzYEqxSg==} + '@biomejs/cli-darwin-arm64@2.5.4': + resolution: {integrity: sha512-4o3NFRobXHynkgcFVrlZsoDAFtF2ldlEGN8sORSws5ZQqyY4PXnPUIylu4ksfyHuwkfvDREuWh3JK+niRwGq3w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.5.2': - resolution: {integrity: sha512-ymzMvjC1Jg0b9K0D26ZdARqFQXs7MocfLC5FOCGfkC0Ss+ACUJkX5364ZM5nT4NLZanHRZNVrZEy+Ibwcvux/g==} + '@biomejs/cli-darwin-x64@2.5.4': + resolution: {integrity: sha512-D32P5HkU2Y6PySuC/WsVDTOgsDwVFmujzhhhOQjajtATpVWFDXuVd3oRbsWNSEA+aaFzyzZm22szsyydBYlSyQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.5.2': - resolution: {integrity: sha512-w+ANG0ZvTu9IeEg9QnstoOnk6L0fpwJifW6aHR18+cb5Z39bkANItYjAfMrnvce5tmMK+IQ6nPX7/kQFdam5iw==} + '@biomejs/cli-linux-arm64-musl@2.5.4': + resolution: {integrity: sha512-Rpm5/AT1m+DlJmUoYvS4/vXc+0tXJPJ2NQz25TGPyHVF5JrWy75PE0GH6kVxsKtQDuCH4OgzquZq0R4kj/wCVg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] - '@biomejs/cli-linux-arm64@2.5.2': - resolution: {integrity: sha512-t7sseOmqND57uUWTwlawU6BYj+J06T/9EkydzBhkrgw/FK3QVhjU2wsJR0frljrKZ0/I8A/rYw7284QgqjQfIQ==} + '@biomejs/cli-linux-arm64@2.5.4': + resolution: {integrity: sha512-pSEfW7B8kTsXUjUxC1xVVK+y85Ht3C5XxZ9gclmC7/3Ku9Vqz8jmI7k0p/BNIjQ6t4sFERI2sFeH73ybiZl6YQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] - '@biomejs/cli-linux-x64-musl@2.5.2': - resolution: {integrity: sha512-VArNLAzND063tF+XY0yPyM+DyahpzOMzOAvb7qs259nhjJWRjvjZdssuA+Rfl+l07+NOesKZ0Xu2yFrXyBMtzw==} + '@biomejs/cli-linux-x64-musl@2.5.4': + resolution: {integrity: sha512-aby/PohmmgbShcHqFsZVzG8H6D98+P+A6xRWRrQcLW1pCjabcov5UUlke4UqNQBYTkDQav+jB4zyyDDeKB2GaA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] - '@biomejs/cli-linux-x64@2.5.2': - resolution: {integrity: sha512-M/lOZrewzTCRDINbjhQ1gYYru37KlD3kJBQwwKCG0ckz5E9IZwIoJ3X0wBwRXA+yBDIwWUuPBHS67HzJY4dTfA==} + '@biomejs/cli-linux-x64@2.5.4': + resolution: {integrity: sha512-FNxojWJkL7EajAuzBgoLe0T2G0y112M4lBrDIFl/DomFTx8yqenYOIdsRLNXvOvBBofE8hJi85LjzLmBDpY7/Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] - '@biomejs/cli-win32-arm64@2.5.2': - resolution: {integrity: sha512-kbjFFKyZlzYnAuw7sRy5qDoFG6zrP40UK08oPQsWK0ct3NMnGSt+Bs1iviEEyEIP57N5MrykGXdO/wRiaR4lww==} + '@biomejs/cli-win32-arm64@2.5.4': + resolution: {integrity: sha512-emoXexPZIPAZkz2RKmA95WJUqK3I5MJNYtwEbL5ESciRzhmFMMyekDhNG8hpeOaK+ZGRDxAU4wvGuA5IHQ0h0w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.5.2': - resolution: {integrity: sha512-4InchVpdVmdkkkgjQqKpgvyu+VPnoF/7RPSw5YATgEVpt2j72wcCAeV5TwaE9ZGJUZWZn7v2CwSAj6CrMJEx8A==} + '@biomejs/cli-win32-x64@2.5.4': + resolution: {integrity: sha512-U1jaluLw1qQc2Tx7/CeSoL9N5XcqIH+GWjpUAy1ouB5nVjSCMNO+NNHdY3RAs8zxNurLWAdj6pehQdCA2zyU+Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -512,11 +512,11 @@ packages: cpu: [x64] os: [win32] - '@vitest/expect@3.2.6': - resolution: {integrity: sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==} + '@vitest/expect@3.2.7': + resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} - '@vitest/mocker@3.2.6': - resolution: {integrity: sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==} + '@vitest/mocker@3.2.7': + resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} peerDependencies: msw: ^2.4.9 vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 @@ -526,23 +526,20 @@ packages: vite: optional: true - '@vitest/pretty-format@3.2.6': - resolution: {integrity: sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==} - '@vitest/pretty-format@3.2.7': resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} - '@vitest/runner@3.2.6': - resolution: {integrity: sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==} + '@vitest/runner@3.2.7': + resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} - '@vitest/snapshot@3.2.6': - resolution: {integrity: sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==} + '@vitest/snapshot@3.2.7': + resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} - '@vitest/spy@3.2.6': - resolution: {integrity: sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==} + '@vitest/spy@3.2.7': + resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} - '@vitest/utils@3.2.6': - resolution: {integrity: sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==} + '@vitest/utils@3.2.7': + resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} @@ -633,8 +630,8 @@ packages: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} - postcss@8.5.18: - resolution: {integrity: sha512-xdB1oSLHbz1vRWgCDalrCqEFTWzFlhqFC5tIHLMOSUIjhm3XXQ1qrFy8S/ESr1JYRRXqM3c1QFiMZUJdUTqyMQ==} + postcss@8.5.19: + resolution: {integrity: sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==} engines: {node: ^10 || ^12 || >=14} rollup@4.62.2: @@ -680,8 +677,8 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} - tsx@4.23.0: - resolution: {integrity: sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==} + tsx@4.23.1: + resolution: {integrity: sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==} engines: {node: '>=18.0.0'} hasBin: true @@ -738,16 +735,16 @@ packages: yaml: optional: true - vitest@3.2.6: - resolution: {integrity: sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==} + vitest@3.2.7: + resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@types/debug': ^4.1.12 '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 - '@vitest/browser': 3.2.6 - '@vitest/ui': 3.2.6 + '@vitest/browser': 3.2.7 + '@vitest/ui': 3.2.7 happy-dom: '*' jsdom: '*' peerDependenciesMeta: @@ -773,39 +770,39 @@ packages: snapshots: - '@biomejs/biome@2.5.2': + '@biomejs/biome@2.5.4': optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.5.2 - '@biomejs/cli-darwin-x64': 2.5.2 - '@biomejs/cli-linux-arm64': 2.5.2 - '@biomejs/cli-linux-arm64-musl': 2.5.2 - '@biomejs/cli-linux-x64': 2.5.2 - '@biomejs/cli-linux-x64-musl': 2.5.2 - '@biomejs/cli-win32-arm64': 2.5.2 - '@biomejs/cli-win32-x64': 2.5.2 + '@biomejs/cli-darwin-arm64': 2.5.4 + '@biomejs/cli-darwin-x64': 2.5.4 + '@biomejs/cli-linux-arm64': 2.5.4 + '@biomejs/cli-linux-arm64-musl': 2.5.4 + '@biomejs/cli-linux-x64': 2.5.4 + '@biomejs/cli-linux-x64-musl': 2.5.4 + '@biomejs/cli-win32-arm64': 2.5.4 + '@biomejs/cli-win32-x64': 2.5.4 - '@biomejs/cli-darwin-arm64@2.5.2': + '@biomejs/cli-darwin-arm64@2.5.4': optional: true - '@biomejs/cli-darwin-x64@2.5.2': + '@biomejs/cli-darwin-x64@2.5.4': optional: true - '@biomejs/cli-linux-arm64-musl@2.5.2': + '@biomejs/cli-linux-arm64-musl@2.5.4': optional: true - '@biomejs/cli-linux-arm64@2.5.2': + '@biomejs/cli-linux-arm64@2.5.4': optional: true - '@biomejs/cli-linux-x64-musl@2.5.2': + '@biomejs/cli-linux-x64-musl@2.5.4': optional: true - '@biomejs/cli-linux-x64@2.5.2': + '@biomejs/cli-linux-x64@2.5.4': optional: true - '@biomejs/cli-win32-arm64@2.5.2': + '@biomejs/cli-win32-arm64@2.5.4': optional: true - '@biomejs/cli-win32-x64@2.5.2': + '@biomejs/cli-win32-x64@2.5.4': optional: true '@esbuild/aix-ppc64@0.28.1': @@ -1036,49 +1033,45 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true - '@vitest/expect@3.2.6': + '@vitest/expect@3.2.7': dependencies: '@types/chai': 5.2.3 - '@vitest/spy': 3.2.6 - '@vitest/utils': 3.2.6 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.6(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0))': + '@vitest/mocker@3.2.7(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1))': dependencies: - '@vitest/spy': 3.2.6 + '@vitest/spy': 3.2.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) - - '@vitest/pretty-format@3.2.6': - dependencies: - tinyrainbow: 2.0.0 + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) '@vitest/pretty-format@3.2.7': dependencies: tinyrainbow: 2.0.0 - '@vitest/runner@3.2.6': + '@vitest/runner@3.2.7': dependencies: - '@vitest/utils': 3.2.6 + '@vitest/utils': 3.2.7 pathe: 2.0.3 strip-literal: 3.1.0 - '@vitest/snapshot@3.2.6': + '@vitest/snapshot@3.2.7': dependencies: - '@vitest/pretty-format': 3.2.6 + '@vitest/pretty-format': 3.2.7 magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/spy@3.2.6': + '@vitest/spy@3.2.7': dependencies: tinyspy: 4.0.4 - '@vitest/utils@3.2.6': + '@vitest/utils@3.2.7': dependencies: - '@vitest/pretty-format': 3.2.6 + '@vitest/pretty-format': 3.2.7 loupe: 3.2.1 tinyrainbow: 2.0.0 @@ -1166,7 +1159,7 @@ snapshots: picomatch@4.0.5: {} - postcss@8.5.18: + postcss@8.5.19: dependencies: nanoid: 3.3.16 picocolors: 1.1.1 @@ -1230,7 +1223,7 @@ snapshots: tinyspy@4.0.4: {} - tsx@4.23.0: + tsx@4.23.1: dependencies: esbuild: 0.28.1 optionalDependencies: @@ -1261,13 +1254,13 @@ snapshots: undici-types@8.3.0: {} - vite-node@3.2.4(@types/node@26.1.1)(tsx@4.23.0): + vite-node@3.2.4(@types/node@26.1.1)(tsx@4.23.1): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) transitivePeerDependencies: - '@types/node' - jiti @@ -1282,29 +1275,29 @@ snapshots: - tsx - yaml - vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0): + vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1): dependencies: esbuild: 0.28.1 fdir: 6.5.0(picomatch@4.0.5) picomatch: 4.0.5 - postcss: 8.5.18 + postcss: 8.5.19 rollup: 4.62.2 tinyglobby: 0.2.17 optionalDependencies: '@types/node': 26.1.1 fsevents: 2.3.3 - tsx: 4.23.0 + tsx: 4.23.1 - vitest@3.2.6(@types/node@26.1.1)(tsx@4.23.0): + vitest@3.2.7(@types/node@26.1.1)(tsx@4.23.1): dependencies: '@types/chai': 5.2.3 - '@vitest/expect': 3.2.6 - '@vitest/mocker': 3.2.6(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.0)) + '@vitest/expect': 3.2.7 + '@vitest/mocker': 3.2.7(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1)) '@vitest/pretty-format': 3.2.7 - '@vitest/runner': 3.2.6 - '@vitest/snapshot': 3.2.6 - '@vitest/spy': 3.2.6 - '@vitest/utils': 3.2.6 + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 chai: 5.3.3 debug: 4.4.3 expect-type: 1.4.0 @@ -1317,8 +1310,8 @@ snapshots: tinyglobby: 0.2.17 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.0) - vite-node: 3.2.4(@types/node@26.1.1)(tsx@4.23.0) + vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) + vite-node: 3.2.4(@types/node@26.1.1)(tsx@4.23.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 26.1.1 From 125b0c5bec20b05503b45003c4e21b61918618d9 Mon Sep 17 00:00:00 2001 From: macanderson Date: Sat, 18 Jul 2026 18:28:14 -0700 Subject: [PATCH 14/18] Fix review findings across harness + adapter; add brand and web frontend (#20) Harness (TypeScript): - parse: brace-matched envelope scan so pretty-printed JSON (gemini-cli) parses; line-based JSONL scan kept as the first pass - adapters/base: kill the whole process group on timeout (agents spawn subprocesses that held the stdio pipes open past the cap); settle on "exit" with a drain grace; cache version (was two spawns per trial) - orchestrator: drop dead loadPricing call and module-level pricing cache (load once, pass down); timeout now takes precedence over a post-kill test pass; pass taskDir through AdapterRunArgs instead of the MockAdapter.currentTaskDir mutable static - workspace: async fs/child_process APIs (no event-loop blocking inside the async run loop); disable commit.gpgsign in seeded repos - report: aggregate via perAgentSummary (was duplicated inline math that contradicted summary.ts's single-source-of-truth contract); no more NaN cells when an agent has zero scored trials - cli: validate numeric flags instead of silently running with NaN - METHODOLOGY: document timeout precedence Harbor adapter (Python): - remove dead tomli fallback (package requires Python >= 3.12) - frozen dataclasses for specs and parsed metrics; validate metrics.kind - collections.abc.Mapping; class-level _agent_output/_spec defaults; tolerate float ARENA_TIMEOUT values - add ruff (lint config in pyproject, run in CI); fix all findings Branding + site: - black & white stadium mark (nested geometric stadium shapes), IBM Plex type, all-neutral palette; static site under web/ for arena.oxagen.sh - logo assets in assets/, logo in README --- .github/workflows/ci.yml | 1 + .gitignore | 5 + METHODOLOGY.md | 2 +- README.md | 7 +- assets/logo-icon-dark.svg | 5 + assets/logo-icon.svg | 7 + assets/logo.svg | 7 + harbor/arena_harbor/__init__.py | 6 +- harbor/arena_harbor/agents.py | 10 +- harbor/arena_harbor/base.py | 17 +- harbor/arena_harbor/command.py | 2 +- harbor/arena_harbor/metrics.py | 32 +- harbor/arena_harbor/spec.py | 33 +- harbor/pyproject.toml | 14 +- harbor/tests/test_agents.py | 7 +- harbor/tests/test_command.py | 7 +- harbor/tests/test_spec.py | 2 +- src/adapters/base.ts | 72 +++- src/adapters/mock.ts | 11 +- src/cli.ts | 44 ++- src/orchestrator.ts | 38 +- src/parse.ts | 73 +++- src/report.ts | 29 +- src/workspace.ts | 83 ++-- test/adapters.test.ts | 1 + test/parse.test.ts | 21 ++ test/pipeline.test.ts | 27 +- web/favicon.svg | 11 + web/index.html | 327 ++++++++++++++++ web/styles.css | 648 ++++++++++++++++++++++++++++++++ web/vercel.json | 14 + 31 files changed, 1351 insertions(+), 212 deletions(-) create mode 100644 assets/logo-icon-dark.svg create mode 100644 assets/logo-icon.svg create mode 100644 assets/logo.svg create mode 100644 web/favicon.svg create mode 100644 web/index.html create mode 100644 web/styles.css create mode 100644 web/vercel.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 221b125..6d5552b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: python-version: "3.12" # Harbor requires >= 3.12 cache: pip - run: pip install -e '.[dev]' + - run: ruff check . # Spec/metrics/command logic is Harbor-free; the agent tests exercise the # real Harbor BaseInstalledAgent subclasses (Harbor is a declared dep). - run: pytest -q diff --git a/.gitignore b/.gitignore index 75c1cfc..478ac3f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ results/ dist/ *.tsbuildinfo .DS_Store +# macOS Finder folder-icon artifact ("Icon\r") +Icon? + +# Vercel project link (contains project/org ids) +.vercel/ # Agent working caches (code graph, context DB) — never commit. .stella/ diff --git a/METHODOLOGY.md b/METHODOLOGY.md index 42086a2..eda2472 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -10,7 +10,7 @@ One run executes N agents × T tasks × K trials. Every trial: 2. **Agent** — the agent CLI runs headlessly in that directory with the shared model/budget/timeout. Arguments are passed as argv arrays (no shell). 3. **Diff** — `git diff` against the seed commit captures exactly what the agent changed. 4. **Verify** — the harness deletes anything at `.arena-verify/`, copies the task's held-out `verify/` tests in, and runs them with `node --test`. The tests were never on disk while the agent ran. -5. **Score** — `passed` iff the held-out tests pass. `timeout` if the agent hit the wall-clock cap. `agent-error` if the CLI could not be invoked at all (spawn failure, or non-zero exit with zero diff and zero tokens) — these are harness-side failures, excluded from every comparison and listed separately. +5. **Score** — `timeout` if the agent hit the wall-clock cap (takes precedence: exceeding the matched budget never scores as a win, even if the tests pass on whatever state the kill left behind). Otherwise `passed` iff the held-out tests pass. `agent-error` if the CLI could not be invoked at all (spawn failure, or non-zero exit with zero diff and zero tokens) — these are harness-side failures, excluded from every comparison and listed separately. ## Fairness controls diff --git a/README.md b/README.md index 35369ab..e736e4f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ + + + Arena — two nested stadium shapes + + # Arena [![CI](https://github.com/oxageninc/arena/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/oxageninc/arena/actions/workflows/ci.yml) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/oxageninc/arena/badge)](https://scorecard.dev/viewer/?uri=github.com/oxageninc/arena) -**Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** +**Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** · [arena.oxagen.sh](https://arena.oxagen.sh) Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/oxageninc/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. diff --git a/assets/logo-icon-dark.svg b/assets/logo-icon-dark.svg new file mode 100644 index 0000000..453257b --- /dev/null +++ b/assets/logo-icon-dark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/assets/logo-icon.svg b/assets/logo-icon.svg new file mode 100644 index 0000000..4d0f6ed --- /dev/null +++ b/assets/logo-icon.svg @@ -0,0 +1,7 @@ + + + + + + diff --git a/assets/logo.svg b/assets/logo.svg new file mode 100644 index 0000000..79884b6 --- /dev/null +++ b/assets/logo.svg @@ -0,0 +1,7 @@ + + + + + + ARENA + diff --git a/harbor/arena_harbor/__init__.py b/harbor/arena_harbor/__init__.py index b5423d2..e7c7059 100644 --- a/harbor/arena_harbor/__init__.py +++ b/harbor/arena_harbor/__init__.py @@ -28,13 +28,13 @@ ) __all__ = [ + "AgentSpec", "ArenaInstalledAgent", "ByoAgent", - "OxagenAgent", - "StellaAgent", - "AgentSpec", "InstallSpec", "MetricsSpec", + "OxagenAgent", + "StellaAgent", "load_spec_file", "load_spec_from_env", "spec_from_dict", diff --git a/harbor/arena_harbor/agents.py b/harbor/arena_harbor/agents.py index 353816f..731f82a 100644 --- a/harbor/arena_harbor/agents.py +++ b/harbor/arena_harbor/agents.py @@ -44,6 +44,8 @@ class ByoAgent(ArenaInstalledAgent): loaded once and cached on the instance. """ + _spec: AgentSpec | None = None + @staticmethod def name() -> str: # Static so Harbor can label results without instantiating; the concrete @@ -51,8 +53,6 @@ def name() -> str: return os.environ.get("ARENA_AGENT_NAME", "arena-byo") def spec(self) -> AgentSpec: - cached = getattr(self, "_spec", None) - if cached is None: - cached = load_spec_from_env() - self._spec = cached - return cached + if self._spec is None: + self._spec = load_spec_from_env() + return self._spec diff --git a/harbor/arena_harbor/base.py b/harbor/arena_harbor/base.py index bdf9c39..f0ad518 100644 --- a/harbor/arena_harbor/base.py +++ b/harbor/arena_harbor/base.py @@ -44,6 +44,9 @@ class ArenaInstalledAgent(BaseInstalledAgent): """Base for Arena's Harbor agents. Subclasses set ``name()`` and ``spec()``.""" + #: Combined stdout/stderr of the last agent run (metrics are read from it). + _agent_output: str = "" + def spec(self) -> AgentSpec: raise NotImplementedError("subclasses must implement spec()") @@ -139,7 +142,7 @@ async def run( result = await environment.exec( command=command, env=env, - timeout_sec=int(timeout) if timeout.isdigit() else None, + timeout_sec=_parse_timeout(timeout), ) self._agent_output = "\n".join( part @@ -149,7 +152,7 @@ async def run( ) if part ) - except Exception as exc: # noqa: BLE001 - never let agent failure kill scoring + except Exception as exc: self._agent_output = f"[arena-harbor] agent execution error: {exc}" self.logger.warning("arena-harbor: agent run raised: %s", exc) @@ -158,7 +161,7 @@ def populate_context_post_run(self, context: AgentContext) -> None: spec = self.spec() if spec.metrics is None: return - output = getattr(self, "_agent_output", "") + output = self._agent_output if not output: return parsed = extract_metrics(output, spec.metrics) @@ -167,3 +170,11 @@ def populate_context_post_run(self, context: AgentContext) -> None: if parsed.cost_usd is not None: context.cost_usd = parsed.cost_usd context.metadata = {**(context.metadata or {}), **parsed.as_metadata(spec.name)} + + +def _parse_timeout(raw: str) -> int | None: + """``ARENA_TIMEOUT`` seconds as an int; None (no limit) when unparseable.""" + try: + return int(float(raw)) + except ValueError: + return None diff --git a/harbor/arena_harbor/command.py b/harbor/arena_harbor/command.py index d94fb73..e243d69 100644 --- a/harbor/arena_harbor/command.py +++ b/harbor/arena_harbor/command.py @@ -10,7 +10,7 @@ import os import re import shlex -from typing import Mapping +from collections.abc import Mapping from .spec import AgentSpec diff --git a/harbor/arena_harbor/metrics.py b/harbor/arena_harbor/metrics.py index faea373..24b2c39 100644 --- a/harbor/arena_harbor/metrics.py +++ b/harbor/arena_harbor/metrics.py @@ -12,13 +12,13 @@ import json import re -from dataclasses import dataclass +from dataclasses import dataclass, fields from typing import Any from .spec import MetricsSpec -@dataclass +@dataclass(frozen=True) class ParsedMetrics: input_tokens: int | None = None output_tokens: int | None = None @@ -28,30 +28,14 @@ class ParsedMetrics: def as_metadata(self, agent_name: str) -> dict[str, Any]: prefix = f"arena_{agent_name.replace('-', '_')}" - out: dict[str, Any] = {} - for field_name in ( - "input_tokens", - "output_tokens", - "cache_read_tokens", - "total_tokens", - "cost_usd", - ): - value = getattr(self, field_name) - if value is not None: - out[f"{prefix}_{field_name}"] = value - return out + return { + f"{prefix}_{f.name}": getattr(self, f.name) + for f in fields(self) + if getattr(self, f.name) is not None + } def is_empty(self) -> bool: - return all( - getattr(self, f) is None - for f in ( - "input_tokens", - "output_tokens", - "cache_read_tokens", - "total_tokens", - "cost_usd", - ) - ) + return all(getattr(self, f.name) is None for f in fields(self)) def last_json_object(text: str) -> dict[str, Any] | None: diff --git a/harbor/arena_harbor/spec.py b/harbor/arena_harbor/spec.py index 5981d79..d3c14d5 100644 --- a/harbor/arena_harbor/spec.py +++ b/harbor/arena_harbor/spec.py @@ -15,6 +15,7 @@ import json import os +import tomllib from dataclasses import dataclass, field from pathlib import Path from typing import Any, Literal @@ -22,7 +23,7 @@ InstallKind = Literal["binary", "script"] -@dataclass +@dataclass(frozen=True) class InstallSpec: """How the agent binary gets into the (Linux) container. @@ -43,7 +44,7 @@ class InstallSpec: system_packages: list[str] = field(default_factory=list) -@dataclass +@dataclass(frozen=True) class MetricsSpec: """How to recover token/cost numbers from the agent's stdout. @@ -70,7 +71,7 @@ class MetricsSpec: total_regex: str | None = None -@dataclass +@dataclass(frozen=True) class AgentSpec: """A complete recipe for running one coding CLI under Harbor.""" @@ -136,8 +137,13 @@ def spec_from_dict(data: dict[str, Any]) -> AgentSpec: if metrics_raw is not None: if not isinstance(metrics_raw, dict): raise ValueError("'metrics' must be a table/object") + metrics_kind = metrics_raw.get("kind", "json_tail") + if metrics_kind not in ("json_tail", "regex"): + raise ValueError( + f"metrics.kind must be 'json_tail' or 'regex', got '{metrics_kind}'" + ) metrics = MetricsSpec( - kind=metrics_raw.get("kind", "json_tail"), + kind=metrics_kind, input_path=metrics_raw.get("input_path"), output_path=metrics_raw.get("output_path"), cache_read_path=metrics_raw.get("cache_read_path"), @@ -171,10 +177,7 @@ def load_spec_file(path: str | Path) -> AgentSpec: if not p.is_file(): raise FileNotFoundError(f"agent spec file not found: {p}") text = p.read_text() - if p.suffix.lower() == ".json": - data = json.loads(text) - else: - data = _load_toml(text) + data = json.loads(text) if p.suffix.lower() == ".json" else tomllib.loads(text) if not isinstance(data, dict): raise ValueError(f"{p}: top level must be a table/object") return spec_from_dict(data) @@ -190,17 +193,3 @@ def load_spec_from_env(env_var: str = "ARENA_AGENT_SPEC") -> AgentSpec: f"(see specs/byo.example.toml)." ) return load_spec_file(path) - - -def _load_toml(text: str) -> Any: - try: - import tomllib # Python 3.11+ - except ModuleNotFoundError: # pragma: no cover - exercised on <3.11 only - try: - import tomli as tomllib # type: ignore[no-redef] - except ModuleNotFoundError as exc: # pragma: no cover - raise RuntimeError( - "Parsing TOML specs on Python < 3.11 needs the 'tomli' package " - "(pip install tomli), or use a .json spec instead." - ) from exc - return tomllib.loads(text) diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index 2b942c4..f4d0907 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=7.0", "pytest-asyncio>=0.21"] +dev = ["pytest>=7.0", "pytest-asyncio>=0.21", "ruff>=0.8"] [project.urls] Homepage = "https://github.com/oxageninc/arena" @@ -41,3 +41,15 @@ arena_harbor = ["py.typed"] [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"] +ignore = [ + # Broad excepts are deliberate at the agent-run boundary (never let an agent + # crash abort scoring); each site carries a comment. + "BLE001", +] diff --git a/harbor/tests/test_agents.py b/harbor/tests/test_agents.py index 6f0a76e..0cc27ce 100644 --- a/harbor/tests/test_agents.py +++ b/harbor/tests/test_agents.py @@ -10,7 +10,7 @@ pytest.importorskip("harbor") -from arena_harbor import ByoAgent, OxagenAgent, StellaAgent # noqa: E402 +from arena_harbor import ByoAgent, OxagenAgent, StellaAgent def _mk(agent_cls, tmp_path, model_name="anthropic/claude-sonnet-5"): @@ -76,7 +76,10 @@ async def test_script_install_uses_override_env(tmp_path, monkeypatch): async def test_run_invokes_agent_and_captures_output(tmp_path): agent = _mk(OxagenAgent, tmp_path) - envelope = '{"type":"result","usage":{"inputTokens":1000,"outputTokens":200,"cachedInputTokens":400}}' + envelope = ( + '{"type":"result","usage":' + '{"inputTokens":1000,"outputTokens":200,"cachedInputTokens":400}}' + ) env = _mock_env(stdout=envelope) context = MagicMock(metadata={}) diff --git a/harbor/tests/test_command.py b/harbor/tests/test_command.py index 25b9db7..036e509 100644 --- a/harbor/tests/test_command.py +++ b/harbor/tests/test_command.py @@ -16,7 +16,10 @@ def _spec(run_template, **kw): def test_render_template_replaces_known_leaves_unknown(): - out = render_template("{bin} run {instruction} ${HOME} awk '{print $1}'", {"bin": "x", "instruction": "'go'"}) + out = render_template( + "{bin} run {instruction} ${HOME} awk '{print $1}'", + {"bin": "x", "instruction": "'go'"}, + ) assert out == "x run 'go' ${HOME} awk '{print $1}'" @@ -71,4 +74,4 @@ def test_builtin_run_templates_render(): assert oxa.strip().endswith("'do it'") stella = build_command(STELLA_SPEC, "zai/glm-5.2", "do it", budget="5") - assert "stella --model zai/glm-5.2 --output-format json --budget 5 run 'do it'" == stella + assert stella == "stella --model zai/glm-5.2 --output-format json --budget 5 run 'do it'" diff --git a/harbor/tests/test_spec.py b/harbor/tests/test_spec.py index 6647e21..7c81666 100644 --- a/harbor/tests/test_spec.py +++ b/harbor/tests/test_spec.py @@ -54,7 +54,7 @@ def test_script_install_requires_script_or_env(): def test_unknown_install_kind_rejected(): data = _valid_dict() data["install"] = {"kind": "docker"} - with pytest.raises(ValueError, match="binary.*script"): + with pytest.raises(ValueError, match=r"binary.*script"): spec_from_dict(data) diff --git a/src/adapters/base.ts b/src/adapters/base.ts index c055d81..a2d84c5 100644 --- a/src/adapters/base.ts +++ b/src/adapters/base.ts @@ -19,6 +19,9 @@ export interface AdapterRunArgs { budgetUsd: number | undefined; timeoutSeconds: number; workDir: string; + /** Task fixture directory. Only the in-process mock may read it; real CLIs + * must never see the fixture (it contains the held-out tests). */ + taskDir: string; } export interface ExecOutcome { @@ -71,17 +74,22 @@ export abstract class Adapter { } } + private cachedVersion: string | undefined; + version(): string { - try { - return execFileSync(this.bin(), ["--version"], { - encoding: "utf8", - timeout: 10_000, - }) - .trim() - .split("\n")[0] as string; - } catch { - return "unknown"; + if (this.cachedVersion === undefined) { + try { + this.cachedVersion = execFileSync(this.bin(), ["--version"], { + encoding: "utf8", + timeout: 10_000, + }) + .trim() + .split("\n")[0] as string; + } catch { + this.cachedVersion = "unknown"; + } } + return this.cachedVersion; } /** @@ -90,37 +98,61 @@ export abstract class Adapter { */ execute(args: AdapterRunArgs): Promise { return new Promise((resolve) => { + // detached: the agent gets its own process group, so a timeout kill + // reaches the whole tree — agents routinely spawn test runners and + // shells that would otherwise survive and hold the stdio pipes open. + const detached = process.platform !== "win32"; const child = spawn(this.bin(), this.buildArgs(args), { cwd: args.workDir, env: { ...process.env, ...this.env(args) }, stdio: ["ignore", "pipe", "pipe"], + detached, }); let stdout = ""; let stderr = ""; let timedOut = false; + let settled = false; + + const killTree = (): void => { + if (detached && child.pid !== undefined) { + try { + process.kill(-child.pid, "SIGKILL"); + return; + } catch { + // Process group already gone — fall through to a direct kill. + } + } + child.kill("SIGKILL"); + }; + const timer = setTimeout(() => { timedOut = true; - child.kill("SIGKILL"); + killTree(); }, args.timeoutSeconds * 1000); + const settle = (outcome: ExecOutcome): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(outcome); + }; + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); child.on("error", (err) => { - clearTimeout(timer); - resolve({ - stdout, - stderr, - exitCode: null, - timedOut: false, - spawnError: err.message, - }); + settle({ stdout, stderr, exitCode: null, timedOut: false, spawnError: err.message }); }); child.on("close", (code) => { - clearTimeout(timer); - resolve({ stdout, stderr, exitCode: code, timedOut }); + settle({ stdout, stderr, exitCode: code, timedOut }); + }); + + // "close" waits for the stdio streams to drain; an escaped grandchild + // holding the pipes must not stall the run forever after exit. + child.on("exit", (code) => { + setTimeout(() => settle({ stdout, stderr, exitCode: code, timedOut }), 2000).unref(); }); }); } diff --git a/src/adapters/mock.ts b/src/adapters/mock.ts index 205dc22..67845eb 100644 --- a/src/adapters/mock.ts +++ b/src/adapters/mock.ts @@ -18,9 +18,6 @@ export class MockAdapter extends Adapter { readonly name = "mock"; protected readonly defaultBinary = "mock"; - /** The orchestrator records the active task dir here before each execute. */ - static currentTaskDir: string | null = null; - override isAvailable(): boolean { return true; } @@ -34,11 +31,9 @@ export class MockAdapter extends Adapter { } override execute(args: AdapterRunArgs): Promise { - const behavior = args.model; - if (behavior === "solve") { - const taskDir = MockAdapter.currentTaskDir; - const solution = taskDir ? join(taskDir, "solution") : null; - if (solution && existsSync(solution)) { + if (args.model === "solve") { + const solution = join(args.taskDir, "solution"); + if (existsSync(solution)) { cpSync(solution, args.workDir, { recursive: true }); } } diff --git a/src/cli.ts b/src/cli.ts index aa01a57..8fdb19c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -55,7 +55,7 @@ switch (command) { cmdReport(rest); break; case "verify": - cmdVerify(rest); + await cmdVerify(rest); break; case "baseline": cmdBaseline(rest); @@ -169,13 +169,14 @@ async function cmdRun(argv: string[]): Promise { } } + const budgetUsd = values.budget !== undefined ? numFlag("budget", values.budget) : undefined; const config = { agents, tasks, - trials: values.trials ? parseInt(values.trials, 10) : 3, - ...(values.budget !== undefined ? { budgetUsd: parseFloat(values.budget) } : {}), - timeoutSeconds: values.timeout ? parseInt(values.timeout, 10) : 600, - seed: values.seed ? parseInt(values.seed, 10) : 42, + trials: intFlag("trials", values.trials, 3), + ...(budgetUsd !== undefined ? { budgetUsd } : {}), + timeoutSeconds: intFlag("timeout", values.timeout, 600), + seed: intFlag("seed", values.seed, 42, 0), outDir: resolve(values.out ?? "results"), }; @@ -185,6 +186,27 @@ async function cmdRun(argv: string[]): Promise { console.log(`\nRun complete. Report: ${join(runDir, "report.md")}`); } +/** Parse an integer CLI flag; reject garbage instead of silently running with NaN. */ +function intFlag(name: string, raw: string | undefined, fallback: number, min = 1): number { + if (raw === undefined) return fallback; + const n = Number.parseInt(raw, 10); + if (!Number.isFinite(n) || n < min || String(n) !== raw.trim()) { + console.error(`--${name} must be an integer ≥ ${String(min)}, got "${raw}"`); + process.exit(1); + } + return n; +} + +/** Parse a positive numeric CLI flag. */ +function numFlag(name: string, raw: string): number { + const n = Number.parseFloat(raw); + if (!Number.isFinite(n) || n <= 0) { + console.error(`--${name} must be a positive number, got "${raw}"`); + process.exit(1); + } + return n; +} + function cmdReport(argv: string[]): void { const runDir = argv[0]; if (!runDir) { @@ -201,7 +223,7 @@ function cmdReport(argv: string[]): void { * workspace (no tautological tests) and PASS against the reference solution * (the task is actually solvable). CI runs this on every push. */ -function cmdVerify(argv: string[]): void { +async function cmdVerify(argv: string[]): Promise { const all = loadTasks(TASK_ROOT); const tasks = argv.length > 0 ? all.filter((t) => argv.includes(t.id)) : all; let failures = 0; @@ -210,12 +232,12 @@ function cmdVerify(argv: string[]): void { const pristineDir = mkdtempSync(join(tmpdir(), "arena-audit-")); const solvedDir = mkdtempSync(join(tmpdir(), "arena-audit-")); try { - seedWorkspace(task, pristineDir); - const pristine = runVerification(task, pristineDir); + await seedWorkspace(task, pristineDir); + const pristine = await runVerification(task, pristineDir); - seedWorkspace(task, solvedDir); - applySolution(task, solvedDir); - const solved = runVerification(task, solvedDir); + await seedWorkspace(task, solvedDir); + await applySolution(task, solvedDir); + const solved = await runVerification(task, solvedDir); const ok = !pristine.passed && solved.passed; if (!ok) failures++; diff --git a/src/orchestrator.ts b/src/orchestrator.ts index ce675f4..6aad23c 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -20,13 +20,13 @@ import { arch, platform, tmpdir } from "node:os"; import { join } from "node:path"; import { type Adapter, createAdapter } from "./adapters/index.js"; -import { MockAdapter } from "./adapters/mock.js"; import { diffStats, sanitizeSegment } from "./parse.js"; import { computeCost, loadPricing } from "./pricing.js"; import type { AgentSpec, LoadedTask, Outcome, + PricingTable, RunConfig, RunManifest, TrialResult, @@ -41,7 +41,7 @@ export async function executeRun( config: RunConfig, log: RunProgress = () => {}, ): Promise<{ runDir: string; manifest: RunManifest; results: TrialResult[] }> { - const _pricing = loadPricing(); + const pricing = loadPricing(); const runId = `run-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}-${randomUUID().slice(0, 8)}`; const runDir = join(config.outDir, runId); mkdirSync(join(runDir, "trials"), { recursive: true }); @@ -96,7 +96,7 @@ export async function executeRun( for (const task of config.tasks) { for (const { spec, adapter } of ordered) { log(`▶ trial ${trial}/${config.trials} · ${task.id} · ${spec.adapter} (${spec.model})`); - const result = await runOne(task, spec, adapter, trial, config, runId, runDir); + const result = await runOne(task, spec, adapter, trial, config, runId, runDir, pricing); results.push(result); writeFileSync(join(runDir, "trials", `${result.id}.json`), JSON.stringify(result, null, 2)); const mark = @@ -122,14 +122,14 @@ async function runOne( config: RunConfig, runId: string, runDir: string, + pricing: PricingTable, ): Promise { const workDir = join(tmpdir(), `arena-${sanitizeSegment(task.id)}-${randomUUID()}`); const startedAt = new Date(); const id = `${task.id}-${spec.adapter}-${sanitizeSegment(spec.model)}-t${trial}`; try { - seedWorkspace(task, workDir); - MockAdapter.currentTaskDir = task.dir; + await seedWorkspace(task, workDir); const exec = await adapter.execute({ prompt: buildPrompt(task), @@ -137,12 +137,13 @@ async function runOne( budgetUsd: config.budgetUsd, timeoutSeconds: config.timeoutSeconds, workDir, + taskDir: task.dir, }); const finishedAt = new Date(); const wallClockSeconds = (finishedAt.getTime() - startedAt.getTime()) / 1000; - const diff = collectDiff(workDir); + const diff = await collectDiff(workDir); const envelope = adapter.parseEnvelope(exec.stdout); // Invocation-level failure: the agent never actually ran (bad flags, @@ -156,9 +157,13 @@ async function runOne( if (invocationFailure) { outcome = "agent-error"; } else { - verify = runVerification(task, workDir); - if (verify.passed) outcome = "passed"; - else outcome = exec.timedOut ? "timeout" : "failed"; + verify = await runVerification(task, workDir); + // A trial that blew the wall-clock cap is a timeout even if the tests + // happen to pass on whatever state the kill left behind — exceeding the + // matched budget must never score as a win (see METHODOLOGY.md). + if (exec.timedOut) outcome = "timeout"; + else if (verify.passed) outcome = "passed"; + else outcome = "failed"; } const transcriptPath = join("transcripts", `${id}.txt`); @@ -206,9 +211,9 @@ async function runOne( }, tokens: envelope.tokens, cost: { - computedUsd: computeCost(envelope.tokens, spec.model, loadPricingCached()), + computedUsd: computeCost(envelope.tokens, spec.model, pricing), agentReportedUsd: envelope.agentReportedUsd, - pricingModel: loadPricingCached()[spec.model] ? spec.model : null, + pricingModel: pricing[spec.model] ? spec.model : null, }, activity: { toolCalls: envelope.toolCalls, @@ -228,17 +233,10 @@ async function runOne( ...(errorText !== undefined ? { error: errorText } : {}), }; } finally { - MockAdapter.currentTaskDir = null; rmSync(workDir, { recursive: true, force: true }); } } -let pricingCache: ReturnType | null = null; -function loadPricingCached(): ReturnType { - pricingCache ??= loadPricing(); - return pricingCache; -} - function gitSha(): string { try { return execFileSync("git", ["rev-parse", "--short", "HEAD"], { @@ -251,9 +249,7 @@ function gitSha(): string { } function buildReproduceCommand(config: RunConfig): string { - const agents = config.agents - .map((a) => (a.model ? `${a.adapter}=${a.model}` : a.adapter)) - .join(","); + const agents = config.agents.map((a) => `${a.adapter}=${a.model}`).join(","); const parts = [ "pnpm arena run", `--agents ${agents}`, diff --git a/src/parse.ts b/src/parse.ts index 1827144..963a4c0 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -22,32 +22,77 @@ export function isRecord(value: unknown): value is Record { /** * Extract the machine-readable JSON envelope from an agent's stdout. * - * Headless agent CLIs print either a single JSON object or JSONL (one event - * per line). We want the last `type: "result"` object, falling back to the - * last parseable object. Banners and log lines are ignored. + * Headless agent CLIs print a single JSON object (compact or pretty-printed — + * gemini-cli pretty-prints), or JSONL (one event per line), often surrounded + * by banners and log lines. We want the last `type: "result"` object, falling + * back to the last parseable object. */ export function parseJsonEnvelope(stdout: string): Record | null { if (!stdout) return null; + // Pass 1 — line-oriented JSONL scan. Robust to banner text because only + // whole lines that are complete JSON objects are considered. + let lineFallback: Record | null = null; const lines = stdout .split("\n") .map((line) => line.trim()) .filter((line) => line.startsWith("{")); + for (let i = lines.length - 1; i >= 0; i--) { + const obj = tryParseRecord(lines[i] as string); + if (!obj) continue; + if (obj["type"] === "result") return obj; + lineFallback ??= obj; + } - let fallback: Record | null = null; + // Pass 2 — brace-matched scan, needed for multi-line (pretty-printed) + // objects the line scan cannot see. + let blockFallback: Record | null = null; + for (const block of topLevelJsonBlocks(stdout).reverse()) { + const obj = tryParseRecord(block); + if (!obj) continue; + if (obj["type"] === "result") return obj; + blockFallback ??= obj; + } - for (let i = lines.length - 1; i >= 0; i--) { - try { - const obj: unknown = JSON.parse(lines[i] as string); - if (!isRecord(obj)) continue; - if (fallback === null) fallback = obj; - if (obj["type"] === "result") return obj; - } catch { - // Not a complete JSON object line — skip. - } + return lineFallback ?? blockFallback; +} + +function tryParseRecord(text: string): Record | null { + try { + const obj: unknown = JSON.parse(text); + return isRecord(obj) ? obj : null; + } catch { + return null; } +} - return fallback; +/** Split text into top-level `{…}` runs, brace-matched and JSON-string-aware. */ +function topLevelJsonBlocks(text: string): string[] { + const blocks: string[] = []; + let depth = 0; + let start = -1; + let inString = false; + let escaped = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (escaped) escaped = false; + else if (ch === "\\") escaped = true; + else if (ch === '"') inString = false; + } else if (ch === '"') { + if (depth > 0) inString = true; + } else if (ch === "{") { + if (depth === 0) start = i; + depth++; + } else if (ch === "}" && depth > 0) { + depth--; + if (depth === 0 && start !== -1) { + blocks.push(text.slice(start, i + 1)); + start = -1; + } + } + } + return blocks; } /** Make a string safe as a single path segment (model slugs contain "/"). */ diff --git a/src/report.ts b/src/report.ts index ab17ae6..a186107 100644 --- a/src/report.ts +++ b/src/report.ts @@ -15,7 +15,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { isRecord } from "./parse.js"; -import { mcnemarExact, median, pairedBootstrapDelta, wilsonInterval } from "./stats.js"; +import { mcnemarExact, median, pairedBootstrapDelta } from "./stats.js"; +import { agentKey, perAgentSummary } from "./summary.js"; import type { RunManifest, TrialResult } from "./types.js"; export function loadRun(runDir: string): { @@ -32,10 +33,6 @@ export function loadRun(runDir: string): { }; } -function agentKey(r: TrialResult): string { - return `${r.agent.adapter} (${r.agent.model})`; -} - function pct(x: number): string { return `${(x * 100).toFixed(1)}%`; } @@ -52,7 +49,10 @@ function fmtDelta(x: number): string { export function generateReport(runDir: string): string { const { manifest, results } = loadRun(runDir); - const keys = [...new Set(results.map(agentKey))]; + // Aggregation lives in perAgentSummary so the report, the baseline snapshot, + // and the regression gate always agree exactly. + const summaries = perAgentSummary(results); + const keys = summaries.map((s) => s.key); const byAgent = new Map( keys.map((k) => [k, results.filter((r) => agentKey(r) === k)]), ); @@ -102,17 +102,14 @@ export function generateReport(runDir: string): string { "| Agent | Scored trials | Passed | Success rate (95% CI) | Median wall clock | Median tokens (billed) | Median computed cost | Self-reported cost |", "|---|---|---|---|---|---|---|---|", ); - for (const key of keys) { - const all = byAgent.get(key) ?? []; - const scored = all.filter((r) => r.outcome !== "agent-error"); - const passed = scored.filter((r) => r.outcome === "passed").length; - const [lo, hi] = wilsonInterval(passed, scored.length); - const wall = median(scored.map((r) => r.timing.wallClockSeconds)); - const toks = median(scored.map((r) => r.tokens.total)); - const costs = scored.map((r) => r.cost.computedUsd).filter((c): c is number => c !== null); - const self = scored.map((r) => r.cost.agentReportedUsd).filter((c): c is number => c !== null); + for (const s of summaries) { + const [lo, hi] = s.successCI; + const wall = + s.medianWallClockSeconds === null ? "—" : `${s.medianWallClockSeconds.toFixed(1)}s`; + const toks = + s.medianTotalTokens === null ? "—" : Math.round(s.medianTotalTokens).toLocaleString(); lines.push( - `| ${key} | ${String(scored.length)} | ${String(passed)} | ${pct(scored.length ? passed / scored.length : 0)} (${pct(lo)}–${pct(hi)}) | ${wall.toFixed(1)}s | ${Math.round(toks).toLocaleString()} | ${fmtUsd(costs.length ? median(costs) : null)} | ${fmtUsd(self.length ? median(self) : null)} |`, + `| ${s.key} | ${String(s.scoredTrials)} | ${String(s.passed)} | ${pct(s.successRate)} (${pct(lo)}–${pct(hi)}) | ${wall} | ${toks} | ${fmtUsd(s.medianComputedCost)} | ${fmtUsd(s.medianAgentReportedCost)} |`, ); } lines.push(""); diff --git a/src/workspace.ts b/src/workspace.ts index e5062bd..efceaab 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -8,23 +8,19 @@ * Node's built-in test runner. Success is decided only by those tests. */ -import { execFileSync } from "node:child_process"; -import { - cpSync, - existsSync, - mkdirSync, - readdirSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { execFile } from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { cp, mkdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { promisify } from "node:util"; import { isRecord } from "./parse.js"; import type { LoadedTask, TaskDef } from "./types.js"; +const execFileAsync = promisify(execFile); + const VERIFY_DIR = ".arena-verify"; -/** Load tasks// directories under a task root. */ +/** Load tasks// directories under a task root (sync: startup-time config read). */ export function loadTasks(taskRoot: string): LoadedTask[] { const tasks: LoadedTask[] = []; for (const entry of readdirSync(taskRoot, { withFileTypes: true })) { @@ -64,33 +60,36 @@ export function buildPrompt(task: TaskDef): string { * fixture plus TASK.md, committed to a fresh git repo so `git diff` isolates * exactly the agent's changes. */ -export function seedWorkspace(task: LoadedTask, workDir: string): void { - mkdirSync(workDir, { recursive: true }); - cpSync(join(task.dir, "workspace"), workDir, { recursive: true }); - writeFileSync(join(workDir, "TASK.md"), `${buildPrompt(task)}\n`); - writeFileSync( +export async function seedWorkspace(task: LoadedTask, workDir: string): Promise { + await mkdir(workDir, { recursive: true }); + await cp(join(task.dir, "workspace"), workDir, { recursive: true }); + await writeFile(join(workDir, "TASK.md"), `${buildPrompt(task)}\n`); + await writeFile( join(workDir, ".gitignore"), `${["node_modules/", "dist/", "coverage/", "*.log", `${VERIFY_DIR}/`].join("\n")}\n`, ); - const git = (args: string[]): void => { - execFileSync("git", args, { cwd: workDir, stdio: "ignore" }); + const git = async (args: string[]): Promise => { + await execFileAsync("git", args, { cwd: workDir }); }; - git(["init"]); - git(["config", "user.email", "arena@localhost"]); - git(["config", "user.name", "arena"]); - git(["add", "-A"]); - git(["commit", "-m", "arena: seed workspace", "--no-verify"]); + await git(["init", "-q"]); + await git(["config", "user.email", "arena@localhost"]); + await git(["config", "user.name", "arena"]); + // Host-level commit.gpgsign=true would make the seed commit prompt or fail. + await git(["config", "commit.gpgsign", "false"]); + await git(["add", "-A"]); + await git(["commit", "-q", "-m", "arena: seed workspace", "--no-verify"]); } /** Diff of the agent's changes (tracked + new files) against the seed commit. */ -export function collectDiff(workDir: string): string { +export async function collectDiff(workDir: string): Promise { try { - execFileSync("git", ["add", "-A"], { cwd: workDir, stdio: "ignore" }); - return execFileSync("git", ["diff", "--cached", "HEAD"], { + await execFileAsync("git", ["add", "-A"], { cwd: workDir }); + const { stdout } = await execFileAsync("git", ["diff", "--cached", "HEAD"], { cwd: workDir, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, }); + return stdout; } catch { return ""; } @@ -106,10 +105,10 @@ export interface VerifyResult { * Any pre-existing `.arena-verify/` content (agent-planted or stale) is * removed first. */ -export function runVerification(task: LoadedTask, workDir: string): VerifyResult { +export async function runVerification(task: LoadedTask, workDir: string): Promise { const target = join(workDir, VERIFY_DIR); - rmSync(target, { recursive: true, force: true }); - cpSync(join(task.dir, "verify"), target, { recursive: true }); + await rm(target, { recursive: true, force: true }); + await cp(join(task.dir, "verify"), target, { recursive: true }); const timeoutMs = (task.verifyTimeoutSeconds ?? 60) * 1000; // node --test needs a glob, not a bare directory (a directory arg is @@ -118,14 +117,18 @@ export function runVerification(task: LoadedTask, workDir: string): VerifyResult const { FORCE_COLOR: _forceColor, ...baseEnv } = process.env; const env = { ...baseEnv, NO_COLOR: "1" }; try { - const output = execFileSync(process.execPath, ["--test", `${VERIFY_DIR}/**/*.test.mjs`], { - cwd: workDir, - encoding: "utf8", - timeout: timeoutMs, - maxBuffer: 16 * 1024 * 1024, - env, - }); - return { passed: true, output: tail(output) }; + const { stdout } = await execFileAsync( + process.execPath, + ["--test", `${VERIFY_DIR}/**/*.test.mjs`], + { + cwd: workDir, + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + env, + }, + ); + return { passed: true, output: tail(stdout) }; } catch (error) { const e = error as { stdout?: string | Buffer; stderr?: string | Buffer; message?: string }; const out = [ @@ -137,13 +140,13 @@ export function runVerification(task: LoadedTask, workDir: string): VerifyResult .join("\n"); return { passed: false, output: tail(out) }; } finally { - rmSync(target, { recursive: true, force: true }); + await rm(target, { recursive: true, force: true }); } } /** Copy the reference solution over a workspace (used by `arena verify`). */ -export function applySolution(task: LoadedTask, workDir: string): void { - cpSync(join(task.dir, "solution"), workDir, { recursive: true }); +export async function applySolution(task: LoadedTask, workDir: string): Promise { + await cp(join(task.dir, "solution"), workDir, { recursive: true }); } function tail(text: string, maxChars = 4000): string { diff --git a/test/adapters.test.ts b/test/adapters.test.ts index 88c834f..79261ee 100644 --- a/test/adapters.test.ts +++ b/test/adapters.test.ts @@ -12,6 +12,7 @@ const baseArgs: AdapterRunArgs = { budgetUsd: 5, timeoutSeconds: 600, workDir: "/tmp/x", + taskDir: "/tmp/task", }; describe("registry", () => { diff --git a/test/parse.test.ts b/test/parse.test.ts index 88129ba..f7a9ef5 100644 --- a/test/parse.test.ts +++ b/test/parse.test.ts @@ -26,6 +26,27 @@ describe("parseJsonEnvelope", () => { const stdout = ["INFO starting", '{"type":"result","ok":true}', "INFO done"].join("\n"); expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", ok: true }); }); + + it("parses a pretty-printed (multi-line) envelope, as gemini-cli emits", () => { + const stdout = [ + "Loaded config", + JSON.stringify( + { type: "result", stats: { models: { m: { tokens: { prompt: 10 } } } } }, + null, + 2, + ), + "done", + ].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ + type: "result", + stats: { models: { m: { tokens: { prompt: 10 } } } }, + }); + }); + + it("is not fooled by braces inside JSON strings", () => { + const stdout = JSON.stringify({ type: "result", note: 'a "}" inside {braces}' }, null, 2); + expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", note: 'a "}" inside {braces}' }); + }); }); describe("num / countOf", () => { diff --git a/test/pipeline.test.ts b/test/pipeline.test.ts index 91fe864..14dbb9a 100644 --- a/test/pipeline.test.ts +++ b/test/pipeline.test.ts @@ -37,30 +37,30 @@ describe("task fixtures", () => { } }); - it("every task discriminates: pristine fails, solution passes", () => { + it("every task discriminates: pristine fails, solution passes", async () => { for (const task of tasks) { const pristineDir = scratch(); - seedWorkspace(task, pristineDir); - const pristine = runVerification(task, pristineDir); + await seedWorkspace(task, pristineDir); + const pristine = await runVerification(task, pristineDir); expect( pristine.passed, `${task.id}: held-out tests must FAIL on the pristine workspace`, ).toBe(false); const solvedDir = scratch(); - seedWorkspace(task, solvedDir); - applySolution(task, solvedDir); - const solved = runVerification(task, solvedDir); + await seedWorkspace(task, solvedDir); + await applySolution(task, solvedDir); + const solved = await runVerification(task, solvedDir); expect(solved.passed, `${task.id}: reference solution must pass:\n${solved.output}`).toBe( true, ); } }, 300_000); - it("verification removes agent-planted .arena-verify content", () => { + it("verification removes agent-planted .arena-verify content", async () => { const task = loadTasks(TASK_ROOT)[0]!; const dir = scratch(); - seedWorkspace(task, dir); + await seedWorkspace(task, dir); // Simulate an adversarial agent planting a trivially-green test suite. const planted = join(dir, ".arena-verify"); mkdirSync(planted, { recursive: true }); @@ -68,7 +68,7 @@ describe("task fixtures", () => { join(planted, "fake.test.mjs"), 'import { test } from "node:test"; test("ok", () => {});', ); - const result = runVerification(task, dir); + const result = await runVerification(task, dir); expect(result.passed).toBe(false); }); }); @@ -119,13 +119,7 @@ describe("full run with mock agents", () => { expect(persisted.results).toHaveLength(results.length); }, 300_000); - it("marks an uninvocable agent as agent-error, not failed", async () => { - const _outDir = scratch(); - const _tasks = loadTasks(TASK_ROOT).slice(0, 1); - - // stella with a bin override pointing at a nonexistent binary would fail - // isAvailable(); instead simulate via a mock whose execute spawn errors by - // overriding bin to a missing path through the real spawn path. + it("surfaces a spawn failure as spawnError (classified agent-error, never a loss)", async () => { const { StellaAdapter } = await import("../src/adapters/stella.js"); const adapter = new StellaAdapter({ bin: "/nonexistent/definitely-missing" }); const outcome = await adapter.execute({ @@ -134,6 +128,7 @@ describe("full run with mock agents", () => { budgetUsd: undefined, timeoutSeconds: 5, workDir: scratch(), + taskDir: scratch(), }); expect(outcome.spawnError).toBeDefined(); }); diff --git a/web/favicon.svg b/web/favicon.svg new file mode 100644 index 0000000..59e163d --- /dev/null +++ b/web/favicon.svg @@ -0,0 +1,11 @@ + + + + + + diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..1921f3d --- /dev/null +++ b/web/index.html @@ -0,0 +1,327 @@ + + + + + + Arena — head-to-head benchmarks for agentic coding CLIs + + + + + + + + + + + + +
+
+ + + ARENA + + +
+
+ +
+
+
+ +

Benchmark harness · Open source · MIT

+

Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.

+

+ Arena runs two or more coding agents — Claude Code, Gemini CLI, Oxagen, + Stella, or your own — on the same tasks with the same model, budget, and timeout. + Held-out tests the agent can never see or author decide every outcome. Each metric + is reported separately, with real statistics and full receipts. +

+
+ View on GitHub + Read the methodology +
+
+
# the only comparison that isolates the harness: same model everywhere
+$ git clone https://github.com/oxageninc/arena && cd arena && pnpm install
+$ pnpm arena run --agents oxagen,claude-code \
+    --model anthropic/claude-sonnet-5 --trials 3 --budget 5
+$ open results/run-*/report.md
+
+
+
+ +
+
+
+

Every number, attacked first

+

Most agent-vs-agent numbers don’t survive five minutes of skeptical reading. + Arena is engineered around the specific ways benchmarks get torn apart.

+
+
+
+
The agent graded itself
+
Verification tests live outside the workspace, are copied in only after the + agent exits, and anything the agent planted at the verify path is deleted + first. CI proves every task’s tests fail on the pristine workspace and pass + on the reference solution.
+
+
+
Different models, budgets, timeouts
+
One config drives every agent. Runs with unmatched models are stamped + matchedModels: false and the report leads with a warning banner.
+
+
+
Your harness broke the competitor
+
A CLI that can’t even be invoked is scored agent-error and + excluded from every comparison — it is never counted as the agent losing.
+
+
+
Token counts aren’t comparable
+
Adapters normalize usage: input never includes cache reads; cache reads and + writes are tracked separately. The rule is documented per adapter.
+
+
+
n = 1, no stats
+
Multiple trials per task–agent pair. Wilson 95% intervals on success rates, + exact McNemar on paired outcomes, seeded paired-bootstrap CIs on wall-clock, + token, and cost deltas. Same raw data + seed ⇒ the same report, bit for bit.
+
+
+
Where’s the raw data?
+
Every trial writes its full transcript, workspace diff, and a manifest with + CLI versions, models, host, seed, and a one-line reproduce command.
+
+
+
+
+ +
+
+
+

What a run reports

+

One row per agent, one number per metric — never blended into a score. + Interval capsules show the 95% Wilson bounds; the tick is the point estimate.

+
+
+
+ results.json → report.md + illustrative output +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AgentTrialsSuccess rate (95% CI)Median wall clockMedian tokensComputed cost
agent-a18 +
+ + 83.3% (60.8–94.2) +
+ 		74.2s148,210$0.4130
agent-b18 +
+ + 61.1% (38.6–79.7) +
+ 		51.9s96,554$0.2871
+
+
+ Exact McNemar on paired outcomes: p = 0.289 — not significant + at α = 0.05. Arena tells you to collect more trials before claiming a + success-rate difference. +
+
+
+
+ +
+
+
+

Anatomy of a trial

+

Agents interleave and their order flips every trial (ABBA); tasks run + sequentially so agents never contend for the host.

+
+
+
+ 01 +

Seed

+

The task’s workspace is copied to a fresh temp directory and committed to a + fresh git repo. Every agent receives the identical prompt.

+
+
+ 02 +

Agent

+

The CLI runs headlessly with the shared model, budget, and timeout. Arguments + are argv arrays — nothing passes through a shell.

+
+
+ 03 +

Diff

+

A git diff against the seed commit captures exactly what the agent changed — + stored as a receipt with the full transcript.

+
+
+ 04 +

Verify

+

The harness deletes anything planted at the verify path, copies the held-out + tests in, and runs them. They were never on disk while the agent ran.

+
+
+ 05 +

Score

+

Passed only if the held-out tests pass within the time budget. A CLI that + couldn’t be invoked is an agent-error — excluded, never a loss.

+
+
+
+
+ +
+
+
+

Supported agents

+

Autonomy levels are matched as closely as each CLI allows — auto-approved + edits, no interactive prompts. Adding your own agent is one ~80-line adapter.

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AdapterInvocationStatus
claude-codeclaude -p --output-format json --permission-mode acceptEditsverified
oxagenoxagen --local --output-format json -- <prompt>verified
stellastella run <prompt> + STELLA_* envverified
geminigemini -p <prompt> --output-format json --approval-mode auto_editenvelope-tested
yoursone small adapter: how to invoke your CLI headlessly, how to map its envelopePRs welcome
+
+
+
+ +
+
+
+

Scale up: SWE-bench Verified via Harbor

+

The built-in suite is a calibration set, not a research benchmark. For + publishable resolve-rate claims, the Harbor adapter runs your agent through the + official, containerized SWE-bench verifier — head-to-head against the agents + Harbor already ships, scored by the same repo-native test suites. Wiring up + your agent is one spec file, no Python.

+
+
+
$ cd harbor && pip install -e .
+$ export ARENA_AGENT_SPEC=$PWD/my-agent.toml
+$ harbor run --agent-import-path arena_harbor:ByoAgent \
+    --dataset swe-bench/swe-bench-verified -m anthropic/claude-sonnet-5
+
+
+
+ +
+
+
+

Before you publish a number

+

The methodology is a contract. If a published claim violates it, the claim — + not the reader — is wrong.

+
+
    +
  • Same model on every agent. The manifest must say + matchedModels: true.
    • +
  • Pre-committed task set and trial count. Link the commit + that fixed them before the run.
    • +
  • Significance on the metric you claim. p < 0.05 on + McNemar, or a bootstrap CI that excludes zero.
    • +
  • The full run directory, published. Manifest, per-trial + JSON, transcripts, diffs, report.
    • +
  • A conflict-of-interest note if you built one of the + agents.
    • +
+
+
+
+ + + + + diff --git a/web/styles.css b/web/styles.css new file mode 100644 index 0000000..3a5e5d6 --- /dev/null +++ b/web/styles.css @@ -0,0 +1,648 @@ +/* Arena — arena.oxagen.sh + All-neutral palette. IBM Plex Sans for prose, IBM Plex Mono for every + number, label, and command. The geometric stadium (rect + semicircular + caps) is the recurring construction: the mark, the pills, the CI capsules. */ + +:root { + --ink: #131312; + --ink-soft: #3a3a37; + --ink-faint: #6b6b67; + --paper: #fcfcfb; + --panel: #f1f1ef; + --hairline: #d8d8d5; + --sans: "IBM Plex Sans", system-ui, -apple-system, "Segoe UI", sans-serif; + --mono: "IBM Plex Mono", ui-monospace, "SFMono-Regular", Menlo, Consolas, monospace; + --measure: 62ch; + --radius-pill: 999px; +} + +@media (prefers-color-scheme: dark) { + :root { + --ink: #e9e9e6; + --ink-soft: #c2c2be; + --ink-faint: #8f8f8a; + --paper: #131312; + --panel: #1d1d1b; + --hairline: #353532; + } +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + *, + *::before, + *::after { + animation: none !important; + transition: none !important; + } +} + +body { + font-family: var(--sans); + font-size: 1.0625rem; + line-height: 1.6; + color: var(--ink); + background: var(--paper); + -webkit-font-smoothing: antialiased; +} + +a { + color: inherit; + text-underline-offset: 3px; + text-decoration-thickness: 1px; +} + +a:focus-visible, +button:focus-visible { + outline: 2px solid var(--ink); + outline-offset: 3px; + border-radius: 2px; +} + +.wrap { + max-width: 68rem; + margin: 0 auto; + padding: 0 1.5rem; +} + +code, +pre, +.mono { + font-family: var(--mono); + font-variant-numeric: tabular-nums; +} + +/* ── header ─────────────────────────────────────────────────────────── */ + +.site-header { + border-bottom: 1px solid var(--hairline); +} + +.site-header .wrap { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + padding-top: 1rem; + padding-bottom: 1rem; +} + +.brand { + display: inline-flex; + align-items: center; + gap: 0.7rem; + text-decoration: none; +} + +.brand svg { + width: 2.1rem; + height: 2.1rem; + display: block; +} + +.brand .wordmark { + font-family: var(--mono); + font-weight: 600; + font-size: 1rem; + letter-spacing: 0.34em; + translate: 0 -1px; +} + +.site-nav { + display: flex; + align-items: center; + gap: 1.4rem; + font-family: var(--mono); + font-size: 0.82rem; +} + +.site-nav a { + text-decoration: none; + color: var(--ink-soft); +} + +.site-nav a:hover { + color: var(--ink); + text-decoration: underline; +} + +@media (max-width: 640px) { + .site-nav a:not(.pill) { + display: none; + } +} + +/* ── stadium pills ──────────────────────────────────────────────────── */ + +.pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + border: 1.5px solid var(--ink); + border-radius: var(--radius-pill); + padding: 0.5rem 1.15rem; + font-family: var(--mono); + font-size: 0.85rem; + font-weight: 500; + text-decoration: none; + color: var(--ink); + background: transparent; + transition: background 120ms ease, color 120ms ease; +} + +.pill:hover { + background: var(--ink); + color: var(--paper); +} + +.pill--solid { + background: var(--ink); + color: var(--paper); +} + +.pill--solid:hover { + background: transparent; + color: var(--ink); +} + +.chip { + display: inline-block; + border: 1px solid var(--hairline); + border-radius: var(--radius-pill); + padding: 0.1rem 0.65rem; + font-family: var(--mono); + font-size: 0.72rem; + color: var(--ink-faint); + white-space: nowrap; +} + +/* ── hero ───────────────────────────────────────────────────────────── */ + +.hero { + padding: 4.5rem 0 4rem; + border-bottom: 1px solid var(--hairline); +} + +.hero-mark { + width: 7.5rem; + height: auto; + display: block; + margin-bottom: 2rem; +} + +.eyebrow { + font-family: var(--mono); + font-size: 0.78rem; + letter-spacing: 0.18em; + text-transform: uppercase; + color: var(--ink-faint); + margin-bottom: 1rem; +} + +.hero h1 { + font-size: clamp(1.9rem, 4.5vw, 3rem); + font-weight: 600; + line-height: 1.15; + letter-spacing: -0.015em; + max-width: 24ch; + margin-bottom: 1.4rem; +} + +.hero .lede { + max-width: var(--measure); + color: var(--ink-soft); + margin-bottom: 2.2rem; +} + +.hero-actions { + display: flex; + flex-wrap: wrap; + gap: 0.8rem; + margin-bottom: 2.6rem; +} + +.terminal { + background: var(--panel); + border: 1px solid var(--hairline); + border-radius: 10px; + padding: 1.1rem 1.3rem; + overflow-x: auto; + max-width: 46rem; +} + +.terminal pre { + font-size: 0.86rem; + line-height: 1.75; + white-space: pre; +} + +.terminal .cmt { + color: var(--ink-faint); +} + +.terminal .prompt { + color: var(--ink-faint); + user-select: none; +} + +/* ── sections ───────────────────────────────────────────────────────── */ + +section { + padding: 4rem 0; + border-bottom: 1px solid var(--hairline); +} + +section:last-of-type { + border-bottom: none; +} + +.section-head { + margin-bottom: 2.4rem; + max-width: var(--measure); +} + +.section-head h2 { + font-size: 1.55rem; + font-weight: 600; + letter-spacing: -0.01em; + margin-bottom: 0.7rem; +} + +.section-head p { + color: var(--ink-soft); +} + +/* attack / defense ledger */ + +.ledger { + display: grid; + gap: 0; + border-top: 1px solid var(--hairline); +} + +.ledger-row { + display: grid; + grid-template-columns: minmax(0, 5fr) minmax(0, 7fr); + gap: 2rem; + padding: 1.35rem 0; + border-bottom: 1px solid var(--hairline); +} + +.ledger-row dt { + font-family: var(--mono); + font-size: 0.9rem; + font-weight: 500; + color: var(--ink); +} + +.ledger-row dt::before { + content: "“"; + color: var(--ink-faint); +} + +.ledger-row dt::after { + content: "”"; + color: var(--ink-faint); +} + +.ledger-row dd { + color: var(--ink-soft); + font-size: 0.98rem; +} + +@media (max-width: 720px) { + .ledger-row { + grid-template-columns: 1fr; + gap: 0.5rem; + } +} + +/* ── specimen report ────────────────────────────────────────────────── */ + +.report-card { + border: 1px solid var(--hairline); + border-radius: 12px; + overflow: hidden; +} + +.report-title { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + flex-wrap: wrap; + padding: 0.85rem 1.25rem; + background: var(--panel); + border-bottom: 1px solid var(--hairline); + font-family: var(--mono); + font-size: 0.8rem; + color: var(--ink-faint); +} + +.report-scroll { + overflow-x: auto; +} + +table.report { + width: 100%; + min-width: 44rem; + border-collapse: collapse; + font-size: 0.92rem; +} + +table.report th { + font-family: var(--mono); + font-size: 0.72rem; + font-weight: 500; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--ink-faint); + text-align: left; + padding: 0.8rem 1.25rem; + border-bottom: 1px solid var(--hairline); + white-space: nowrap; +} + +table.report td { + padding: 1rem 1.25rem; + border-bottom: 1px solid var(--hairline); + vertical-align: middle; + white-space: nowrap; +} + +table.report tr:last-child td { + border-bottom: none; +} + +table.report td.num { + font-family: var(--mono); + font-variant-numeric: tabular-nums; +} + +.agent-name { + font-family: var(--mono); + font-weight: 600; + font-size: 0.9rem; +} + +/* Wilson CI capsule: hairline track 0–100%, ink capsule spanning the CI, + a 2px tick at the point estimate. Values are also printed as text, so the + graphic is never the only encoding. */ + +.ci { + display: flex; + align-items: center; + gap: 0.9rem; +} + +.ci-track { + position: relative; + width: 11rem; + height: 10px; + background: var(--panel); + border: 1px solid var(--hairline); + border-radius: var(--radius-pill); + flex: none; +} + +.ci-band { + position: absolute; + top: 1px; + bottom: 1px; + background: var(--ink-faint); + opacity: 0.45; + border-radius: var(--radius-pill); +} + +.ci-point { + position: absolute; + top: -3px; + bottom: -3px; + width: 3px; + border-radius: 2px; + background: var(--ink); + translate: -50% 0; +} + +.ci-label { + font-family: var(--mono); + font-variant-numeric: tabular-nums; + font-size: 0.88rem; +} + +.ci-label small { + color: var(--ink-faint); + font-size: 0.78rem; +} + +.report-foot { + padding: 0.9rem 1.25rem; + border-top: 1px solid var(--hairline); + font-size: 0.86rem; + color: var(--ink-faint); +} + +.report-foot code { + font-size: 0.82rem; +} + +/* ── trial anatomy ──────────────────────────────────────────────────── */ + +.steps { + display: grid; + grid-template-columns: repeat(5, 1fr); + gap: 1.5rem; + counter-reset: step; +} + +.step { + border-top: 2px solid var(--ink); + padding-top: 1rem; +} + +.step .step-num { + display: inline-block; + border: 1px solid var(--hairline); + border-radius: var(--radius-pill); + padding: 0.05rem 0.6rem; + font-family: var(--mono); + font-size: 0.72rem; + color: var(--ink-faint); + margin-bottom: 0.7rem; +} + +.step h3 { + font-size: 1rem; + font-weight: 600; + margin-bottom: 0.4rem; +} + +.step p { + font-size: 0.88rem; + color: var(--ink-soft); +} + +@media (max-width: 900px) { + .steps { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 560px) { + .steps { + grid-template-columns: 1fr; + } +} + +/* ── agents table ───────────────────────────────────────────────────── */ + +.agents-scroll { + overflow-x: auto; + border: 1px solid var(--hairline); + border-radius: 12px; +} + +table.agents { + width: 100%; + min-width: 40rem; + border-collapse: collapse; + font-size: 0.92rem; +} + +table.agents th { + font-family: var(--mono); + font-size: 0.72rem; + font-weight: 500; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--ink-faint); + text-align: left; + padding: 0.8rem 1.25rem; + border-bottom: 1px solid var(--hairline); + background: var(--panel); +} + +table.agents td { + padding: 0.85rem 1.25rem; + border-bottom: 1px solid var(--hairline); + vertical-align: top; +} + +table.agents tr:last-child td { + border-bottom: none; +} + +table.agents td:first-child { + font-family: var(--mono); + font-weight: 600; + font-size: 0.88rem; + white-space: nowrap; +} + +table.agents td.inv { + font-family: var(--mono); + font-size: 0.82rem; + color: var(--ink-soft); +} + +/* ── checklist ──────────────────────────────────────────────────────── */ + +.checklist { + list-style: none; + max-width: var(--measure); + border-top: 1px solid var(--hairline); +} + +.checklist li { + display: grid; + grid-template-columns: auto 1fr; + gap: 1rem; + align-items: baseline; + padding: 1rem 0; + border-bottom: 1px solid var(--hairline); + color: var(--ink-soft); +} + +.checklist li::before { + content: "☐"; + font-family: var(--mono); + color: var(--ink); + font-size: 1rem; +} + +.checklist strong { + color: var(--ink); + font-weight: 600; +} + +/* ── footer ─────────────────────────────────────────────────────────── */ + +.site-footer { + border-top: 1px solid var(--hairline); + padding: 2.2rem 0 3rem; +} + +.site-footer .wrap { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + gap: 1rem; +} + +.site-footer p, +.site-footer a { + font-family: var(--mono); + font-size: 0.8rem; + color: var(--ink-faint); +} + +.site-footer .tagline { + color: var(--ink-soft); +} + +/* ── load-in (hero only, disabled under reduced motion) ─────────────── */ + +@media (prefers-reduced-motion: no-preference) { + .hero > .wrap > * { + animation: rise 480ms ease both; + } + .hero > .wrap > *:nth-child(2) { + animation-delay: 60ms; + } + .hero > .wrap > *:nth-child(3) { + animation-delay: 120ms; + } + .hero > .wrap > *:nth-child(4) { + animation-delay: 180ms; + } + .hero > .wrap > *:nth-child(5) { + animation-delay: 240ms; + } + .hero > .wrap > *:nth-child(6) { + animation-delay: 300ms; + } +} + +@keyframes rise { + from { + opacity: 0; + translate: 0 10px; + } + to { + opacity: 1; + translate: 0 0; + } +} diff --git a/web/vercel.json b/web/vercel.json new file mode 100644 index 0000000..8e0726c --- /dev/null +++ b/web/vercel.json @@ -0,0 +1,14 @@ +{ + "cleanUrls": true, + "trailingSlash": false, + "headers": [ + { + "source": "/(.*)", + "headers": [ + { "key": "X-Content-Type-Options", "value": "nosniff" }, + { "key": "X-Frame-Options", "value": "DENY" }, + { "key": "Referrer-Policy", "value": "strict-origin-when-cross-origin" } + ] + } + ] +} From f864d11db6e612132ff193bdcc7b1fe3266a6107 Mon Sep 17 00:00:00 2001 From: macanderson Date: Sat, 18 Jul 2026 19:28:55 -0700 Subject: [PATCH 15/18] chore: update GitHub repo URLs from oxageninc to macanderson org (#21) The arena and stella repositories moved from the oxageninc org to the macanderson account; point all badges, links, and package metadata at the new home. --- .github/ISSUE_TEMPLATE/config.yml | 2 +- CHANGELOG.md | 4 ++-- CONTRIBUTING.md | 4 ++-- README.md | 10 +++++----- SECURITY.md | 2 +- harbor/pyproject.toml | 4 ++-- package.json | 2 +- web/index.html | 10 +++++----- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 0ee3cf4..ec8d3ec 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Security vulnerability (private report) - url: https://github.com/oxageninc/arena/security/advisories/new + url: https://github.com/macanderson/arena/security/advisories/new about: Report security issues privately via GitHub Security Advisories — do NOT open a public issue. See SECURITY.md. diff --git a/CHANGELOG.md b/CHANGELOG.md index 81d2219..365d5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,5 +52,5 @@ CLIs, engineered to survive scrutiny. Dependabot, and a dependency-review gate that blocks known-vulnerable PRs. - `SECURITY.md` with a private vulnerability reporting channel. -[Unreleased]: https://github.com/oxageninc/arena/compare/5122005...HEAD -[0.1.0]: https://github.com/oxageninc/arena/releases/tag/v0.1.0 +[Unreleased]: https://github.com/macanderson/arena/compare/5122005...HEAD +[0.1.0]: https://github.com/macanderson/arena/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2360e59..c699f7d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ each undermine the one thing the project exists to provide. ## Before you write code -- **Search [open issues](https://github.com/oxageninc/arena/issues) first** — +- **Search [open issues](https://github.com/macanderson/arena/issues) first** — someone may already be on it. - For anything non-trivial (new adapter, new task, stat/report change), **open an issue** and sketch the approach before investing in code. A 5-minute @@ -21,7 +21,7 @@ each undermine the one thing the project exists to provide. Arena needs **Node 22+** and **pnpm** (the CI pins pnpm 11): ```bash -git clone https://github.com/oxageninc/arena +git clone https://github.com/macanderson/arena cd arena pnpm install pnpm typecheck diff --git a/README.md b/README.md index e736e4f..d725e70 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,16 @@ # Arena -[![CI](https://github.com/oxageninc/arena/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/oxageninc/arena/actions/workflows/ci.yml) +[![CI](https://github.com/macanderson/arena/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/macanderson/arena/actions/workflows/ci.yml) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/oxageninc/arena/badge)](https://scorecard.dev/viewer/?uri=github.com/oxageninc/arena) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/macanderson/arena/badge)](https://scorecard.dev/viewer/?uri=github.com/macanderson/arena) **Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** · [arena.oxagen.sh](https://arena.oxagen.sh) -Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/oxageninc/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. +Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. ```bash -git clone https://github.com/oxageninc/arena +git clone https://github.com/macanderson/arena cd arena && pnpm install pnpm arena doctor # which agent CLIs are installed? @@ -139,7 +139,7 @@ Contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) (adding an adapter or task fixture is a great first PR). We also have a [Code of Conduct](.github/CODE_OF_CONDUCT.md) and a [changelog](CHANGELOG.md). Report security issues privately via the repo's -[Security tab](https://github.com/oxageninc/arena/security/advisories/new) — not +[Security tab](https://github.com/macanderson/arena/security/advisories/new) — not as a public issue. MIT © Mac Anderson diff --git a/SECURITY.md b/SECURITY.md index 9b3fe59..cea5ebc 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Please **do not** open a public issue for security vulnerabilities. -Report privately via GitHub's [**Report a vulnerability**](https://github.com/oxageninc/arena/security/advisories/new) button on the repository's Security tab (Security → Advisories → Report a vulnerability). This opens a private channel with the maintainers. +Report privately via GitHub's [**Report a vulnerability**](https://github.com/macanderson/arena/security/advisories/new) button on the repository's Security tab (Security → Advisories → Report a vulnerability). This opens a private channel with the maintainers. We aim to acknowledge reports within 3 business days and to ship a fix or mitigation for confirmed high-severity issues as quickly as is practical. We'll coordinate a disclosure timeline with you and credit you in the advisory unless you prefer otherwise. diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index f4d0907..de11f10 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -29,8 +29,8 @@ dependencies = [ dev = ["pytest>=7.0", "pytest-asyncio>=0.21", "ruff>=0.8"] [project.urls] -Homepage = "https://github.com/oxageninc/arena" -Repository = "https://github.com/oxageninc/arena" +Homepage = "https://github.com/macanderson/arena" +Repository = "https://github.com/macanderson/arena" [tool.setuptools] packages = ["arena_harbor"] diff --git a/package.json b/package.json index d89b2eb..33d714e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "type": "module", "repository": { "type": "git", - "url": "https://github.com/oxageninc/arena.git" + "url": "https://github.com/macanderson/arena.git" }, "keywords": [ "benchmark", diff --git a/web/index.html b/web/index.html index 1921f3d..ba595f0 100644 --- a/web/index.html +++ b/web/index.html @@ -32,7 +32,7 @@ Method Agents Harbor - GitHub + GitHub @@ -54,12 +54,12 @@

Head-to-head benchmarks for agentic coding CLIs — built to survive scrutin is reported separately, with real statistics and full receipts.

- View on GitHub - Read the methodology + View on GitHub + Read the methodology 
# the only comparison that isolates the harness: same model everywhere
-$ git clone https://github.com/oxageninc/arena && cd arena && pnpm install
+$ git clone https://github.com/macanderson/arena && cd arena && pnpm install
 $ pnpm arena run --agents oxagen,claude-code \
     --model anthropic/claude-sonnet-5 --trials 3 --budget 5
 $ open results/run-*/report.md
@@ -317,7 +317,7 @@

Before you publish a number

Same model. Same budget. Same timeout. Full receipts.

- github.com/oxageninc/arena + github.com/macanderson/arena · MIT · an Oxagen project

From de7d7d2a404d8808c034b5c19429e4ba1fd447d9 Mon Sep 17 00:00:00 2001 From: macanderson Date: Sat, 18 Jul 2026 21:53:47 -0700 Subject: [PATCH 16/18] Verification integrity fixes, arena.oxagen.sh site, ABP spec + research paper (#22) * fix: verification integrity, metric honesty, and CLI hardening Highest-severity first: - Timeouts kill the agent's whole process tree (POSIX process groups); trials can no longer hang on pipes held by orphaned grandchildren, and survivors can no longer tamper with held-out verification. - Zero-token (unparseable-envelope) trials are excluded from token/cost medians and deltas instead of dragging them toward zero past the gate. - claude-code preserves full model ids (no floating "sonnet" alias), so version pinning, matchedModels, and pricing lookups hold. - Gemini normalization counts thoughts (reasoning) and tool tokens. - Pretty-printed JSON envelopes parse (TS + Harbor Python mirror). - Per-trial error containment + incremental results.json; wall clock measures spawn-to-exit only; git-diff failure is "unknown", never "empty"; report table shares perAgentSummary with the gate (no NaN). - CLI: -o short flag (as documented), strict numeric flags, unknown verify ids error, duplicate agent specs rejected, typo'd gate thresholds error instead of silently disabling the check. - Cost with unpriced cache writes is null, never guessed. - Harbor: float-tolerant ARENA_TIMEOUT (never silently unbounded), loud empty-{budget} failure, metrics.kind validated at load. * feat(site): arena.oxagen.sh landing page + Agent Benchmark Protocol draft Static site (site/) for arena.oxagen.sh: landing page with the matched-model scoreboard, and the ABP design spec. Spec source lives at docs/agent-benchmark-protocol.md: four interchange schemas (task, run, trace, verdict), the live GitHub-issue task pipeline with golden traces, smallest-normalized-diff scoring, conformance levels, and roadmap. * docs: The State of Agent Benchmarking (paper) + site rendering Research paper surveying the July 2026 agent-benchmark landscape through the engine-vs-model lens: 28 cited references, gap analysis of four capabilities (matched-model isolation, efficiency grading, statistical CI gating, live tasks with golden traces), COI disclosed. Markdown source in docs/, rendered at arena.oxagen.sh/paper/. README links the site, paper, and spec. * docs: architecture map (module graph, trial data flow, contracts) --- CHANGELOG.md | 40 ++ README.md | 2 + docs/ARCHITECTURE.md | 62 +++ docs/agent-benchmark-protocol.md | 227 +++++++++ docs/agent-engine-benchmarks-2026.md | 114 +++++ harbor/arena_harbor/base.py | 21 +- harbor/arena_harbor/command.py | 7 + harbor/arena_harbor/metrics.py | 77 ++- harbor/pyproject.toml | 3 +- site/.gitignore | 2 + site/index.html | 194 ++++++++ site/paper/index.html | 335 +++++++++++++ site/spec/index.html | 351 ++++++++++++++ site/style.css | 691 +++++++++++++++++++++++++++ site/vercel.json | 14 + src/adapters/base.ts | 52 +- src/adapters/claude-code.ts | 12 +- src/adapters/gemini.ts | 16 +- src/cli.ts | 44 +- src/orchestrator.ts | 73 ++- src/parse.ts | 48 +- src/pricing.ts | 7 +- src/report.ts | 15 +- src/summary.ts | 13 +- src/workspace.ts | 17 +- test/adapters.test.ts | 26 +- test/parse.test.ts | 33 ++ 27 files changed, 2413 insertions(+), 83 deletions(-) create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/agent-benchmark-protocol.md create mode 100644 docs/agent-engine-benchmarks-2026.md create mode 100644 site/.gitignore create mode 100644 site/index.html create mode 100644 site/paper/index.html create mode 100644 site/spec/index.html create mode 100644 site/style.css create mode 100644 site/vercel.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 365d5a9..cfd1ffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,46 @@ minor bumps may include breaking changes). - Open-source launch artifacts: `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, issue and PR templates, and this changelog. - `.stella/` agent caches are now gitignored. +- **Site** (`site/`) — arena.oxagen.sh: landing page, the research paper + *The State of Agent Benchmarking*, and the draft **Agent Benchmark Protocol** + spec (`docs/agent-benchmark-protocol.md`, + `docs/agent-engine-benchmarks-2026.md`). +- Pretty-printed (multi-line) JSON envelopes are now parsed — both in the TS + harness (`parseJsonEnvelope`) and the Harbor adapter (`last_json_object`). +- `arena run -o ` short flag (the form the README documents). + +### Fixed +- **Timeouts kill the agent's whole process tree** (POSIX process groups), and + a trial can no longer hang on stdio pipes held open by orphaned + grandchildren — which could previously also outlive the agent and tamper + with verification. +- **Zero-token trials no longer poison medians or the gate**: scored trials + whose envelope had no parseable usage are excluded from token/cost medians + and deltas (reported separately), instead of dragging them toward zero. +- **claude-code model pinning**: `anthropic/claude-sonnet-5` now resolves to + `claude-sonnet-5` instead of the floating `sonnet` alias, preserving version + pinning, `matchedModels`, and pricing lookups. +- **Gemini token normalization** counts `thoughts` (reasoning) and `tool` + tokens; both were previously dropped, understating Gemini usage. +- Per-trial error containment: a harness-side failure (git/FS error) scores + that one trial `agent-error` instead of aborting the run; `results.json` is + rewritten after every trial so a crash never loses completed trials. +- Wall clock now measures spawn-to-exit only (workspace seeding excluded), + matching METHODOLOGY.md. +- `arena verify ` errors instead of vacuously passing; numeric + CLI flags are validated (`--timeout 10m` errors instead of parsing as 10); + a typo'd gate threshold errors instead of silently disabling the check; + duplicate `--agents` specs are rejected (their trial ids would collide). +- `git diff` failures are now distinguished from an empty diff, so a trial + with real changes can no longer be misclassified `agent-error`. +- Cache-write tokens with no `cacheWritePerM` price now yield a null cost + (never guessed with the input rate). +- Report per-agent table shares `perAgentSummary` with the baseline/gate (no + more NaN rows when every trial of an agent errored). +- Harbor adapter: non-integer `ARENA_TIMEOUT` falls back to 1800s with a + warning instead of silently disabling the container timeout; an empty + `{budget}` in a run template fails loudly; unknown `metrics.kind` values are + rejected at spec load. ## [0.1.0] — initial release diff --git a/README.md b/README.md index d725e70..1c0dcac 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ **Head-to-head benchmarks for agentic coding CLIs — built to survive scrutiny.** · [arena.oxagen.sh](https://arena.oxagen.sh) +Paper: [The State of Agent Benchmarking](docs/agent-engine-benchmarks-2026.md) · Spec: [Agent Benchmark Protocol (draft)](docs/agent-benchmark-protocol.md) + Arena runs two or more coding agents (Claude Code, Gemini CLI, [Oxagen](https://oxagen.sh), [Stella](https://github.com/macanderson/stella), or your own) on the **same tasks, same model, same budget, same timeout**, grades them with **held-out tests the agent can never see or author**, and reports each metric separately with real statistics and full receipts. ```bash diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..65f4fe3 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,62 @@ +# Arena architecture + +Two subsystems share one repo: a TypeScript CLI harness (`src/`) and a Python Harbor adapter (`harbor/`). They share no code, only a token-normalization convention: `input` never includes cache reads. + +```mermaid +graph TD + subgraph CLI["TypeScript harness"] + cli["cli.ts (entry / dispatch)"] + cli --> c_run["run"] & c_verify["verify"] & c_gate["gate / baseline"] & c_report["report"] & c_list["list / doctor"] + + subgraph CORE["core pipeline"] + orch["orchestrator.ts executeRun/runOne"] + ws["workspace.ts seed/diff/verify"] + adp["adapters/ claude-code · gemini · oxagen · stella · mock"] + parse["parse.ts envelope/diff parsing"] + pricing["pricing.ts + pricing.json"] + end + + stats["stats.ts wilson/mcnemar/bootstrap"] + summary["summary.ts perAgentSummary"] + report["report.ts"] + baseline["baseline.ts gate/baseline"] + tasksdir["tasks// task.json + workspace + verify + solution"] + + c_run --> orch + c_verify --> ws + c_report --> report + c_gate --> baseline + orch --> ws & adp & parse & pricing + ws --> tasksdir + report --> stats & summary + baseline --> summary + summary --> stats + end + + subgraph HARBOR["harbor/ Python adapter (separate process)"] + h_agents["agents.py Byo/Oxagen/StellaAgent"] + h_base["base.py ArenaInstalledAgent"] + h_spec["spec.py AgentSpec (TOML/JSON)"] + h_cmd["command.py build_command"] + h_metrics["metrics.py extract_metrics"] + HarborFW["Harbor framework (Docker verifier)"] + h_agents --> h_base --> h_cmd & h_metrics & h_spec + h_base --> HarborFW + end + + CLI -. "token normalization convention only" .- HARBOR +``` + +## One trial, end to end + +1. `cli.ts:cmdRun` parses argv into a `RunConfig`, loads tasks. +2. `orchestrator.ts:executeRun` creates the run dir, checks each adapter's availability, writes `manifest.json`, then loops trials × tasks × agents with ABBA order flipping. +3. `runOne`: `seedWorkspace` copies the fixture into a temp dir and git-commits the seed → `adapter.execute` spawns the CLI (own process group, SIGKILL tree on timeout) → `collectDiff` captures exactly what the agent changed → `parseEnvelope` normalizes tokens → held-out `runVerification` (wipe `.arena-verify/`, copy tests in, `node --test`, wipe again) → `TrialResult` written; `results.json` rewritten every trial. +4. `report.ts` renders per-agent summaries (via `summary.ts`, the same aggregation the gate uses), pairwise McNemar and bootstrap deltas, the per-task matrix, and receipts. +5. `baseline.ts` snapshots a run (`baseline save`) and gates later runs (`gate`), refusing task-set mismatches and, with `--require-significant`, only failing accuracy drops that clear the 95% CIs. + +## Contracts + +- **Adapter** (`src/adapters/base.ts`): implement `name`, `defaultBinary`, `buildArgs(args)` (argv array, never shell), and `parseEnvelope(stdout)` returning normalized tokens (`input` excludes cache reads; use `totalize`/`emptyEnvelope`). Optional overrides: `resolveModel`, `env`, `execute`, `isAvailable`/`version`. Register in `src/adapters/index.ts`. +- **Task fixture** (`tasks//`): `task.json` (`id` must equal the dir name), `workspace/` (what the agent sees), `verify/` (held-out `node:test` suite, never on disk during the run), `solution/` (reference proving solvability). `arena verify` enforces: pristine fails, solution passes. +- **Harbor spec** (`harbor/arena_harbor/spec.py`): a TOML/JSON file with `name`, `binary`, `run_template` (`{bin} {model} {budget} {timeout} {instruction}` placeholders; instruction shell-quoted for you), plus install and metrics blocks. Point `ARENA_AGENT_SPEC` at it; no Python needed. diff --git a/docs/agent-benchmark-protocol.md b/docs/agent-benchmark-protocol.md new file mode 100644 index 0000000..7d26e68 --- /dev/null +++ b/docs/agent-benchmark-protocol.md @@ -0,0 +1,227 @@ +# Agent Benchmark Protocol (ABP) + +**Draft 0.1 · July 2026 · Oxagen** + +An open standard for benchmarking agent engines: the harness, prompts, tools, and loop an agent ships with, measured separately from the model it happens to run on. + +Status: draft for public comment. The reference implementation is [Arena](https://github.com/macanderson/arena). Nothing here is final, including the name. + +--- + +## 1. The problem this standard exists to solve + +Teams that ship agents change their engine every week: a new system prompt, a new tool, a reworked planning loop, a different context strategy. None of those changes show up on a model leaderboard, because model leaderboards hold the harness fixed and vary the model. The people who build agent engines have the opposite need: hold the model fixed and vary the engine. + +Today that comparison is nearly impossible to run honestly: + +1. Every harness invokes agents differently, so "same task" rarely means same prompt, same budget, same timeout, or same autonomy level. +2. Every harness logs differently, so traces cannot be compared across engines. +3. Every harness grades differently, and most let the agent's own output influence the grade. +4. Almost no benchmark measures efficiency. Two agents that both fix a bug are scored as equals even when one shipped a 4-line patch in 90 seconds and the other shipped a 400-line rewrite in 20 minutes. +5. Almost no benchmark runs in CI. A benchmark you run once for a launch tweet cannot catch the Tuesday your agent quietly got worse. + +ABP standardizes the four interfaces that make honest engine comparison possible: how a task is packaged, how a run is configured, how a trace is recorded, and how a verdict is reported. Any engine that speaks these formats can be benchmarked by any conforming harness, and any harness can be audited by anyone. + +## 2. Why "protocol" + +We considered "standard", "format", "spec", and "benchmark". Protocol is the right word for the same reason it was right for the Model Context Protocol: the deliverable is a set of interchange contracts between independent parties (task authors, engines, harnesses, graders, and auditors), not a single dataset or a single leaderboard. Datasets and leaderboards are built on top of ABP; they are products, and ABP is the plumbing. It also keeps the family resemblance with the Open Context Protocol, which Oxagen authored and built [Stella](https://github.com/macanderson/stella) on. + +One naming caution, stated openly: "protocol" sets an expectation of wire-level rigor. This draft earns that word only in Sections 5 through 8, where the schemas live. If the community reads those sections and concludes the word oversells, the fallback name is the Agent Benchmark Format (ABF). The acronym to protect is ABP, one syllable per letter, easy to say in CI logs: `abp gate failed`. + +## 3. Design principles + +These are inherited from Arena's methodology and are non-negotiable for conformance: + +1. **The engine is the unit under test.** The model, provider, budget, and timeout are pinned inputs. Runs that vary the model are stamped as such and cannot back engine-vs-engine claims. +2. **The agent never grades itself.** Verification material must be absent from the workspace while the agent runs, injected only after it exits, and immune to anything the agent planted. +3. **Every number carries its uncertainty.** Success rates get Wilson intervals. Paired comparisons get exact McNemar. Continuous metrics get seeded bootstrap CIs. No blended scores, ever. +4. **Harness failures are not agent failures.** An engine that could not be invoked is excluded from comparison, not counted as a loss. +5. **Receipts or it did not happen.** A conforming result ships its manifest, transcripts, diffs, and a one-line reproduce command. +6. **Efficiency is a first-class metric.** Resolution alone is a tie. Diff size, tokens, wall clock, and cost break it. + +## 4. The live task pipeline + +This is the part of ABP that no existing benchmark provides end to end. + +### 4.1 Sourcing + +A curated registry of the top 500 open-source repositories on GitHub, filtered for: OSI license, active maintainership (merged PR in the last 30 days), a runnable test suite, and issue hygiene (reproducible reports, labeled backlog). The registry is versioned and published; entries rotate quarterly so the pool tracks the real ecosystem. + +From the registry, tasks are drawn by seeded random selection: pick a repo, pick an open issue from its backlog that passes eligibility (reproducible, in-scope for a code change, not a question or a feature debate). The seed is published with every draw, so the selection itself is auditable. + +### 4.2 Solving and promotion + +A frontier engine (or a human, or both) builds a fix for the issue against the repo's contribution rules and submits it upstream. The fix is **verified** when the project's own CI passes and a maintainer merges it into the production branch. Maintainer merge is the ground truth no synthetic benchmark has: a real reviewer accepted this change into a real codebase. + +Two artifacts fall out of every verified fix: + +- **Training data.** The (issue, repo state, verified diff) triple. +- **A golden trace.** The full ABP trace (Section 6) the solving engine printed along the way: every tool call, every file read, every edit, every token count, timestamped. + +### 4.3 Replay and grading + +Once a task is verified, it freezes: repo pinned at the pre-fix commit, issue text captured, and a verification bundle built from the repo's own test suite plus the tests the verified fix added (the FAIL_TO_PASS set). Other engines then attempt the same frozen task under matched config. They are graded on: + +1. **Resolution.** The held-out verification bundle passes. +2. **Efficiency, in strict order:** smaller normalized git diff wins, then fewer tokens, then less wall clock, then lower cost. + +The golden trace is published as evidence and as a study aid, not as the rubric. Version 1 deliberately does not grade trace similarity: an engine that reaches a passing, smaller fix by a different route beats the golden trace's author. Judging architectural quality and trajectory quality is a stated version 2 direction, and it needs its own adversarial review before it can be a metric. + +### 4.4 Contamination control + +Live sourcing is also the contamination story. Issues are selected from the current backlog, after every candidate model's training cutoff, and each task carries a `sourcedAt` timestamp plus the model cutoffs it was verified against. Tasks age out of the scored set once their fix has been public for 180 days; they remain available as a practice set. Frozen-set benchmarks decay into training data. A pipeline that keeps drawing from the living backlog does not. + +### 4.5 Known objections, answered in the design + +- **"Smallest diff is gameable."** Diff size is measured on a normalized form: formatting-only changes are canonicalized before counting, generated files and lockfiles are excluded, and test files are counted separately (deleting or weakening tests disqualifies the run, it does not shrink the diff). Resolution is still the gate; diff size only ranks engines that already passed. And the metric is honest about what it is: a proxy for surgical precision, not for design quality. Arena publishes the raw diffs, so anyone can audit whether small meant surgical or small meant degenerate. +- **"Maintainers did not consent to being a benchmark."** Sourcing follows each repo's contribution guidelines, fixes are submitted as normal PRs with disclosure, registry entries can opt out, and replay runs happen on frozen clones, never against the live repo. +- **"Issue selection will favor easy tasks."** Selection is seeded-random within eligibility, the seed is published, and difficulty is stratified in reporting (task metadata records diff size and files touched of the verified fix). + +## 5. Task manifest (`abp-task/v1`) + +```jsonc +{ + "schema": "abp-task/v1", + "id": "gh-vercel-next.js-81234", + "source": { + "kind": "github-issue", // or "fixture" for synthetic tasks + "repo": "vercel/next.js", + "issue": 81234, + "baseCommit": "9f2c41d…", // repo frozen here (pre-fix) + "sourcedAt": "2026-07-02T14:11:09Z", + "registryVersion": "abp-registry/2026Q3", + "drawSeed": 424242 + }, + "prompt": "…full task statement given to every engine…", + "environment": { + "image": "ghcr.io/macanderson/abp-node:22", // container digest-pinned + "setup": ["pnpm install --frozen-lockfile"] + }, + "verification": { + "kind": "commands", + "failToPass": ["pnpm test -- test/router/edge-case.test.ts"], + "passToPass": ["pnpm test -- test/router/"], + "timeoutSeconds": 900, + "heldOut": true // must not be on disk during the run + }, + "golden": { + "diffBytesNormalized": 1843, + "filesTouched": 2, + "trace": "traces/gh-vercel-next.js-81234.abp-trace.jsonl", + "mergedPr": "https://github.com/vercel/next.js/pull/81301" + }, + "cutoffsVerified": ["claude-sonnet-5:2026-01", "gpt-5.2:2026-03"] +} +``` + +Synthetic fixtures (like Arena's built-in tasks) use the same schema with `source.kind: "fixture"` and no `golden` block. A harness must treat both identically at run time. + +## 6. Trace format (`abp-trace/v1`) + +One JSONL file per (task, engine, trial). Every line is an event: + +```jsonc +{"t":"2026-07-02T14:12:01.402Z","seq":1,"kind":"run.start","engine":"oxagen/1.4.2","model":"anthropic/claude-sonnet-5","task":"gh-vercel-next.js-81234","config":{"budgetUsd":5,"timeoutSeconds":1200}} +{"t":"…","seq":2,"kind":"model.call","tokens":{"input":1204,"output":388,"cacheRead":9120,"cacheWrite":0}} +{"t":"…","seq":3,"kind":"tool.call","tool":"read_file","args":{"path":"src/router/match.ts"},"durationMs":12} +{"t":"…","seq":4,"kind":"tool.call","tool":"edit_file","args":{"path":"src/router/match.ts"},"diffBytes":412} +{"t":"…","seq":5,"kind":"subagent.start","id":"sa-1","purpose":"run tests"} +{"t":"…","seq":6,"kind":"run.end","outcome":"resolved","wallClockMs":184201,"tokens":{"input":48210,"output":9114,"cacheRead":301200,"cacheWrite":8100},"diffBytesNormalized":1610,"filesTouched":2} +``` + +Rules: + +- `seq` is a strictly increasing integer; `t` is RFC 3339 UTC. Together they make traces diffable and mergeable. +- `tool.call` events record the tool name and a redacted argument summary, never raw secrets. A published redaction list is part of conformance. +- Multi-agent engines emit `subagent.start` / `subagent.end` with nested attribution, so a fan-out engine's token bill is visible per branch. Multi-model engines record `model` per `model.call`. This is how ABP stays meaningful for orchestrators, not just single-loop CLIs. +- Token counts follow Arena's normalization: `input` never includes cache reads; cache reads and writes are tracked separately. +- An engine that cannot emit ABP traces natively can ship a sidecar translator; the harness records which path produced the trace. + +## 7. Run configuration (`abp-run/v1`) + +The manifest a harness must write before the first trial: + +```jsonc +{ + "schema": "abp-run/v1", + "engines": [{"name": "oxagen", "version": "1.4.2"}, {"name": "claude-code", "version": "2.1.0"}], + "model": "anthropic/claude-sonnet-5", + "matchedModels": true, + "budgetUsd": 5, + "timeoutSeconds": 1200, + "trials": 3, + "ordering": "abba", + "seed": 20260718, + "tasks": ["gh-vercel-next.js-81234", "…"], + "host": {"os": "linux", "arch": "arm64", "containerized": true}, + "reproduce": "abp run --config run.json" +} +``` + +`matchedModels: false` runs are legal (you may want to test your engine across models) but a conforming report must lead with the warning and must not present engine-vs-engine conclusions from them. + +## 8. Verdict format (`abp-verdict/v1`) + +```jsonc +{ + "schema": "abp-verdict/v1", + "run": "run-20260718-a41c", + "perEngine": [{ + "engine": "oxagen", + "scoredTrials": 72, + "excludedEngineErrors": 1, + "resolveRate": {"point": 0.84, "wilson95": [0.71, 0.92]}, + "medianDiffBytesNormalized": 1650, + "medianTokensTotal": 361000, + "medianWallClockMs": 190334, + "medianComputedUsd": 1.42 + }], + "pairwise": [{ + "a": "oxagen", "b": "claude-code", + "mcnemar": {"discordant": [14, 5], "pExact": 0.041}, + "diffDelta": {"relMedian": -0.22, "bootstrap95": [-0.31, -0.09], "seed": 20260718} + }], + "receipts": {"trials": "trials/", "transcripts": "transcripts/", "diffs": "diffs/", "traces": "traces/"} +} +``` + +## 9. Conformance levels + +- **ABP-core.** The harness consumes `abp-task/v1`, enforces held-out verification and matched config, and emits `abp-run/v1` + `abp-verdict/v1` with the required statistics. Arena today is one file-format migration away from ABP-core. +- **ABP-traces.** Everything in core, plus `abp-trace/v1` emission for every trial, with subagent and multi-model attribution. +- **ABP-live.** Everything in traces, plus participation in the live task pipeline: consuming registry draws, contributing verified fixes, and publishing golden traces. + +Conformance is claimed by shipping a passing run of the public conformance suite (a set of adversarial fixtures: a task that tries to plant verification files, an envelope that lies about tokens, a trial that must be scored engine-error) and linking the receipts. + +## 10. CI integration + +The reason ABP should win: it is the only benchmark designed to be run on every pull request, not once per launch. + +```yaml +# .github/workflows/agent-quality.yml +- run: abp run --config abp-run.json -o results +- run: abp gate results/run-latest --baseline abp-baseline.json --require-significant +``` + +The gate semantics are Arena's, generalized: fail on a statistically significant resolve-rate drop, or on median token, cost, diff-size, or wall-clock growth past committed thresholds; refuse to compare across mismatched task sets. The baseline is a committed JSON artifact, so the PR that degrades the agent goes red the same day, with receipts attached. + +## 11. Roadmap + +| Phase | Deliverable | Exit criterion | +|---|---|---| +| 0 (now) | Arena as reference harness; this draft public at arena.oxagen.sh | Draft survives public review | +| 1 | Schemas frozen at v1; Arena emits and consumes all four formats; conformance suite published | Two non-Oxagen engines emit valid traces | +| 2 | Live pipeline pilot: 25-repo registry subset, 100 verified tasks, golden traces published | External team reproduces a published verdict from receipts alone | +| 3 | Registry at 500 repos; hosted leaderboard with matched-model brackets; GitHub Action in the marketplace | ABP gate running in 100 external repos' CI | +| 4 | Version 2 exploration: trajectory quality, architectural review, multi-agent orchestration scoring | Community RFC process | + +## 12. Open questions + +1. Trace redaction: how much tool-argument detail can be published from runs on private code without leaking it? Current answer: conformance requires the redaction list, and private runs may withhold traces while still claiming ABP-core. +2. Who signs verified tasks? A task's freeze bundle should be content-addressed and signed by the registry, or a poisoned mirror can grade dishonestly. +3. Diff normalization is specified per-language (formatter canonicalization). The v1 scope is the languages with deterministic formatters; the escape hatch is byte counts on `git diff --numstat` with published exclusions. +4. Should golden traces ever become the rubric? Version 1 says no. If v2 explores it, trace-similarity grading must never punish a better route to a passing fix. + +--- + +*Feedback: open an issue at [github.com/macanderson/arena](https://github.com/macanderson/arena/issues) with the `abp` label.* diff --git a/docs/agent-engine-benchmarks-2026.md b/docs/agent-engine-benchmarks-2026.md new file mode 100644 index 0000000..ec55bc2 --- /dev/null +++ b/docs/agent-engine-benchmarks-2026.md @@ -0,0 +1,114 @@ +# The State of Agent Benchmarking + +## Everyone measures the model. Almost nobody measures the engine you ship. + +**Mac Anderson (Oxagen) · July 2026 · v1.0** + +*Conflict of interest, stated up front: the author builds Arena, an open-source agent-engine benchmark harness, and the Oxagen and Stella agents that appear in its examples. This paper argues a position. Every load-bearing claim carries a citation, and the gap analysis in Section 6 names the systems that already do parts of what Arena does.* + +--- + +## Abstract + +Teams that ship AI agents change their engine every week: prompts, tools, planning loops, context strategies, and orchestration. The model changes a few times a year. Yet nearly every public benchmark ranks models with the harness held fixed, which is the exact inverse of the question a working team needs answered: did my last change make my agent better or worse, on the model I already run? Recent controlled evidence says this inversion is not cosmetic. In a 3x3 factorial study on SWE-bench Verified, harness-induced variance exceeded model-induced variance by 7.8x, and model rankings reversed in six of nine harness pairings [3]. Two 2026 surveys independently name model-vs-harness conflation as an open methodological gap [1, 2]. This paper surveys the benchmark landscape as of July 2026 through that lens: what each system actually measures, whether it can isolate the engine from the model, whether it grades efficiency, whether it can run in CI as a regression gate, and whether its tasks stay fresh. We find that no existing system combines matched-model engine isolation, efficiency grading (diff size, tokens, wall clock, cost), statistically honest CI gating, and live task sourcing with verified fixes as reference traces. We close by describing that missing system and an interchange standard, the Agent Benchmark Protocol, that would let any harness provide it. + +## 1. The question benchmarks answer, and the question teams ask + +A model leaderboard answers: given a fixed scaffold, which model scores highest? That is the right question for a lab choosing a base model, and the wrong question for everyone downstream. Downstream, the model is a config value. The product is the engine: the system prompt, the tool set, the retrieval and context policy, the planning loop, the subagent topology, and the guardrails. One survey puts the distinction plainly: evaluating the LLM alone is like examining an engine on a stand, while agent evaluation must assess the whole car under driving conditions [2]. + +The scale of what the scaffold contributes has been measured repeatedly. SWE-agent showed in 2024 that the agent-computer interface alone moves resolve rates at a fixed model [24]. Agentless showed the same year that a deliberately non-agentic pipeline could outperform elaborate agents at a fraction of the cost [25]. And in 2026 a controlled factorial study quantified it: across three models and three harness configurations on a 100-task SWE-bench Verified subset, the harness contributed 18.48 pp² of score variance against the model's 2.37 pp², a ratio of 7.8x, with six model-ranking reversals across nine harness pairings [3]. The authors' conclusion is the thesis of this paper: the execution harness, the infrastructure layer that governs context construction, tool interaction, orchestration, and verification, is often a stronger determinant of agent performance than the model it wraps [3]. + +The benchmark ecosystem has not caught up. The most comprehensive survey of agent evaluation to date lists "Decoupling LLM & Harness Evaluation" as future work: most current benchmarks conflate the two targets, and no established benchmark provides controlled protocols that vary each factor independently [1]. The same survey finds cost and efficiency metrics broadly neglected, echoing the argument of "AI Agents That Matter" that accuracy-only leaderboards drive needlessly costly agents [1, 23]. + +## 2. Survey: coding-agent benchmarks + +**SWE-bench** (Princeton/Stanford, 2023) established the template: 2,294 real GitHub issues from 12 Python repos, graded by the repo's own FAIL_TO_PASS and PASS_TO_PASS tests [4]. **SWE-bench Verified** (OpenAI, 2024) human-filtered 500 instances to remove impossible or underspecified tasks [5]. **SWE-bench Multimodal** adds visual, JavaScript-centric issues; **SWE-bench Pro** (Scale AI, 2025) raises difficulty with long-horizon, enterprise-style tasks and a held-out commercial subset [6]. **SWE-bench Live** (Microsoft Research and community, 2025) attacks contamination directly with a monthly-refreshed feed of new issues [7]. All of these grade one thing: did the repo's tests pass. None grades diff size, token spend, or wall clock as a ranked metric, and their leaderboards mix (model, scaffold) pairs, so a score is attributable to neither alone. They are, however, the raw material engine benchmarking needs: real repos, real issues, executable verification. + +**Terminal-Bench** (Stanford and the Laude Institute, 2025) measures terminal-native task completion in containers; its 2.0 release ships on **Harbor** [8, 9]. Harbor deserves its own entry: it is a meta-harness from the Terminal-Bench creators for "evaluating and optimizing agents and language models", wrapping third-party benchmarks (Terminal-Bench 2.0, SWE-bench, Aider Polyglot) behind one containerized runner, with the agent scaffold (Claude Code, OpenHands, Codex CLI, and more) selected independently of the `--model` flag [9]. That structure enables matched-model harness comparison, and a 2026 survey names Harbor (with Exgentic) as the first frameworks pushing a unified protocol for general agent assessment [1]. What Harbor does not do: rank efficiency, gate CI on regressions with statistics, or source fresh tasks with reference traces. **Harness-Bench** (2026) is the academic complement: a 6-harness by 8-model factorial over 106 sandboxed tasks (5,194 trajectories) that fixes task, sandbox, budget, timeout, and evaluator while letting each harness keep its native behavior [10]. It is a study rather than an ongoing service, but it is the cleanest matched-model prior art to date, and this paper leans on its design. + +**Aider Polyglot** (225 hard Exercism exercises) is honest about being a model benchmark: one fixed harness (Aider), many models, with cost per run displayed [11]. **Commit0** (build a library from scratch against a spec and tests) and **RepoBench** (repo-level completion) sit closer to the model end. **SWE-Gym** and **R2E** are environment generators, aimed at training rather than refereeing [12]. + +## 3. Survey: general agent benchmarks + +**AgentBench** (2023) spans eight environments from OS shells to web shopping [13]. **GAIA** (Meta/HF, 2023) asks 466 tool-requiring questions with unambiguous answers [14]. **WebArena** and **VisualWebArena** grade functional task completion on self-hosted websites [15]; **OSWorld** does the same for 369 tasks on real desktop operating systems [16]. **tau-bench** and **tau2-bench** (Sierra) simulate customer-service conversations with an LLM user and domain policies, and contribute the ecosystem's best reliability statistic: pass^k, the probability that an agent succeeds on all k independent attempts, which exposes flakiness that a single-run pass rate hides [17]. **TheAgentCompany** (CMU, 2024) drops agents into a simulated software company (GitLab, RocketChat, ownCloud) and grades checkpointed long-horizon work with partial credit [18]. **BFCL** (Berkeley) is the standard for function-call correctness, including multi-turn tool use [19]; **ToolBench** covers 16,000+ real APIs [20]. **MLE-bench** (OpenAI) uses 75 Kaggle competitions; **CORE-bench** (Princeton) tests computational reproducibility of real papers [21]. + +Two observations. First, nearly all of these publish leaderboards keyed by model, with the benchmark's own reference scaffold underneath: they are model benchmarks with agentic tasks. Second, the exceptions prove the demand: **HAL**, the Holistic Agent Leaderboard (Princeton), re-runs (model, scaffold) pairs across many of the benchmarks above and plots accuracy against cost, an explicit two-axis Pareto view [22]. HAL is the closest thing to a public engine-aware leaderboard. It still is not a tool a team can point at its own agent in its own CI, and cost is its only efficiency axis. + +## 4. Survey: trace-based evaluation and eval platforms + +Trajectory grading has real prior art. **AgentBoard**'s progress rate compares an agent's actual trajectory against an expected one for per-step progress [2, 26]. **T-Eval** decomposes tool use into step-wise next-call alignment [26]. LLM-as-judge trajectory scoring is now a stock feature of eval platforms. What does not exist is what we will call a golden-trace corpus: reference traces from verified, production-merged fixes, published as auditable evidence alongside outcome grades. + +The observability and eval platforms are where CI actually happens today. **LangSmith** (LangChain), **Langfuse** (open source), **Braintrust**, **Arize Phoenix** (open source, OpenTelemetry-based), and **W&B Weave** all offer tracing plus dataset-driven evals, and most can fail a CI job on a metric drop [27]. **DeepEval** runs evals as pytest tests; **promptfoo** runs config-driven evals and red-team suites in CI; **OpenAI Evals** seeded the genre [27]. On standards: OpenTelemetry's GenAI semantic conventions are emerging as the de facto wire format for agent traces (spans for model calls and tool executions), and the Model Context Protocol showed that a small interchange spec can reorganize an ecosystem within a year [28]. There is no equivalent interchange standard for benchmark tasks, runs, or verdicts. A 2026 factorial study proposes "Harness Cards", a structured disclosure taxonomy (execution, tool, context, scheduling, observability, verification, governance) for exactly this reason [3]. + +The platforms' limitation is the mirror image of the benchmarks'. They grade your agent on your dataset with judge-model rubrics: perfect for product regression, but with no held-out verification (the eval data lives in your repo, next to the agent that will be graded on it), no cross-engine comparability, and no shared task corpus, so a score means nothing outside your org. + +## 5. Best practices: what a credible agent benchmark must do in 2026 + +The literature has converged on a checklist, even if no system implements all of it. + +1. **Separate the engine from the model, by construction.** Hold one fixed while varying the other; stamp the run; refuse cross-attribution [1, 3, 10]. +2. **Held-out verification.** The grader must be absent from the workspace while the agent runs, and immune to anything the agent planted. Benchmarks with repo-native tests (SWE-bench) get this half right; eval platforms mostly do not attempt it [4]. +3. **Cost and efficiency as ranked metrics, not footnotes.** Accuracy-cost Pareto curves (AI Agents That Matter, HAL), plus tokens, wall clock, and, we argue, diff size: no surveyed system ranks the size of the change an agent makes, though every reviewer knows a 4-line fix and a 400-line rewrite are not equal [22, 23, 2]. +4. **Statistics that respect small n.** Confidence intervals on rates; paired tests for head-to-heads; reliability metrics like pass^k; seeded, reproducible resampling [17, 23]. +5. **Contamination control.** Frozen sets decay into training data; live sourcing (SWE-bench Live) with recorded cutoffs is the credible answer [7]. +6. **Receipts.** Full transcripts, diffs, manifests, and a reproduce command; disclosure of the harness configuration (Harness Cards) [3]. +7. **Run in CI.** A benchmark you run at launch is marketing; a benchmark that fails a pull request is engineering. + +## 6. Gap analysis: the four capabilities, and who has them + +**(a) Matched-model engine isolation.** Harbor enables it structurally; Harness-Bench and the factorial study execute it as research; HAL reports (model, scaffold) pairs [9, 10, 3, 22]. None operationalizes it as a stamped, enforced property of every published run. + +**(b) Efficiency grading beyond cost.** HAL plots cost; Aider Polyglot displays it; tau-bench measures reliability [22, 11, 17]. No surveyed system ranks normalized git-diff size, and only ad hoc reporting covers tokens and wall clock together [2]. + +**(c) CI regression gating with statistics.** Eval platforms gate CI on judge-scored private datasets; no benchmark harness offers a significance-aware gate over held-out, executable verification [27]. + +**(d) Live tasks with verified fixes as golden traces.** SWE-bench Live sources fresh issues; nobody promotes fixes upstream, treats maintainer merge as ground truth, and publishes the solving trace as a reference artifact [7]. + +No system does all four. That is the hole in the landscape, stated with the evidence above. + +## 7. Arena, and the Agent Benchmark Protocol + +**Arena** (github.com/macanderson/arena, MIT) is our attempt at the first three capabilities, today: two or more coding agents on the same tasks, same model, same budget, same timeout; held-out verification injected only after the agent exits; Wilson intervals, exact McNemar on paired outcomes, and seeded paired-bootstrap CIs on wall-clock, token, and cost deltas; no blended score; full receipts with a one-line reproduce command; and a CI gate (`arena baseline save` / `arena gate --require-significant`) that fails a pull request only when a regression clears interval noise. A Harbor adapter scales the same discipline to SWE-bench Verified with the official containerized verifier [9]. + +The fourth capability needs a standard more than a product. The **Agent Benchmark Protocol** (draft spec published alongside this paper) defines four interchange schemas: a task manifest, a run configuration, a trace format with per-subagent and per-model attribution, and a verdict format with required statistics. On top of them it specifies a live pipeline: a versioned registry of the top 500 open-source GitHub repositories; seeded random draws of real backlog issues; fixes built and submitted upstream under each repo's contribution rules; maintainer merge as verification; and the solving engine's full trace published as a golden trace beside the frozen task. Replays are graded on resolution first, then efficiency in strict order: smaller normalized diff, then fewer tokens, then less wall clock, then lower cost. Golden traces are evidence and study material, deliberately not the rubric, because trajectory-similarity grading punishes better routes to a passing fix; AgentBoard-style progress metrics are a version 2 question [26]. Contamination is handled by construction: tasks are drawn after candidate model cutoffs, stamped, and retired from the scored set 180 days after their fix goes public. + +Why a protocol rather than another leaderboard: the measurement problem is an interoperability problem. Tasks, runs, traces, and verdicts need to move between task authors, engines, harnesses, and auditors, exactly as context moves between hosts and tools under MCP [28]. Arena is the reference implementation, not the standard; the standard is the four schemas and the conformance suite. + +## 8. Limitations + +This survey characterizes fast-moving systems from their public documentation as of July 2026; version specifics will drift. The 7.8x variance ratio comes from one study on one benchmark family and one task size; it needs replication, though its direction agrees with SWE-agent, Agentless, and Harness-Bench [3, 24, 25, 10]. Diff-size ranking is a proxy for surgical precision, not design quality, and is honest only with the normalization and disqualification rules the spec defines. And the author's conflict of interest is real: the reader should treat Section 7 as a proposal to be attacked, with the receipts to attack it. + +## References + +1. Yehudai et al., *Survey on Evaluation of LLM-based Agents*, v2 (2026). arxiv.org/abs/2503.16416 +2. *A Survey of AI Agent Evaluation* (2025). arxiv.org/abs/2507.21504 +3. *Harness variance factorial study: Harness Cards and ETCSOVG taxonomy* (2026). arxiv.org/abs/2605.23950 +4. Jimenez et al., *SWE-bench: Can Language Models Resolve Real-World GitHub Issues?* (2023). arxiv.org/abs/2310.06770 +5. OpenAI, *Introducing SWE-bench Verified* (2024). openai.com/index/introducing-swe-bench-verified +6. Scale AI, *SWE-bench Pro* (2025). github.com/scaleapi/SWE-bench_Pro-os +7. *SWE-bench Live* (2025). swe-bench-live.github.io +8. Terminal-Bench (2025). tbench.ai +9. Harbor Framework (2026). github.com/harbor-framework/harbor +10. *Harness-Bench* (2026). arxiv.org/abs/2605.27922 +11. Aider Polyglot leaderboard. aider.chat/docs/leaderboards +12. SWE-Gym: github.com/SWE-Gym · R2E: r2e.dev · Commit0: github.com/commit-0/commit0 · RepoBench (2023) +13. Liu et al., *AgentBench* (2023). arxiv.org/abs/2308.03688 +14. Mialon et al., *GAIA* (2023). arxiv.org/abs/2311.12983 +15. Zhou et al., *WebArena* (2023). arxiv.org/abs/2307.13854 +16. Xie et al., *OSWorld* (2024). arxiv.org/abs/2404.07972 +17. Yao et al., *tau-bench* (2024). arxiv.org/abs/2406.12045 · tau2-bench: github.com/sierra-research/tau2-bench +18. Xu et al., *TheAgentCompany* (2024). arxiv.org/abs/2412.14161 +19. Berkeley Function-Calling Leaderboard. gorilla.cs.berkeley.edu/leaderboard.html +20. Qin et al., *ToolLLM/ToolBench* (2023). arxiv.org/abs/2307.16789 +21. OpenAI, *MLE-bench* (2024). arxiv.org/abs/2410.07095 · CORE-bench (Princeton, 2024) +22. HAL: Holistic Agent Leaderboard (Princeton). hal.cs.princeton.edu +23. Kapoor et al., *AI Agents That Matter* (2024). arxiv.org/abs/2407.01502 +24. Yang et al., *SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering* (2024). arxiv.org/abs/2405.15793 +25. Xia et al., *Agentless* (2024). arxiv.org/abs/2407.01489 +26. Ma et al., *AgentBoard* (2024). arxiv.org/abs/2401.13178 · T-Eval (2023). arxiv.org/abs/2312.14033 +27. LangSmith: smith.langchain.com · Langfuse: langfuse.com · Braintrust: braintrust.dev · Arize Phoenix: phoenix.arize.com · W&B Weave: wandb.ai · DeepEval: github.com/confident-ai/deepeval · promptfoo: promptfoo.dev · OpenAI Evals: github.com/openai/evals +28. Model Context Protocol. modelcontextprotocol.io · OpenTelemetry GenAI semantic conventions. opentelemetry.io + +--- + +*Companion documents: the [Agent Benchmark Protocol draft spec](agent-benchmark-protocol.md) and [Arena's methodology](../METHODOLOGY.md). Corrections: open an issue with the `paper` label.* diff --git a/harbor/arena_harbor/base.py b/harbor/arena_harbor/base.py index f0ad518..d1161e2 100644 --- a/harbor/arena_harbor/base.py +++ b/harbor/arena_harbor/base.py @@ -135,6 +135,18 @@ async def run( ) env = forwarded_env(spec) + # Accept "600", "600.5", etc. A non-numeric ARENA_TIMEOUT must not + # silently disable the container timeout. + try: + timeout_sec: int | None = int(float(timeout)) + except ValueError: + timeout_sec = 1800 + self.logger.warning( + "arena-harbor: ARENA_TIMEOUT=%r is not numeric; using %ss", + timeout, + timeout_sec, + ) + # Run directly (not exec_as_agent) so a non-zero agent exit does NOT # abort the trial before Harbor's verifier gets to judge the workspace. # The verifier — never the CLI's exit code — decides pass/fail. @@ -142,7 +154,7 @@ async def run( result = await environment.exec( command=command, env=env, - timeout_sec=_parse_timeout(timeout), + timeout_sec=timeout_sec, ) self._agent_output = "\n".join( part @@ -171,10 +183,3 @@ def populate_context_post_run(self, context: AgentContext) -> None: context.cost_usd = parsed.cost_usd context.metadata = {**(context.metadata or {}), **parsed.as_metadata(spec.name)} - -def _parse_timeout(raw: str) -> int | None: - """``ARENA_TIMEOUT`` seconds as an int; None (no limit) when unparseable.""" - try: - return int(float(raw)) - except ValueError: - return None diff --git a/harbor/arena_harbor/command.py b/harbor/arena_harbor/command.py index e243d69..050ef62 100644 --- a/harbor/arena_harbor/command.py +++ b/harbor/arena_harbor/command.py @@ -55,6 +55,13 @@ def build_command( "timeout": timeout or "1800", "instruction": shlex.quote(instruction), } + if "{budget}" in spec.run_template and not mapping["budget"]: + # An empty {budget} would render e.g. "--max-usd --json", making the + # flag silently eat the next token. Fail loudly instead. + raise ValueError( + f"agent '{spec.name}': run_template references {{budget}} but no " + "budget is set (set ARENA_BUDGET or default_budget in the spec)." + ) return render_template(spec.run_template, mapping) diff --git a/harbor/arena_harbor/metrics.py b/harbor/arena_harbor/metrics.py index 24b2c39..0e57a8c 100644 --- a/harbor/arena_harbor/metrics.py +++ b/harbor/arena_harbor/metrics.py @@ -39,9 +39,10 @@ def is_empty(self) -> bool: def last_json_object(text: str) -> dict[str, Any] | None: - """Return the last ``{"type":"result"}`` JSON object in ``text`` (JSONL or a - single object), else the last parseable object. Non-JSON lines are ignored. - Mirrors the TS ``parseJsonEnvelope``. + """Return the last ``{"type":"result"}`` JSON object in ``text`` (JSONL, a + single object, or a pretty-printed multi-line object), else the last + parseable object. Non-JSON lines are ignored. Mirrors the TS + ``parseJsonEnvelope``. """ if not text: return None @@ -57,7 +58,75 @@ def last_json_object(text: str) -> dict[str, Any] | None: fallback = obj if obj.get("type") == "result": return obj - return fallback + + # No single-line result envelope: pretty-printed envelopes span lines, so + # fall back to a string-aware brace scan over the whole output. + scanned = _scan_json_objects(text) + for obj in reversed(scanned): + if obj.get("type") == "result": + return obj + if fallback is not None: + return fallback + return scanned[-1] if scanned else None + + +def _scan_json_objects(text: str) -> list[dict[str, Any]]: + """Find top-level JSON objects in text that may contain non-JSON noise. + + Candidates are anchored at lines that START with ``{`` (a pretty-printed + envelope's opening line), so a stray brace mid-way through a log line can + never derail the scan. From each anchor, string-aware brace matching finds + the balanced end; spans that fail to parse are skipped. + """ + found: list[dict[str, Any]] = [] + offset = 0 + consumed_up_to = -1 + for line in text.split("\n"): + line_start = offset + offset += len(line) + 1 + if line_start < consumed_up_to: + continue # inside an already-parsed object + stripped = line.lstrip() + if not stripped.startswith("{"): + continue + start = line_start + (len(line) - len(stripped)) + end = _balanced_object_end(text, start) + if end == -1: + continue + try: + obj = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + continue # balanced braces but not JSON (shell/awk) — skip + if isinstance(obj, dict): + found.append(obj) + consumed_up_to = end + 1 + return found + + +def _balanced_object_end(text: str, start: int) -> int: + """Index of the ``}`` closing the object opened at ``start``, or -1.""" + depth = 0 + in_string = False + escaped = False + for i in range(start, len(text)): + ch = text[i] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return i + return -1 def dotted_get(obj: Any, path: str | None) -> Any: diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index de11f10..865bdcb 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -20,7 +20,8 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -# Pinned to the 0.6.x line the adapter is built and tested against. +# Built and tested against 0.6.x; the range admits later 0.x releases the +# spec-driven surface has stayed compatible with. Tighten if Harbor breaks API. dependencies = [ "harbor>=0.6,<0.20", ] diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 0000000..245259b --- /dev/null +++ b/site/.gitignore @@ -0,0 +1,2 @@ +.vercel +.env* diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..d56df8a --- /dev/null +++ b/site/index.html @@ -0,0 +1,194 @@ + + + + + + Arena: benchmark the agent engine, not the model + + + + + + + + +
+
+ arena.oxagen.sh + +
+
+ +
+
+
+

Your agent got worse last Tuesday. Nobody noticed.

+

+ Every leaderboard measures the model. Nobody measures your agent engine: + the harness, the prompts, the tools, the loop you actually ship. Arena runs coding agents + head-to-head on the same model, same budget, same timeout, grades them with + held-out tests they can never see, and reports each metric separately with + real statistics and full receipts. +

+
+ git clone macanderson/arena + Read the paper + Read the spec +
+ + +

Report format shown with example data. Every real report carries its own intervals, p-values, and raw run directory.

+
+ +
+

The problem

+

Model benchmarks answer a question you are not asking

+
+

+ When your coding agent gets better or worse, the model is usually not what changed. + You changed a system prompt, added a tool, reworked the planning loop, or bumped a + dependency. Model leaderboards cannot see any of that. They hold the harness fixed + and vary the model. Your job is the opposite: the model is a config value, and the + engine is the product. +

+

+ Arena isolates the engine. Two or more agents, one model, one budget, one timeout, + the same tasks. If the numbers differ, the harness is the reason. That is the + comparison that tells you whether your last sprint helped, and the one no public + leaderboard runs for you. +

+
+
+ +
+

Trust

+

Built to survive skeptical reading

+
+
+
The agent graded itself
+
Held-out verification. Tests live outside the workspace, are copied in only after the agent exits, and anything planted at the verify path is deleted first. CI proves every task fails pristine and passes the reference solution.
+
+
+
Different models or budgets
+
One config drives every agent. Unmatched runs are stamped matchedModels: false and the report leads with a warning banner.
+
+
+
Your harness broke the competitor
+
Harness failures are excluded. A CLI that cannot be invoked scores agent-error and is removed from every comparison. It never counts as the agent losing.
+
+
+
n = 1, no stats
+
Real statistics. Wilson 95% intervals, exact McNemar on paired outcomes, seeded paired-bootstrap CIs on speed, tokens, and cost. Same data plus same seed gives the same report, bit for bit.
+
+
+
Where is the raw data
+
Full receipts. Every trial writes its transcript, workspace diff, and a manifest with CLI versions, models, host, seed, and a one-line reproduce command.
+
+
+
+ +
+

Measurement

+

Four metrics, never blended

+

+ A single blended score hides the tradeoff you care about. Arena reports each metric + separately, with its own interval, and refuses to average them. +

+
+
+
accuracy
+
Held-out tests pass. Wilson 95% interval on every rate.
+
+
+
speed
+
Wall-clock, harness-measured. Medians with bootstrap CIs.
+
+
+
tokens
+
Normalized per adapter. Cache reads never inflate input.
+
+
+
cost
+
One shared price table, or no number at all. Never a guess.
+
+
+
+ +
+

CI integration

+

The regression gate: a tripwire for agent quality

+
+

+ Benchmarks are not only for bragging. Snapshot a good run as a baseline, commit it, + and fail CI the day a change makes your agent less accurate, slower, or more + expensive. With --require-significant, a drop only fails once it clears + 95% interval noise, so small samples do not cry wolf. +

+
+ 
# establish a baseline once, commit it
+pnpm arena run --agents mine --model anthropic/claude-sonnet-5 --trials 5
+pnpm arena baseline save results/run-…        # writes arena-baseline.json
+
+# in CI: rerun and gate. non-zero exit = regression
+pnpm arena run --agents mine --model anthropic/claude-sonnet-5 --trials 5
+pnpm arena gate results/run-… --require-significant
+
+ +
+

Go deeper

+

The paper, the spec, and the methodology

+
+ +
The State of Agent Benchmarking
+
A survey of the 2026 landscape: what exists, what it measures, and why almost none of it evaluates the engine you ship.
+
research paper
+ + +
Agent Benchmark Protocol
+
A draft open standard: live tasks from real repos, verified fixes, golden traces, and grading any engine can plug into.
+
design spec · draft
+ + +
Methodology
+
The contract behind any number Arena produces, including what it takes to publish an A-beats-B claim.
+
github.com/macanderson/arena
+ +
+
+
+
+ +
+ +
+ + diff --git a/site/paper/index.html b/site/paper/index.html new file mode 100644 index 0000000..e9f14fb --- /dev/null +++ b/site/paper/index.html @@ -0,0 +1,335 @@ + + + + + + The State of Agent Benchmarking · Arena + + + + + +
+
+ arena.oxagen.sh + +
+ +
+

Research paper · July 2026 · v1.0

+

The State of Agent Benchmarking

+

Everyone measures the model. Almost nobody measures the engine you ship. · Mac Anderson, Oxagen

+ +
+ Abstract. Teams that ship AI agents change their engine every week: prompts, + tools, planning loops, context strategies, and orchestration. The model changes a few times a + year. Yet nearly every public benchmark ranks models with the harness held fixed, the exact + inverse of the question a working team needs answered: did my last change make my agent + better or worse, on the model I already run? Recent controlled evidence says this inversion + is not cosmetic. In a 3x3 factorial study on SWE-bench Verified, harness-induced variance + exceeded model-induced variance by 7.8x, and model rankings reversed in six of nine harness + pairings [3]. This paper surveys the landscape as of July 2026 through that lens and finds + that no existing system combines matched-model engine isolation, efficiency grading, + statistically honest CI gating, and live task sourcing with verified fixes as reference + traces. We close by describing that missing system, and an interchange standard that would + let any harness provide it. +
+ +
+ Conflict of interest, stated up front: the author builds + Arena, an open-source agent-engine benchmark + harness, and the Oxagen and Stella agents that appear in its examples. This paper argues a + position. Every load-bearing claim carries a citation, and Section 6 names the systems that + already do parts of what Arena does. +
+ + + +
+

1. The question benchmarks answer, and the question teams ask

+

+ A model leaderboard answers: given a fixed scaffold, which model scores highest? That is + the right question for a lab choosing a base model, and the wrong question for everyone + downstream. Downstream, the model is a config value. The product is the engine: the system + prompt, the tool set, the retrieval and context policy, the planning loop, the subagent + topology, and the guardrails. One survey puts the distinction plainly: evaluating the LLM + alone is like examining an engine on a stand, while agent evaluation must assess the whole + car under driving conditions [2]. +

+

+ The scale of what the scaffold contributes has been measured repeatedly. SWE-agent showed + in 2024 that the agent-computer interface alone moves resolve rates at a fixed model [24]. + Agentless showed the same year that a deliberately non-agentic pipeline could outperform + elaborate agents at a fraction of the cost [25]. And in 2026 a controlled factorial study + quantified it: across three models and three harness configurations on a 100-task + SWE-bench Verified subset, the harness contributed 18.48 pp² of score variance against the + model's 2.37 pp², a ratio of 7.8x, with six model-ranking reversals across nine harness + pairings [3]. The authors' conclusion is the thesis of this paper: the execution harness, + the infrastructure layer that governs context construction, tool interaction, + orchestration, and verification, is often a stronger determinant of agent performance than + the model it wraps [3]. +

+

+ The benchmark ecosystem has not caught up. The most comprehensive survey of agent + evaluation to date lists "Decoupling LLM & Harness Evaluation" as future work: most + current benchmarks conflate the two targets, and no established benchmark provides + controlled protocols that vary each factor independently [1]. The same survey finds cost + and efficiency metrics broadly neglected, echoing "AI Agents That Matter": accuracy-only + leaderboards drive needlessly costly agents [1, 23]. +

+ +

2. Survey: coding-agent benchmarks

+

+ SWE-bench (Princeton/Stanford, 2023) established the template: 2,294 real + GitHub issues from 12 Python repos, graded by the repo's own FAIL_TO_PASS and PASS_TO_PASS + tests [4]. SWE-bench Verified (OpenAI, 2024) human-filtered 500 instances + to remove impossible or underspecified tasks [5]. SWE-bench Multimodal + adds visual, JavaScript-centric issues; SWE-bench Pro (Scale AI, 2025) + raises difficulty with long-horizon, enterprise-style tasks and a held-out commercial + subset [6]. SWE-bench Live (Microsoft Research and community, 2025) + attacks contamination directly with a monthly-refreshed feed of new issues [7]. All of + these grade one thing: did the repo's tests pass. None grades diff size, token spend, or + wall clock as a ranked metric, and their leaderboards mix (model, scaffold) pairs, so a + score is attributable to neither alone. They are, however, the raw material engine + benchmarking needs: real repos, real issues, executable verification. +

+

+ Terminal-Bench (Stanford and the Laude Institute, 2025) measures + terminal-native task completion in containers; its 2.0 release ships on + Harbor [8, 9]. Harbor deserves its own entry: it is a meta-harness from + the Terminal-Bench creators for "evaluating and optimizing agents and language models", + wrapping third-party benchmarks (Terminal-Bench 2.0, SWE-bench, Aider Polyglot) behind one + containerized runner, with the agent scaffold (Claude Code, OpenHands, Codex CLI, and + more) selected independently of the --model flag [9]. That structure enables + matched-model harness comparison, and a 2026 survey names Harbor (with Exgentic) as the + first frameworks pushing a unified protocol for general agent assessment [1]. What Harbor + does not do: rank efficiency, gate CI on regressions with statistics, or source fresh + tasks with reference traces. Harness-Bench (2026) is the academic + complement: a 6-harness by 8-model factorial over 106 sandboxed tasks (5,194 trajectories) + that fixes task, sandbox, budget, timeout, and evaluator while letting each harness keep + its native behavior [10]. It is a study rather than an ongoing service, but it is the + cleanest matched-model prior art to date. +

+

+ Aider Polyglot (225 hard Exercism exercises) is honest about being a + model benchmark: one fixed harness, many models, with cost per run displayed [11]. + Commit0 (build a library from scratch against a spec and tests) and + RepoBench (repo-level completion) sit closer to the model end. + SWE-Gym and R2E are environment generators, aimed at + training rather than refereeing [12]. +

+ +

3. Survey: general agent benchmarks

+

+ AgentBench (2023) spans eight environments from OS shells to web shopping + [13]. GAIA (Meta/HF, 2023) asks 466 tool-requiring questions with + unambiguous answers [14]. WebArena and VisualWebArena + grade functional task completion on self-hosted websites [15]; OSWorld + does the same for 369 tasks on real desktop operating systems [16]. + tau-bench and tau2-bench (Sierra) simulate + customer-service conversations with an LLM user and domain policies, and contribute the + ecosystem's best reliability statistic: pass^k, the probability that an agent succeeds on + all k independent attempts, which exposes flakiness a single-run pass rate hides [17]. + TheAgentCompany (CMU, 2024) drops agents into a simulated software + company (GitLab, RocketChat, ownCloud) and grades checkpointed long-horizon work with + partial credit [18]. BFCL (Berkeley) is the standard for function-call + correctness, including multi-turn tool use [19]; ToolBench covers + 16,000+ real APIs [20]. MLE-bench (OpenAI) uses 75 Kaggle competitions; + CORE-bench (Princeton) tests computational reproducibility of real + papers [21]. +

+

+ Two observations. First, nearly all of these publish leaderboards keyed by model, with the + benchmark's own reference scaffold underneath: they are model benchmarks with agentic + tasks. Second, the exceptions prove the demand: HAL, the Holistic Agent + Leaderboard (Princeton), re-runs (model, scaffold) pairs across many of the benchmarks + above and plots accuracy against cost, an explicit two-axis Pareto view [22]. HAL is the + closest thing to a public engine-aware leaderboard. It still is not a tool a team can + point at its own agent in its own CI, and cost is its only efficiency axis. +

+ +

4. Survey: trace-based evaluation and eval platforms

+

+ Trajectory grading has real prior art. AgentBoard's progress rate + compares an agent's actual trajectory against an expected one for per-step progress + [2, 26]. T-Eval decomposes tool use into step-wise next-call alignment + [26]. LLM-as-judge trajectory scoring is now a stock feature of eval platforms. What does + not exist is what we will call a golden-trace corpus: reference traces from verified, + production-merged fixes, published as auditable evidence alongside outcome grades. +

+

+ The observability and eval platforms are where CI actually happens today. + LangSmith, Langfuse, Braintrust, + Arize Phoenix, and W&B Weave all offer tracing plus + dataset-driven evals, and most can fail a CI job on a metric drop [27]. + DeepEval runs evals as pytest tests; promptfoo runs + config-driven evals and red-team suites in CI; OpenAI Evals seeded the + genre [27]. On standards: OpenTelemetry's GenAI semantic conventions are emerging as the + de facto wire format for agent traces, and the Model Context Protocol showed that a small + interchange spec can reorganize an ecosystem within a year [28]. There is no equivalent + interchange standard for benchmark tasks, runs, or verdicts. A 2026 factorial study + proposes "Harness Cards", a structured disclosure taxonomy for exactly this reason [3]. +

+

+ The platforms' limitation is the mirror image of the benchmarks'. They grade your agent on + your dataset with judge-model rubrics: perfect for product regression, but with no + held-out verification (the eval data lives in your repo, next to the agent that will be + graded on it), no cross-engine comparability, and no shared task corpus, so a score means + nothing outside your org. +

+ +

5. Best practices: what a credible agent benchmark must do in 2026

+
    +
  1. Separate the engine from the model, by construction. Hold one fixed while varying the other; stamp the run; refuse cross-attribution [1, 3, 10].
    2. +
  2. Held-out verification. The grader must be absent from the workspace while the agent runs, and immune to anything the agent planted [4].
    3. +
  3. Cost and efficiency as ranked metrics, not footnotes. Accuracy-cost Pareto curves, plus tokens, wall clock, and, we argue, diff size: no surveyed system ranks the size of the change an agent makes, though every reviewer knows a 4-line fix and a 400-line rewrite are not equal [22, 23, 2].
    4. +
  4. Statistics that respect small n. Confidence intervals on rates; paired tests for head-to-heads; reliability metrics like pass^k; seeded, reproducible resampling [17, 23].
    5. +
  5. Contamination control. Frozen sets decay into training data; live sourcing with recorded cutoffs is the credible answer [7].
    6. +
  6. Receipts. Full transcripts, diffs, manifests, a reproduce command, and disclosure of the harness configuration [3].
    7. +
  7. Run in CI. A benchmark you run at launch is marketing; a benchmark that fails a pull request is engineering.
    8. +
+ +

6. Gap analysis: the four capabilities, and who has them

+
+ + + + + + + + + + + + + + + + + + + + + + + + +
CapabilityClosest todayWhat's missing
(a) Matched-model engine isolationHarbor (structural) [9]; Harness-Bench, factorial study (research) [10, 3]; HAL (reporting) [22]An enforced, stamped property of every published run
(b) Efficiency grading beyond costHAL and Aider show cost [22, 11]; tau-bench measures reliability [17]Nobody ranks normalized git-diff size; tokens + wall clock rarely reported together
(c) CI regression gating with statisticsEval platforms gate CI on judge-scored private datasets [27]No significance-aware gate over held-out, executable verification
(d) Live tasks, verified fixes, golden tracesSWE-bench Live sources fresh issues [7]Nobody promotes fixes upstream, uses maintainer merge as ground truth, or publishes the solving trace
+
+

No system does all four. That is the hole in the landscape.

+ +

7. Arena, and the Agent Benchmark Protocol

+

+ Arena (github.com/macanderson/arena, + MIT) is our attempt at the first three capabilities, today: two or more coding agents on + the same tasks, same model, same budget, same timeout; held-out verification injected only + after the agent exits; Wilson intervals, exact McNemar on paired outcomes, and seeded + paired-bootstrap CIs on wall-clock, token, and cost deltas; no blended score; full + receipts with a one-line reproduce command; and a CI gate that fails a pull request only + when a regression clears interval noise. A Harbor adapter scales the same discipline to + SWE-bench Verified with the official containerized verifier [9]. +

+

+ The fourth capability needs a standard more than a product. The + Agent Benchmark Protocol defines four interchange schemas: a task + manifest, a run configuration, a trace format with per-subagent and per-model attribution, + and a verdict format with required statistics. On top of them it specifies a live + pipeline: a versioned registry of the top 500 open-source GitHub repositories; seeded + random draws of real backlog issues; fixes built and submitted upstream under each repo's + contribution rules; maintainer merge as verification; and the solving engine's full trace + published as a golden trace beside the frozen task. Replays are graded on resolution + first, then efficiency in strict order: smaller normalized diff, then fewer tokens, then + less wall clock, then lower cost. Golden traces are evidence and study material, + deliberately not the rubric, because trajectory-similarity grading punishes better routes + to a passing fix [26]. Contamination is handled by construction: tasks are drawn after + candidate model cutoffs, stamped, and retired from the scored set 180 days after their fix + goes public. +

+

+ Why a protocol rather than another leaderboard: the measurement problem is an + interoperability problem. Tasks, runs, traces, and verdicts need to move between task + authors, engines, harnesses, and auditors, exactly as context moves between hosts and + tools under MCP [28]. Arena is the reference implementation, not the standard; the + standard is the four schemas and the conformance suite. +

+ +

8. Limitations

+

+ This survey characterizes fast-moving systems from their public documentation as of July + 2026; version specifics will drift. The 7.8x variance ratio comes from one study on one + benchmark family and one task size; it needs replication, though its direction agrees with + SWE-agent, Agentless, and Harness-Bench [3, 24, 25, 10]. Diff-size ranking is a proxy for + surgical precision, not design quality, and is honest only with the normalization and + disqualification rules the spec defines. And the author's conflict of interest is real: + the reader should treat Section 7 as a proposal to be attacked, with the receipts to + attack it. +

+ +

References

+
    +
  1. Yehudai et al., Survey on Evaluation of LLM-based Agents, v2 (2026). arxiv.org/abs/2503.16416
    2. +
  2. A Survey of AI Agent Evaluation (2025). arxiv.org/abs/2507.21504
    3. +
  3. Harness variance factorial study: Harness Cards and the ETCSOVG taxonomy (2026). arxiv.org/abs/2605.23950
    4. +
  4. Jimenez et al., SWE-bench (2023). arxiv.org/abs/2310.06770
    5. +
  5. OpenAI, Introducing SWE-bench Verified (2024). openai.com/index/introducing-swe-bench-verified
    6. +
  6. Scale AI, SWE-bench Pro (2025). github.com/scaleapi/SWE-bench_Pro-os
    7. +
  7. SWE-bench Live (2025). swe-bench-live.github.io
    8. +
  8. Terminal-Bench (2025). tbench.ai
    9. +
  9. Harbor Framework (2026). github.com/harbor-framework/harbor
    10. +
  10. Harness-Bench (2026). arxiv.org/abs/2605.27922
    11. +
  11. Aider Polyglot leaderboard. aider.chat/docs/leaderboards
    12. +
  12. SWE-Gym · R2E (r2e.dev) · Commit0 (github.com/commit-0/commit0) · RepoBench (2023)
    13. +
  13. Liu et al., AgentBench (2023). arxiv.org/abs/2308.03688
    14. +
  14. Mialon et al., GAIA (2023). arxiv.org/abs/2311.12983
    15. +
  15. Zhou et al., WebArena (2023). arxiv.org/abs/2307.13854
    16. +
  16. Xie et al., OSWorld (2024). arxiv.org/abs/2404.07972
    17. +
  17. Yao et al., tau-bench (2024). arxiv.org/abs/2406.12045 · tau2-bench: github.com/sierra-research/tau2-bench
    18. +
  18. Xu et al., TheAgentCompany (2024). arxiv.org/abs/2412.14161
    19. +
  19. Berkeley Function-Calling Leaderboard. gorilla.cs.berkeley.edu/leaderboard.html
    20. +
  20. Qin et al., ToolLLM/ToolBench (2023). arxiv.org/abs/2307.16789
    21. +
  21. OpenAI, MLE-bench (2024). arxiv.org/abs/2410.07095 · CORE-bench (Princeton, 2024)
    22. +
  22. HAL: Holistic Agent Leaderboard (Princeton). hal.cs.princeton.edu
    23. +
  23. Kapoor et al., AI Agents That Matter (2024). arxiv.org/abs/2407.01502
    24. +
  24. Yang et al., SWE-agent (2024). arxiv.org/abs/2405.15793
    25. +
  25. Xia et al., Agentless (2024). arxiv.org/abs/2407.01489
    26. +
  26. Ma et al., AgentBoard (2024). arxiv.org/abs/2401.13178 · T-Eval (2023). arxiv.org/abs/2312.14033
    27. +
  27. LangSmith · Langfuse · Braintrust · Arize Phoenix · W&B Weave · DeepEval · promptfoo · OpenAI Evals
    28. +
  28. Model Context Protocol (modelcontextprotocol.io) · OpenTelemetry GenAI semantic conventions
    29. +
+ +

+ The markdown source of this paper lives in the repo at + docs/agent-engine-benchmarks-2026.md. Corrections: open an issue with the + paper label. +

+
+
+
+ +
+ +
+ + diff --git a/site/spec/index.html b/site/spec/index.html new file mode 100644 index 0000000..f7df1dc --- /dev/null +++ b/site/spec/index.html @@ -0,0 +1,351 @@ + + + + + + Agent Benchmark Protocol (ABP) · Draft 0.1 + + + + + +
+
+ arena.oxagen.sh + +
+ +
+

Design spec · Draft 0.1 · July 2026

+

Agent Benchmark Protocol

+

Oxagen · reference implementation: Arena · status: draft for public comment

+ +
+ An open standard for benchmarking agent engines: the harness, prompts, + tools, and loop an agent ships with, measured separately from the model it happens to run + on. ABP standardizes four interchange contracts (task manifest, run config, trace, and + verdict) plus a live task pipeline in which verified fixes to real GitHub issues become + golden traces. Nothing here is final, including the name. +
+ + + +
+

1. The problem this standard exists to solve

+

+ Teams that ship agents change their engine every week: a new system prompt, a new tool, a + reworked planning loop, a different context strategy. None of those changes show up on a + model leaderboard, because model leaderboards hold the harness fixed and vary the model. + The people who build agent engines have the opposite need: hold the model fixed and vary + the engine. +

+

Today that comparison is nearly impossible to run honestly:

+
    +
  1. Every harness invokes agents differently, so "same task" rarely means same prompt, same budget, same timeout, or same autonomy level.
    2. +
  2. Every harness logs differently, so traces cannot be compared across engines.
    3. +
  3. Every harness grades differently, and most let the agent's own output influence the grade.
    4. +
  4. Almost no benchmark measures efficiency. Two agents that both fix a bug are scored as equals even when one shipped a 4-line patch in 90 seconds and the other shipped a 400-line rewrite in 20 minutes.
    5. +
  5. Almost no benchmark runs in CI. A benchmark you run once for a launch tweet cannot catch the Tuesday your agent quietly got worse.
    6. +
+

+ ABP standardizes the four interfaces that make honest engine comparison possible: how a + task is packaged, how a run is configured, how a trace is recorded, and how a verdict is + reported. Any engine that speaks these formats can be benchmarked by any conforming + harness, and any harness can be audited by anyone. +

+ +

2. Why "protocol"

+

+ We considered "standard", "format", "spec", and "benchmark". Protocol is the right word + for the same reason it was right for the Model Context Protocol: the deliverable is a set + of interchange contracts between independent parties (task authors, engines, harnesses, + graders, and auditors), not a single dataset or a single leaderboard. Datasets and + leaderboards are built on top of ABP; they are products, and ABP is the plumbing. It also + keeps the family resemblance with the Open Context Protocol, which Oxagen authored and + built Stella on. +

+

+ One naming caution, stated openly: "protocol" sets an expectation of wire-level rigor. + This draft earns that word only in Sections 5 through 8, where the schemas live. If the + community reads those sections and concludes the word oversells, the fallback name is the + Agent Benchmark Format (ABF). The acronym to protect is ABP, one syllable per letter, easy + to say in CI logs: abp gate failed. +

+ +

3. Design principles

+

These are inherited from Arena's methodology and are non-negotiable for conformance:

+
    +
  1. The engine is the unit under test. The model, provider, budget, and timeout are pinned inputs. Runs that vary the model are stamped as such and cannot back engine-vs-engine claims.
    2. +
  2. The agent never grades itself. Verification material must be absent from the workspace while the agent runs, injected only after it exits, and immune to anything the agent planted.
    3. +
  3. Every number carries its uncertainty. Success rates get Wilson intervals. Paired comparisons get exact McNemar. Continuous metrics get seeded bootstrap CIs. No blended scores, ever.
    4. +
  4. Harness failures are not agent failures. An engine that could not be invoked is excluded from comparison, not counted as a loss.
    5. +
  5. Receipts or it did not happen. A conforming result ships its manifest, transcripts, diffs, and a one-line reproduce command.
    6. +
  6. Efficiency is a first-class metric. Resolution alone is a tie. Diff size, tokens, wall clock, and cost break it.
    7. +
+ +

4. The live task pipeline

+

This is the part of ABP that no existing benchmark provides end to end.

+ +

4.1 Sourcing

+

+ A curated registry of the top 500 open-source repositories on GitHub, filtered for: OSI + license, active maintainership (merged PR in the last 30 days), a runnable test suite, and + issue hygiene (reproducible reports, labeled backlog). The registry is versioned and + published; entries rotate quarterly so the pool tracks the real ecosystem. +

+

+ From the registry, tasks are drawn by seeded random selection: pick a repo, pick an open + issue from its backlog that passes eligibility (reproducible, in-scope for a code change, + not a question or a feature debate). The seed is published with every draw, so the + selection itself is auditable. +

+ +

4.2 Solving and promotion

+

+ A frontier engine (or a human, or both) builds a fix for the issue against the repo's + contribution rules and submits it upstream. The fix is verified when the + project's own CI passes and a maintainer merges it into the production branch. Maintainer + merge is the ground truth no synthetic benchmark has: a real reviewer accepted this change + into a real codebase. +

+

Two artifacts fall out of every verified fix:

+
    +
  • Training data. The (issue, repo state, verified diff) triple.
    • +
  • A golden trace. The full ABP trace (Section 6) the solving engine printed along the way: every tool call, every file read, every edit, every token count, timestamped.
    • +
+ +

4.3 Replay and grading

+

+ Once a task is verified, it freezes: repo pinned at the pre-fix commit, issue text + captured, and a verification bundle built from the repo's own test suite plus the tests + the verified fix added (the FAIL_TO_PASS set). Other engines then attempt the same frozen + task under matched config. They are graded on: +

+
    +
  1. Resolution. The held-out verification bundle passes.
    2. +
  2. Efficiency, in strict order: smaller normalized git diff wins, then fewer tokens, then less wall clock, then lower cost.
    3. +
+

+ The golden trace is published as evidence and as a study aid, not as the rubric. Version 1 + deliberately does not grade trace similarity: an engine that reaches a passing, smaller + fix by a different route beats the golden trace's author. Judging architectural quality + and trajectory quality is a stated version 2 direction, and it needs its own adversarial + review before it can be a metric. +

+ +

4.4 Contamination control

+

+ Live sourcing is also the contamination story. Issues are selected from the current + backlog, after every candidate model's training cutoff, and each task carries a + sourcedAt timestamp plus the model cutoffs it was verified against. Tasks age + out of the scored set once their fix has been public for 180 days; they remain available + as a practice set. Frozen-set benchmarks decay into training data. A pipeline that keeps + drawing from the living backlog does not. +

+ +

4.5 Known objections, answered in the design

+
    +
  • "Smallest diff is gameable." Diff size is measured on a normalized form: formatting-only changes are canonicalized before counting, generated files and lockfiles are excluded, and test files are counted separately (deleting or weakening tests disqualifies the run, it does not shrink the diff). Resolution is still the gate; diff size only ranks engines that already passed. And the metric is honest about what it is: a proxy for surgical precision, not for design quality. Arena publishes the raw diffs, so anyone can audit whether small meant surgical or small meant degenerate.
    • +
  • "Maintainers did not consent to being a benchmark." Sourcing follows each repo's contribution guidelines, fixes are submitted as normal PRs with disclosure, registry entries can opt out, and replay runs happen on frozen clones, never against the live repo.
    • +
  • "Issue selection will favor easy tasks." Selection is seeded-random within eligibility, the seed is published, and difficulty is stratified in reporting (task metadata records diff size and files touched of the verified fix).
    • +
+ +

5. Task manifest (abp-task/v1)

+ 
{
+  "schema": "abp-task/v1",
+  "id": "gh-vercel-next.js-81234",
+  "source": {
+    "kind": "github-issue",              // or "fixture" for synthetic tasks
+    "repo": "vercel/next.js",
+    "issue": 81234,
+    "baseCommit": "9f2c41d…",            // repo frozen here (pre-fix)
+    "sourcedAt": "2026-07-02T14:11:09Z",
+    "registryVersion": "abp-registry/2026Q3",
+    "drawSeed": 424242
+  },
+  "prompt": "…full task statement given to every engine…",
+  "environment": {
+    "image": "ghcr.io/macanderson/abp-node:22",
+    "setup": ["pnpm install --frozen-lockfile"]
+  },
+  "verification": {
+    "kind": "commands",
+    "failToPass": ["pnpm test -- test/router/edge-case.test.ts"],
+    "passToPass": ["pnpm test -- test/router/"],
+    "timeoutSeconds": 900,
+    "heldOut": true
+  },
+  "golden": {
+    "diffBytesNormalized": 1843,
+    "filesTouched": 2,
+    "trace": "traces/gh-vercel-next.js-81234.abp-trace.jsonl",
+    "mergedPr": "https://github.com/vercel/next.js/pull/81301"
+  },
+  "cutoffsVerified": ["claude-sonnet-5:2026-01", "gpt-5.2:2026-03"]
+}
+

+ Synthetic fixtures (like Arena's built-in tasks) use the same schema with + source.kind: "fixture" and no golden block. A harness must treat + both identically at run time. +

+ +

6. Trace format (abp-trace/v1)

+

One JSONL file per (task, engine, trial). Every line is an event:

+ 
{"t":"2026-07-02T14:12:01.402Z","seq":1,"kind":"run.start",
+ "engine":"oxagen/1.4.2","model":"anthropic/claude-sonnet-5",
+ "task":"gh-vercel-next.js-81234","config":{"budgetUsd":5,"timeoutSeconds":1200}}
+{"t":"…","seq":2,"kind":"model.call",
+ "tokens":{"input":1204,"output":388,"cacheRead":9120,"cacheWrite":0}}
+{"t":"…","seq":3,"kind":"tool.call","tool":"read_file",
+ "args":{"path":"src/router/match.ts"},"durationMs":12}
+{"t":"…","seq":4,"kind":"tool.call","tool":"edit_file",
+ "args":{"path":"src/router/match.ts"},"diffBytes":412}
+{"t":"…","seq":5,"kind":"subagent.start","id":"sa-1","purpose":"run tests"}
+{"t":"…","seq":6,"kind":"run.end","outcome":"resolved","wallClockMs":184201,
+ "tokens":{"input":48210,"output":9114,"cacheRead":301200,"cacheWrite":8100},
+ "diffBytesNormalized":1610,"filesTouched":2}
+

Rules:

+
    +
  • seq is a strictly increasing integer; t is RFC 3339 UTC. Together they make traces diffable and mergeable.
    • +
  • tool.call events record the tool name and a redacted argument summary, never raw secrets. A published redaction list is part of conformance.
    • +
  • Multi-agent engines emit subagent.start / subagent.end with nested attribution, so a fan-out engine's token bill is visible per branch. Multi-model engines record model per model.call. This is how ABP stays meaningful for orchestrators, not just single-loop CLIs.
    • +
  • Token counts follow Arena's normalization: input never includes cache reads; cache reads and writes are tracked separately.
    • +
  • An engine that cannot emit ABP traces natively can ship a sidecar translator; the harness records which path produced the trace.
    • +
+ +

7. Run configuration (abp-run/v1)

+

The manifest a harness must write before the first trial:

+ 
{
+  "schema": "abp-run/v1",
+  "engines": [{"name": "oxagen", "version": "1.4.2"},
+              {"name": "claude-code", "version": "2.1.0"}],
+  "model": "anthropic/claude-sonnet-5",
+  "matchedModels": true,
+  "budgetUsd": 5,
+  "timeoutSeconds": 1200,
+  "trials": 3,
+  "ordering": "abba",
+  "seed": 20260718,
+  "tasks": ["gh-vercel-next.js-81234", "…"],
+  "host": {"os": "linux", "arch": "arm64", "containerized": true},
+  "reproduce": "abp run --config run.json"
+}
+

+ matchedModels: false runs are legal (you may want to test your engine across + models) but a conforming report must lead with the warning and must not present + engine-vs-engine conclusions from them. +

+ +

8. Verdict format (abp-verdict/v1)

+ 
{
+  "schema": "abp-verdict/v1",
+  "run": "run-20260718-a41c",
+  "perEngine": [{
+    "engine": "oxagen",
+    "scoredTrials": 72,
+    "excludedEngineErrors": 1,
+    "resolveRate": {"point": 0.84, "wilson95": [0.71, 0.92]},
+    "medianDiffBytesNormalized": 1650,
+    "medianTokensTotal": 361000,
+    "medianWallClockMs": 190334,
+    "medianComputedUsd": 1.42
+  }],
+  "pairwise": [{
+    "a": "oxagen", "b": "claude-code",
+    "mcnemar": {"discordant": [14, 5], "pExact": 0.041},
+    "diffDelta": {"relMedian": -0.22, "bootstrap95": [-0.31, -0.09],
+                  "seed": 20260718}
+  }],
+  "receipts": {"trials": "trials/", "transcripts": "transcripts/",
+               "diffs": "diffs/", "traces": "traces/"}
+}
+ +

9. Conformance levels

+
    +
  • ABP-core. The harness consumes abp-task/v1, enforces held-out verification and matched config, and emits abp-run/v1 + abp-verdict/v1 with the required statistics. Arena today is one file-format migration away from ABP-core.
    • +
  • ABP-traces. Everything in core, plus abp-trace/v1 emission for every trial, with subagent and multi-model attribution.
    • +
  • ABP-live. Everything in traces, plus participation in the live task pipeline: consuming registry draws, contributing verified fixes, and publishing golden traces.
    • +
+

+ Conformance is claimed by shipping a passing run of the public conformance suite (a set of + adversarial fixtures: a task that tries to plant verification files, an envelope that lies + about tokens, a trial that must be scored engine-error) and linking the receipts. +

+ +

10. CI integration

+

+ The reason ABP should win: it is the only benchmark designed to be run on every pull + request, not once per launch. +

+ 
# .github/workflows/agent-quality.yml
+- run: abp run --config abp-run.json -o results
+- run: abp gate results/run-latest --baseline abp-baseline.json --require-significant
+

+ The gate semantics are Arena's, generalized: fail on a statistically significant + resolve-rate drop, or on median token, cost, diff-size, or wall-clock growth past + committed thresholds; refuse to compare across mismatched task sets. The baseline is a + committed JSON artifact, so the PR that degrades the agent goes red the same day, with + receipts attached. +

+ +

11. Roadmap

+
+ + + + + + + + + +
PhaseDeliverableExit criterion
0 (now)Arena as reference harness; this draft public at arena.oxagen.shDraft survives public review
1Schemas frozen at v1; Arena emits and consumes all four formats; conformance suite publishedTwo non-Oxagen engines emit valid traces
2Live pipeline pilot: 25-repo registry subset, 100 verified tasks, golden traces publishedExternal team reproduces a published verdict from receipts alone
3Registry at 500 repos; hosted leaderboard with matched-model brackets; GitHub Action in the marketplaceABP gate running in 100 external repos' CI
4Version 2 exploration: trajectory quality, architectural review, multi-agent orchestration scoringCommunity RFC process
+
+ +

12. Open questions

+
    +
  1. Trace redaction: how much tool-argument detail can be published from runs on private code without leaking it? Current answer: conformance requires the redaction list, and private runs may withhold traces while still claiming ABP-core.
    2. +
  2. Who signs verified tasks? A task's freeze bundle should be content-addressed and signed by the registry, or a poisoned mirror can grade dishonestly.
    3. +
  3. Diff normalization is specified per-language (formatter canonicalization). The v1 scope is the languages with deterministic formatters; the escape hatch is byte counts on git diff --numstat with published exclusions.
    4. +
  4. Should golden traces ever become the rubric? Version 1 says no. If v2 explores it, trace-similarity grading must never punish a better route to a passing fix.
    5. +
+ +

+ Feedback: open an issue at + github.com/macanderson/arena with the + abp label. The markdown source of this spec lives in the repo at + docs/agent-benchmark-protocol.md. +

+
+
+
+ +
+ +
+ + diff --git a/site/style.css b/site/style.css new file mode 100644 index 0000000..565dc55 --- /dev/null +++ b/site/style.css @@ -0,0 +1,691 @@ +/* Arena — arena.oxagen.sh + Light "evidence" page, one dark scoreboard signature element. + Mono display (transcripts, diffs, receipts are the brand material). */ + +:root { + --paper: #f7f8f7; + --ink: #14171a; + --muted: #5c6670; + --line: #dde1de; + --panel: #101317; + --panel-line: #262c33; + --amber: #ffb020; + --amber-dim: #b98217; + --pass: #0b7a3e; + --fail: #c6362c; + --link: #0b5aa2; + --mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, monospace; + --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, + Arial, sans-serif; + --serif: Charter, "Iowan Old Style", Georgia, Cambria, serif; +} + +@media (prefers-color-scheme: dark) { + :root { + --paper: #0e1114; + --ink: #e8eaec; + --muted: #98a2ab; + --line: #232930; + --panel: #090b0d; + --panel-line: #232930; + --link: #6db3f2; + } +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +html { + scroll-behavior: smooth; +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + * { + animation: none !important; + transition: none !important; + } +} + +body { + background: var(--paper); + color: var(--ink); + font-family: var(--sans); + font-size: 17px; + line-height: 1.6; + -webkit-font-smoothing: antialiased; +} + +a { + color: var(--link); + text-decoration-thickness: 1px; + text-underline-offset: 3px; +} + +a:focus-visible, +button:focus-visible { + outline: 2px solid var(--link); + outline-offset: 3px; + border-radius: 2px; +} + +.wrap { + max-width: 880px; + margin: 0 auto; + padding: 0 24px; +} + +/* ---------- header ---------- */ + +.site-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 16px; + padding: 28px 0 0; + font-family: var(--mono); + font-size: 14px; +} + +.site-head .brand { + color: var(--ink); + text-decoration: none; + font-weight: 700; + letter-spacing: 0.02em; +} + +.site-head .brand span { + color: var(--muted); + font-weight: 400; +} + +.site-head nav { + display: flex; + gap: 20px; + flex-wrap: wrap; +} + +.site-head nav a { + color: var(--muted); + text-decoration: none; +} + +.site-head nav a:hover, +.site-head nav a[aria-current="page"] { + color: var(--ink); +} + +/* ---------- hero ---------- */ + +.hero { + padding: 88px 0 40px; +} + +.hero h1 { + font-family: var(--mono); + font-weight: 700; + font-size: clamp(30px, 5.4vw, 52px); + line-height: 1.12; + letter-spacing: -0.015em; + max-width: 21ch; +} + +.hero .sub { + margin-top: 24px; + max-width: 58ch; + font-size: 19px; + color: var(--muted); +} + +.hero .sub strong { + color: var(--ink); + font-weight: 600; +} + +.cta-row { + display: flex; + gap: 14px; + flex-wrap: wrap; + margin-top: 32px; + font-family: var(--mono); + font-size: 14px; +} + +.btn { + display: inline-block; + padding: 11px 18px; + border-radius: 6px; + text-decoration: none; + border: 1px solid var(--ink); +} + +.btn.solid { + background: var(--ink); + color: var(--paper); +} + +.btn.ghost { + color: var(--ink); +} + +.btn:hover { + transform: translateY(-1px); +} + +/* ---------- scoreboard (signature) ---------- */ + +.scoreboard { + margin: 48px 0 8px; + background: var(--panel); + border: 1px solid var(--panel-line); + border-radius: 10px; + overflow: hidden; + font-family: var(--mono); + color: #cfd6dd; + box-shadow: 0 24px 60px -32px rgb(0 0 0 / 0.5); +} + +.scoreboard .bar { + display: flex; + justify-content: space-between; + gap: 12px; + padding: 12px 18px; + border-bottom: 1px solid var(--panel-line); + font-size: 12.5px; + color: #8a949e; + flex-wrap: wrap; +} + +.scoreboard .bar .cmd { + color: #cfd6dd; + overflow-wrap: anywhere; +} + +.scoreboard .bar .cmd::before { + content: "$ "; + color: var(--amber); +} + +.sb-body { + padding: 20px 18px 8px; + overflow-x: auto; +} + +.sb-row { + display: grid; + grid-template-columns: 130px 1fr 150px; + gap: 14px; + align-items: center; + padding: 10px 0; + min-width: 520px; +} + +.sb-row .name { + font-size: 14px; + color: #e8eaec; +} + +.sb-row .rate { + font-size: 14px; + color: var(--amber); + text-align: right; + white-space: nowrap; +} + +.sb-row .rate small { + color: #8a949e; + font-size: 11.5px; + display: block; +} + +.ci { + position: relative; + height: 14px; + background: #1b2129; + border-radius: 7px; +} + +.ci .band { + position: absolute; + top: 3px; + bottom: 3px; + background: rgb(255 176 32 / 0.28); + border-radius: 4px; +} + +.ci .pt { + position: absolute; + top: 0; + bottom: 0; + width: 3px; + background: var(--amber); + border-radius: 2px; +} + +.sb-foot { + display: flex; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + padding: 12px 18px 14px; + border-top: 1px solid var(--panel-line); + font-size: 12.5px; + color: #8a949e; +} + +.sb-foot .p { + color: #cfd6dd; +} + +.sb-note { + font-family: var(--mono); + font-size: 12.5px; + color: var(--muted); + margin-bottom: 56px; +} + +/* ---------- sections ---------- */ + +section { + padding: 56px 0 8px; +} + +.eyebrow { + font-family: var(--mono); + font-size: 12.5px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--muted); + margin-bottom: 10px; +} + +section h2 { + font-family: var(--mono); + font-size: clamp(22px, 3vw, 30px); + font-weight: 700; + letter-spacing: -0.01em; + line-height: 1.2; + max-width: 30ch; +} + +section > .wrap > p, +section .prose p { + margin-top: 18px; + max-width: 66ch; +} + +section .prose p + p { + margin-top: 14px; +} + +.muted { + color: var(--muted); +} + +/* defense table */ + +.defense { + margin-top: 28px; + border-top: 1px solid var(--line); +} + +.defense .row { + display: grid; + grid-template-columns: minmax(180px, 2fr) 3fr; + gap: 8px 28px; + padding: 16px 0; + border-bottom: 1px solid var(--line); +} + +.defense .attack { + font-family: var(--mono); + font-size: 14px; + font-weight: 600; +} + +.defense .attack::before { + content: "“"; + color: var(--muted); +} + +.defense .attack::after { + content: "”"; + color: var(--muted); +} + +.defense .answer { + font-size: 15.5px; + color: var(--muted); +} + +.defense .answer strong { + color: var(--ink); + font-weight: 600; +} + +@media (max-width: 640px) { + .defense .row { + grid-template-columns: 1fr; + } +} + +/* metric cards */ + +.metrics { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); + gap: 14px; + margin-top: 28px; +} + +.metric { + border: 1px solid var(--line); + border-radius: 8px; + padding: 16px; +} + +.metric .k { + font-family: var(--mono); + font-size: 13px; + font-weight: 700; +} + +.metric .v { + margin-top: 6px; + font-size: 14px; + color: var(--muted); +} + +/* code block */ + +pre.shell { + margin-top: 24px; + background: var(--panel); + color: #cfd6dd; + border: 1px solid var(--panel-line); + border-radius: 10px; + padding: 18px; + font-family: var(--mono); + font-size: 13.5px; + line-height: 1.7; + overflow-x: auto; +} + +pre.shell .c { + color: #6f7a84; +} + +pre.shell .y { + color: var(--amber); +} + +/* doc cards */ + +.docs { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 14px; + margin-top: 28px; +} + +.doc-card { + display: block; + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + text-decoration: none; + color: var(--ink); +} + +.doc-card:hover { + border-color: var(--muted); +} + +.doc-card .t { + font-family: var(--mono); + font-size: 14.5px; + font-weight: 700; +} + +.doc-card .d { + margin-top: 8px; + font-size: 14.5px; + color: var(--muted); +} + +.doc-card .m { + margin-top: 12px; + font-family: var(--mono); + font-size: 12px; + color: var(--muted); +} + +/* ---------- footer ---------- */ + +.site-foot { + margin-top: 88px; + border-top: 1px solid var(--line); + padding: 28px 0 48px; + font-family: var(--mono); + font-size: 13px; + color: var(--muted); + display: flex; + justify-content: space-between; + gap: 16px; + flex-wrap: wrap; +} + +.site-foot a { + color: var(--muted); +} + +/* ---------- article pages (paper, spec) ---------- */ + +.article { + padding: 64px 0 24px; +} + +.article .kicker { + font-family: var(--mono); + font-size: 12.5px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--muted); +} + +.article h1 { + margin-top: 14px; + font-family: var(--mono); + font-weight: 700; + font-size: clamp(26px, 4vw, 40px); + line-height: 1.15; + letter-spacing: -0.015em; + max-width: 26ch; +} + +.article .byline { + margin-top: 18px; + font-family: var(--mono); + font-size: 13px; + color: var(--muted); +} + +.article .abstract { + margin-top: 28px; + padding: 20px 22px; + border: 1px solid var(--line); + border-radius: 8px; + font-size: 16px; + color: var(--muted); + max-width: 72ch; +} + +.article .abstract strong { + color: var(--ink); +} + +.article-body { + font-family: var(--serif); + font-size: 18px; + line-height: 1.72; + max-width: 72ch; + padding-bottom: 40px; +} + +.article-body h2 { + font-family: var(--mono); + font-size: 22px; + font-weight: 700; + letter-spacing: -0.01em; + margin: 52px 0 4px; + line-height: 1.3; +} + +.article-body h3 { + font-family: var(--mono); + font-size: 16.5px; + font-weight: 700; + margin: 36px 0 0; +} + +.article-body p { + margin-top: 16px; +} + +.article-body ul, +.article-body ol { + margin: 16px 0 0 26px; +} + +.article-body li { + margin-top: 8px; +} + +.article-body li::marker { + color: var(--muted); + font-family: var(--mono); + font-size: 14px; +} + +.article-body code { + font-family: var(--mono); + font-size: 0.82em; + background: rgb(127 127 127 / 0.12); + padding: 2px 5px; + border-radius: 4px; +} + +.article-body pre { + margin-top: 18px; + background: var(--panel); + color: #cfd6dd; + border: 1px solid var(--panel-line); + border-radius: 10px; + padding: 18px; + font-family: var(--mono); + font-size: 13px; + line-height: 1.65; + overflow-x: auto; +} + +.article-body pre code { + background: none; + padding: 0; + font-size: 1em; +} + +.article-body blockquote { + margin-top: 18px; + padding-left: 18px; + border-left: 3px solid var(--line); + color: var(--muted); +} + +.article-body table { + margin-top: 20px; + border-collapse: collapse; + width: 100%; + font-family: var(--sans); + font-size: 14.5px; +} + +.article-body .tablewrap { + overflow-x: auto; + margin-top: 20px; +} + +.article-body .tablewrap table { + margin-top: 0; + min-width: 640px; +} + +.article-body th { + text-align: left; + font-family: var(--mono); + font-size: 12.5px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + border-bottom: 1px solid var(--ink); + padding: 8px 12px 8px 0; + vertical-align: bottom; +} + +.article-body td { + border-bottom: 1px solid var(--line); + padding: 10px 12px 10px 0; + vertical-align: top; +} + +.article-body .refs { + font-family: var(--sans); + font-size: 14.5px; + color: var(--muted); +} + +.article-body .refs li { + margin-top: 10px; + overflow-wrap: anywhere; +} + +.article-body sup a { + text-decoration: none; + font-family: var(--mono); + font-size: 0.72em; +} + +.toc { + margin-top: 28px; + padding: 18px 22px; + border: 1px solid var(--line); + border-radius: 8px; + font-family: var(--mono); + font-size: 13.5px; + max-width: 72ch; +} + +.toc .t { + font-weight: 700; + margin-bottom: 10px; +} + +.toc ol { + margin-left: 20px; +} + +.toc li { + margin-top: 6px; +} + +.toc a { + text-decoration: none; +} + +.toc a:hover { + text-decoration: underline; +} diff --git a/site/vercel.json b/site/vercel.json new file mode 100644 index 0000000..530b6a0 --- /dev/null +++ b/site/vercel.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "cleanUrls": true, + "trailingSlash": true, + "headers": [ + { + "source": "/(.*)", + "headers": [ + { "key": "X-Content-Type-Options", "value": "nosniff" }, + { "key": "Referrer-Policy", "value": "strict-origin-when-cross-origin" } + ] + } + ] +} diff --git a/src/adapters/base.ts b/src/adapters/base.ts index a2d84c5..6dd9769 100644 --- a/src/adapters/base.ts +++ b/src/adapters/base.ts @@ -95,6 +95,13 @@ export abstract class Adapter { /** * Run the agent. Overridable (the mock adapter runs in-process). argv arrays * only — nothing is ever interpolated through a shell. + * + * On POSIX the child gets its own process group so a timeout (or exit) can + * kill the agent's whole process tree, not just the direct child: coding + * agents routinely spawn shells, watchers, and servers, and a surviving + * grandchild could otherwise hold the stdio pipes open forever (hanging the + * run past its timeout) or keep mutating the workspace while the held-out + * verifier runs. */ execute(args: AdapterRunArgs): Promise { return new Promise((resolve) => { @@ -113,6 +120,7 @@ export abstract class Adapter { let stderr = ""; let timedOut = false; let settled = false; + let drainTimer: NodeJS.Timeout | undefined; const killTree = (): void => { if (detached && child.pid !== undefined) { @@ -126,34 +134,50 @@ export abstract class Adapter { child.kill("SIGKILL"); }; - const timer = setTimeout(() => { - timedOut = true; - killTree(); - }, args.timeoutSeconds * 1000); - const settle = (outcome: ExecOutcome): void => { if (settled) return; settled = true; clearTimeout(timer); + if (drainTimer !== undefined) clearTimeout(drainTimer); resolve(outcome); }; - child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); - child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + const timer = setTimeout(() => { + timedOut = true; + killTree(); + }, args.timeoutSeconds * 1000); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => (stdout += chunk)); + child.stderr.on("data", (chunk: string) => (stderr += chunk)); child.on("error", (err) => { - settle({ stdout, stderr, exitCode: null, timedOut: false, spawnError: err.message }); + settle({ + stdout, + stderr, + exitCode: null, + timedOut: false, + spawnError: err.message, + }); }); + // "close" waits for the stdio pipes to drain, which an orphaned + // grandchild that inherited them can prevent forever. "exit" is the + // authoritative signal: reap anything left in the process group, then + // give the pipes a short grace period to flush before settling. The + // grace timer is NOT unref'd: it must keep the event loop alive so the + // promise always settles even if "close" never fires. + child.on("exit", (code) => { + clearTimeout(timer); + killTree(); + drainTimer = setTimeout(() => { + settle({ stdout, stderr, exitCode: code, timedOut }); + }, 2000); + }); child.on("close", (code) => { settle({ stdout, stderr, exitCode: code, timedOut }); }); - - // "close" waits for the stdio streams to drain; an escaped grandchild - // holding the pipes must not stall the run forever after exit. - child.on("exit", (code) => { - setTimeout(() => settle({ stdout, stderr, exitCode: code, timedOut }), 2000).unref(); - }); }); } } diff --git a/src/adapters/claude-code.ts b/src/adapters/claude-code.ts index 96e2b7e..845edf8 100644 --- a/src/adapters/claude-code.ts +++ b/src/adapters/claude-code.ts @@ -21,11 +21,15 @@ export class ClaudeCodeAdapter extends Adapter { readonly name = "claude-code"; protected readonly defaultBinary = "claude"; - /** `anthropic/claude-sonnet-5` → `sonnet`; unrecognized slugs pass through. */ + /** + * `anthropic/claude-sonnet-5` → `claude-sonnet-5`; bare ids and aliases + * pass through. The full id is preserved (never collapsed to a family + * alias like `sonnet`): an alias floats to whatever the installed CLI's + * default is, which silently breaks version pinning, `matchedModels`, and + * pricing-table lookups. + */ override resolveModel(model: string): string { - const bare = model.replace(/^anthropic\//, ""); - const family = bare.match(/^claude-(opus|sonnet|haiku|fable)/)?.[1]; - return family ?? bare; + return model.replace(/^anthropic\//, ""); } buildArgs(args: AdapterRunArgs): string[] { diff --git a/src/adapters/gemini.ts b/src/adapters/gemini.ts index 2ae1f5f..390e7b4 100644 --- a/src/adapters/gemini.ts +++ b/src/adapters/gemini.ts @@ -8,9 +8,13 @@ * `--permission-mode acceptEdits` (edits auto-approved, arbitrary shell not). * * JSON output shape (gemini-cli ≥ 0.x): { response, stats: { models: { - * "": { tokens: { prompt, candidates, cached, total, ... } } } } } + * "": { tokens: { prompt, candidates, thoughts, tool, cached, + * total, ... } } } } } * Multiple model entries are summed. Gemini's `prompt` count INCLUDES cached - * tokens, so normalized input = prompt − cached. + * tokens, so normalized input = prompt + tool − cached. `thoughts` (reasoning) + * tokens are billed as output, so normalized output = candidates + thoughts — + * dropping them would systematically understate Gemini vs. agents whose + * output counts already include reasoning. * * NOTE: gemini-cli flags/envelope evolve quickly; `arena doctor` surfaces the * installed version, and this adapter is covered by unit tests over a captured @@ -52,6 +56,8 @@ export class GeminiAdapter extends Adapter { let prompt = 0; let candidates = 0; + let thoughts = 0; + let tool = 0; let cached = 0; let toolCalls: number | null = null; for (const entry of Object.values(models)) { @@ -59,6 +65,8 @@ export class GeminiAdapter extends Adapter { const tokens = isRecord(entry["tokens"]) ? entry["tokens"] : {}; prompt += num(tokens["prompt"]); candidates += num(tokens["candidates"]); + thoughts += num(tokens["thoughts"]); + tool += num(tokens["tool"]); cached += num(tokens["cached"]); } const tools = isRecord(stats["tools"]) ? stats["tools"] : {}; @@ -70,8 +78,8 @@ export class GeminiAdapter extends Adapter { const cacheRead = Math.min(cached, prompt); return { tokens: totalize({ - input: prompt - cacheRead, - output: candidates, + input: prompt + tool - cacheRead, + output: candidates + thoughts, cacheRead, cacheWrite: 0, }), diff --git a/src/cli.ts b/src/cli.ts index 8fdb19c..0cc4715 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -137,7 +137,7 @@ async function cmdRun(argv: string[]): Promise { budget: { type: "string" }, timeout: { type: "string" }, seed: { type: "string" }, - out: { type: "string" }, + out: { type: "string", short: "o" }, }, }); @@ -157,6 +157,18 @@ async function cmdRun(argv: string[]): Promise { return { adapter: adapter.trim(), model: resolved.trim() }; }); + // Duplicate (adapter, model) specs would produce colliding trial ids and + // silently overwrite each other's receipts. + const specKeys = agents.map((a) => `${a.adapter}=${a.model}`); + const dupes = [...new Set(specKeys.filter((k, i) => specKeys.indexOf(k) !== i))]; + if (dupes.length > 0) { + console.error( + `Duplicate agent spec(s): ${dupes.join(", ")}. ` + + "Trial ids would collide — run A/A comparisons as two separate runs.", + ); + process.exit(1); + } + const allTasks = loadTasks(TASK_ROOT); let tasks: LoadedTask[] = allTasks; if (values.tasks && values.tasks !== "all") { @@ -225,6 +237,15 @@ function cmdReport(argv: string[]): void { */ async function cmdVerify(argv: string[]): Promise { const all = loadTasks(TASK_ROOT); + if (argv.length > 0) { + // An id that matches nothing must fail loudly: CI configured with a + // renamed task would otherwise "audit" zero tasks and pass forever. + const missing = argv.filter((id) => !all.some((t) => t.id === id)); + if (missing.length > 0) { + console.error(`Unknown task id(s): ${missing.join(", ")}. Run \`arena list\`.`); + process.exit(1); + } + } const tasks = argv.length > 0 ? all.filter((t) => argv.includes(t.id)) : all; let failures = 0; @@ -337,20 +358,25 @@ function cmdGate(argv: string[]): void { const configPath = values.config ?? (existsSync(resolve("arena-gate.json")) ? "arena-gate.json" : undefined); const thresholds: GateThresholds = loadGateConfig(configPath ? resolve(configPath) : undefined); - // A finite number, or null (a non-numeric flag value disables the check). - const num = (v: string | undefined): number | null => { + // Strict: a typo'd threshold ("--tokens-increase ten") must error, not + // silently disable the guard it was meant to set. + const num = (flag: string, v: string | undefined): number | null => { if (v === undefined) return null; - const n = parseFloat(v); - return Number.isFinite(n) ? n : null; + const n = Number(v); + if (!Number.isFinite(n)) { + console.error(`--${flag} must be a number, got "${v}"`); + process.exit(1); + } + return n; }; if (values["accuracy-drop"] !== undefined) - thresholds.accuracyMaxDropPoints = num(values["accuracy-drop"]) ?? 0; + thresholds.accuracyMaxDropPoints = num("accuracy-drop", values["accuracy-drop"]) ?? 0; if (values["tokens-increase"] !== undefined) - thresholds.tokensMaxIncreasePct = num(values["tokens-increase"]); + thresholds.tokensMaxIncreasePct = num("tokens-increase", values["tokens-increase"]); if (values["cost-increase"] !== undefined) - thresholds.costMaxIncreasePct = num(values["cost-increase"]); + thresholds.costMaxIncreasePct = num("cost-increase", values["cost-increase"]); if (values["speed-increase"] !== undefined) - thresholds.speedMaxIncreasePct = num(values["speed-increase"]); + thresholds.speedMaxIncreasePct = num("speed-increase", values["speed-increase"]); if (values["require-significant"]) thresholds.accuracyRequireSignificant = true; if (values["allow-task-mismatch"]) thresholds.allowTaskMismatch = true; diff --git a/src/orchestrator.ts b/src/orchestrator.ts index 6aad23c..2177e8f 100644 --- a/src/orchestrator.ts +++ b/src/orchestrator.ts @@ -99,6 +99,9 @@ export async function executeRun( const result = await runOne(task, spec, adapter, trial, config, runId, runDir, pricing); results.push(result); writeFileSync(join(runDir, "trials", `${result.id}.json`), JSON.stringify(result, null, 2)); + // Rewritten after every trial so a mid-run crash still leaves an + // aggregatable results.json behind. + writeFileSync(join(runDir, "results.json"), JSON.stringify({ manifest, results }, null, 2)); const mark = result.outcome === "passed" ? "✅" : result.outcome === "agent-error" ? "⚠️" : "❌"; log( @@ -109,8 +112,6 @@ export async function executeRun( } } - writeFileSync(join(runDir, "results.json"), JSON.stringify({ manifest, results }, null, 2)); - return { runDir, manifest, results }; } @@ -125,12 +126,14 @@ async function runOne( pricing: PricingTable, ): Promise { const workDir = join(tmpdir(), `arena-${sanitizeSegment(task.id)}-${randomUUID()}`); - const startedAt = new Date(); const id = `${task.id}-${spec.adapter}-${sanitizeSegment(spec.model)}-t${trial}`; + let startedAt = new Date(); try { await seedWorkspace(task, workDir); + // Started only now: wall clock measures spawn-to-exit, not harness seeding. + startedAt = new Date(); const exec = await adapter.execute({ prompt: buildPrompt(task), model: spec.model, @@ -143,14 +146,21 @@ async function runOne( const finishedAt = new Date(); const wallClockSeconds = (finishedAt.getTime() - startedAt.getTime()) / 1000; - const diff = await collectDiff(workDir); + const diffRaw = await collectDiff(workDir); + const diff = diffRaw ?? ""; const envelope = adapter.parseEnvelope(exec.stdout); // Invocation-level failure: the agent never actually ran (bad flags, - // missing binary, immediate non-zero exit with no work product). + // missing binary, immediate non-zero exit with no work product). A null + // diff means git failed — the diff is unknown, not empty, so it can never + // count as evidence that the agent did nothing. const invocationFailure = exec.spawnError !== undefined || - (exec.exitCode !== 0 && !exec.timedOut && diff.length === 0 && envelope.tokens.total === 0); + (exec.exitCode !== 0 && + !exec.timedOut && + diffRaw !== null && + diff.length === 0 && + envelope.tokens.total === 0); let outcome: Outcome; let verify = { passed: false, output: "" }; @@ -232,6 +242,57 @@ async function runOne( diffPath, ...(errorText !== undefined ? { error: errorText } : {}), }; + } catch (error) { + // Per-trial containment: a harness-side failure (git error, FS hiccup, + // missing fixture) scores this one trial `agent-error` — consistent with + // the project rule that harness failures never read as the agent losing — + // instead of aborting the whole run and losing every completed trial. + const finishedAt = new Date(); + const message = error instanceof Error ? error.message : String(error); + const transcriptPath = join("transcripts", `${id}.txt`); + const diffPath = join("diffs", `${id}.patch`); + try { + writeFileSync(join(runDir, transcriptPath), `# ${id}\n# harness error\n\n${message}\n`); + writeFileSync(join(runDir, diffPath), ""); + } catch { + // Receipts are best-effort here; the error itself is preserved below. + } + return { + id, + taskId: task.id, + trial, + agent: { + adapter: spec.adapter, + model: spec.model, + resolvedModel: adapter.resolveModel(spec.model), + version: adapter.version(), + bin: adapter.bin(), + }, + outcome: "agent-error", + verify: { passed: false, output: "" }, + timing: { + wallClockSeconds: (finishedAt.getTime() - startedAt.getTime()) / 1000, + agentReportedSeconds: null, + }, + tokens: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + cost: { computedUsd: null, agentReportedUsd: null, pricingModel: null }, + activity: { + toolCalls: null, + iterations: null, + filesTouched: 0, + linesAdded: 0, + linesRemoved: 0, + diffBytes: 0, + }, + provenance: { + runId, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + }, + transcriptPath, + diffPath, + error: `harness error: ${message}`, + }; } finally { rmSync(workDir, { recursive: true, force: true }); } diff --git a/src/parse.ts b/src/parse.ts index 963a4c0..f484d03 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -66,33 +66,53 @@ function tryParseRecord(text: string): Record | null { } } -/** Split text into top-level `{…}` runs, brace-matched and JSON-string-aware. */ +/** + * Split text into top-level `{…}` runs, brace-matched and JSON-string-aware. + * Candidates are anchored at lines that START with `{` (a pretty-printed + * envelope's opening line): a stray brace mid-way through a log line can + * therefore never derail the scan into treating the rest of the output as + * one unterminated string. + */ function topLevelJsonBlocks(text: string): string[] { const blocks: string[] = []; + let offset = 0; + let consumedUpTo = -1; + for (const line of text.split("\n")) { + const lineStart = offset; + offset += line.length + 1; + if (lineStart < consumedUpTo) continue; // inside a previous block + const firstNonSpace = line.search(/\S/); + if (firstNonSpace === -1 || line[firstNonSpace] !== "{") continue; + const start = lineStart + firstNonSpace; + const end = balancedObjectEnd(text, start); + if (end === -1) continue; + blocks.push(text.slice(start, end + 1)); + consumedUpTo = end + 1; + } + return blocks; +} + +/** Index of the `}` closing the object opened at `start`, or -1. */ +function balancedObjectEnd(text: string, start: number): number { let depth = 0; - let start = -1; let inString = false; let escaped = false; - for (let i = 0; i < text.length; i++) { + for (let i = start; i < text.length; i++) { const ch = text[i]; if (inString) { if (escaped) escaped = false; else if (ch === "\\") escaped = true; else if (ch === '"') inString = false; - } else if (ch === '"') { - if (depth > 0) inString = true; - } else if (ch === "{") { - if (depth === 0) start = i; - depth++; - } else if (ch === "}" && depth > 0) { + continue; + } + if (ch === '"') inString = true; + else if (ch === "{") depth++; + else if (ch === "}") { depth--; - if (depth === 0 && start !== -1) { - blocks.push(text.slice(start, i + 1)); - start = -1; - } + if (depth === 0) return i; } } - return blocks; + return -1; } /** Make a string safe as a single path segment (model slugs contain "/"). */ diff --git a/src/pricing.ts b/src/pricing.ts index b351de2..ed756b3 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -44,11 +44,14 @@ export function computeCost( ): number | null { const rate = pricing[model]; if (!rate) return null; - const cacheWriteRate = rate.cacheWritePerM ?? rate.inputPerM; + // Cache-write rates differ per vendor. If the trial wrote cache tokens and + // the table has no cacheWritePerM, the true cost is unknown — report null + // rather than guessing with the input rate ("cost is never guessed"). + if (tokens.cacheWrite > 0 && rate.cacheWritePerM === undefined) return null; return ( (tokens.input / 1e6) * rate.inputPerM + (tokens.output / 1e6) * rate.outputPerM + (tokens.cacheRead / 1e6) * rate.cacheReadPerM + - (tokens.cacheWrite / 1e6) * cacheWriteRate + (tokens.cacheWrite / 1e6) * (rate.cacheWritePerM ?? 0) ); } diff --git a/src/report.ts b/src/report.ts index a186107..f226e07 100644 --- a/src/report.ts +++ b/src/report.ts @@ -117,6 +117,13 @@ export function generateReport(runDir: string): string { "Token counts are normalized (input excludes cache reads; cache reads and writes tracked separately). Computed cost applies one shared pricing table to every agent; “—” means the model has no pricing entry (cost is never guessed).", "", ); + const unmetered = summaries.reduce((n, s) => n + s.unmeteredTrials, 0); + if (unmetered > 0) { + lines.push( + `> ℹ️ ${String(unmetered)} scored trial(s) reported no parseable token usage (envelope missing or truncated). They count toward success rates but are excluded from token/cost medians and deltas.`, + "", + ); + } // ── Pairwise comparisons ── if (keys.length >= 2) { @@ -225,11 +232,13 @@ function pairwiseSection( "", ); + // Zero-token trials mean "usage unknown" (unparseable envelope), so token + // and cost metrics treat them as missing rather than as literal zeros. const metrics: { label: string; get: (r: TrialResult) => number | null }[] = [ { label: "Wall clock (s)", get: (r) => r.timing.wallClockSeconds }, - { label: "Total tokens", get: (r) => r.tokens.total }, - { label: "Output tokens", get: (r) => r.tokens.output }, - { label: "Computed cost (USD)", get: (r) => r.cost.computedUsd }, + { label: "Total tokens", get: (r) => (r.tokens.total > 0 ? r.tokens.total : null) }, + { label: "Output tokens", get: (r) => (r.tokens.total > 0 ? r.tokens.output : null) }, + { label: "Computed cost (USD)", get: (r) => (r.tokens.total > 0 ? r.cost.computedUsd : null) }, ]; lines.push(`| Metric | ${aKey} (median) | ${bKey} (median) | Δ relative to ${aKey} (95% CI) |`); lines.push("|---|---|---|---|"); diff --git a/src/summary.ts b/src/summary.ts index 6c8508c..cba0611 100644 --- a/src/summary.ts +++ b/src/summary.ts @@ -19,6 +19,8 @@ export interface AgentSummary { /** Trials excluding `agent-error`. */ scoredTrials: number; errorTrials: number; + /** Scored trials whose envelope had no parseable token usage (total = 0). */ + unmeteredTrials: number; passed: number; /** passed / scoredTrials (0 when nothing scored). */ successRate: number; @@ -58,7 +60,13 @@ export function perAgentSummary(results: TrialResult[]): AgentSummary[] { const scored = all.filter((r) => r.outcome !== "agent-error"); const passed = scored.filter((r) => r.outcome === "passed").length; const first = all[0] as TrialResult; - const costs = scored.map((r) => r.cost.computedUsd).filter((c): c is number => c !== null); + // A scored trial with zero total tokens means the envelope was + // unparseable (timeout kill mid-print, envelope drift), not that the + // agent used zero tokens. Excluding them keeps token/cost medians — and + // the drift gate built on them — from being dragged toward zero by parse + // failures. + const metered = scored.filter((r) => r.tokens.total > 0); + const costs = metered.map((r) => r.cost.computedUsd).filter((c): c is number => c !== null); const selfCosts = scored .map((r) => r.cost.agentReportedUsd) .filter((c): c is number => c !== null); @@ -68,11 +76,12 @@ export function perAgentSummary(results: TrialResult[]): AgentSummary[] { model: first.agent.model, scoredTrials: scored.length, errorTrials: all.length - scored.length, + unmeteredTrials: scored.length - metered.length, passed, successRate: scored.length ? passed / scored.length : 0, successCI: wilsonInterval(passed, scored.length), medianWallClockSeconds: medianOrNull(scored.map((r) => r.timing.wallClockSeconds)), - medianTotalTokens: medianOrNull(scored.map((r) => r.tokens.total)), + medianTotalTokens: medianOrNull(metered.map((r) => r.tokens.total)), medianComputedCost: medianOrNull(costs), medianAgentReportedCost: medianOrNull(selfCosts), }; diff --git a/src/workspace.ts b/src/workspace.ts index efceaab..57bfd1e 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -28,7 +28,12 @@ export function loadTasks(taskRoot: string): LoadedTask[] { const dir = join(taskRoot, entry.name); const defPath = join(dir, "task.json"); if (!existsSync(defPath)) continue; - const def: unknown = JSON.parse(readFileSync(defPath, "utf8")); + let def: unknown; + try { + def = JSON.parse(readFileSync(defPath, "utf8")); + } catch (error) { + throw new Error(`Invalid JSON in ${defPath}: ${(error as Error).message}`); + } if (!isRecord(def) || typeof def["id"] !== "string") { throw new Error(`Malformed task.json in ${dir}`); } @@ -80,8 +85,12 @@ export async function seedWorkspace(task: LoadedTask, workDir: string): Promise< await git(["commit", "-q", "-m", "arena: seed workspace", "--no-verify"]); } -/** Diff of the agent's changes (tracked + new files) against the seed commit. */ -export async function collectDiff(workDir: string): Promise { +/** + * Diff of the agent's changes (tracked + new files) against the seed commit. + * Returns null when git itself failed (oversized diff, corrupted repo) — an + * unknown diff, which callers must not conflate with "no changes". + */ +export async function collectDiff(workDir: string): Promise { try { await execFileAsync("git", ["add", "-A"], { cwd: workDir }); const { stdout } = await execFileAsync("git", ["diff", "--cached", "HEAD"], { @@ -91,7 +100,7 @@ export async function collectDiff(workDir: string): Promise { }); return stdout; } catch { - return ""; + return null; } } diff --git a/test/adapters.test.ts b/test/adapters.test.ts index 79261ee..c898d0a 100644 --- a/test/adapters.test.ts +++ b/test/adapters.test.ts @@ -27,9 +27,9 @@ describe("registry", () => { describe("claude-code", () => { const adapter = new ClaudeCodeAdapter(); - it("maps slugs to family aliases", () => { - expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe("sonnet"); - expect(adapter.resolveModel("anthropic/claude-opus-4.8")).toBe("opus"); + it("preserves the full model id (version pinning), stripping only the provider prefix", () => { + expect(adapter.resolveModel("anthropic/claude-sonnet-5")).toBe("claude-sonnet-5"); + expect(adapter.resolveModel("anthropic/claude-opus-4.8")).toBe("claude-opus-4.8"); expect(adapter.resolveModel("weird-model")).toBe("weird-model"); }); @@ -39,7 +39,7 @@ describe("claude-code", () => { "--output-format", "json", "--model", - "sonnet", + "claude-sonnet-5", "--permission-mode", "acceptEdits", "--max-budget-usd", @@ -148,13 +148,20 @@ describe("gemini", () => { expect(argv).toContain("auto_edit"); }); - it("sums per-model token stats and subtracts cached from prompt", () => { + it("sums per-model token stats, subtracts cached, and counts thoughts + tool tokens", () => { const stdout = JSON.stringify({ response: "done", stats: { models: { "gemini-2.5-pro": { - tokens: { prompt: 5000, candidates: 700, cached: 2000, total: 5700 }, + tokens: { + prompt: 5000, + candidates: 700, + thoughts: 300, + tool: 50, + cached: 2000, + total: 6050, + }, }, "gemini-2.5-flash": { tokens: { prompt: 100, candidates: 20, cached: 0, total: 120 } }, }, @@ -162,9 +169,12 @@ describe("gemini", () => { }, }); const parsed = adapter.parseEnvelope(stdout); - expect(parsed.tokens.input).toBe(5100 - 2000); + // input = prompt (5100) + tool (50) − cached (2000) + expect(parsed.tokens.input).toBe(5100 + 50 - 2000); expect(parsed.tokens.cacheRead).toBe(2000); - expect(parsed.tokens.output).toBe(720); + // output = candidates (720) + thoughts (300): reasoning tokens are billed + // as output and must not be dropped from the head-to-head comparison. + expect(parsed.tokens.output).toBe(720 + 300); expect(parsed.toolCalls).toBe(9); }); }); diff --git a/test/parse.test.ts b/test/parse.test.ts index f7a9ef5..2414de7 100644 --- a/test/parse.test.ts +++ b/test/parse.test.ts @@ -27,6 +27,39 @@ describe("parseJsonEnvelope", () => { expect(parseJsonEnvelope(stdout)).toEqual({ type: "result", ok: true }); }); + it("parses a pretty-printed (multi-line) envelope", () => { + const stdout = [ + "INFO starting", + "{", + ' "type": "result",', + ' "usage": {', + ' "input_tokens": 1200', + " }", + "}", + "INFO done", + ].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ + type: "result", + usage: { input_tokens: 1200 }, + }); + }); + + it("prefers a multi-line result envelope and survives braces inside strings", () => { + const stdout = [ + 'log with a stray { brace and "quote', + "{", + ' "type": "result",', + ' "note": "contains } and { inside a string",', + ' "n": 9', + "}", + ].join("\n"); + expect(parseJsonEnvelope(stdout)).toEqual({ + type: "result", + note: "contains } and { inside a string", + n: 9, + }); + }); + it("parses a pretty-printed (multi-line) envelope, as gemini-cli emits", () => { const stdout = [ "Loaded config", From 39acaba1daaece2a51d8ae6bf0f8e9ffc85189bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:52:59 +0000 Subject: [PATCH 17/18] chore(deps-dev): Bump vitest from 3.2.7 to 4.1.10 Bumps [vitest](https://github.com/vitest-dev/vitest/tree/HEAD/packages/vitest) from 3.2.7 to 4.1.10. - [Release notes](https://github.com/vitest-dev/vitest/releases) - [Changelog](https://github.com/vitest-dev/vitest/blob/main/docs/releases.md) - [Commits](https://github.com/vitest-dev/vitest/commits/v4.1.10/packages/vitest) --- updated-dependencies: - dependency-name: vitest dependency-version: 4.1.10 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- package.json | 2 +- pnpm-lock.yaml | 317 +++++++++++++++++-------------------------------- 2 files changed, 107 insertions(+), 212 deletions(-) diff --git a/package.json b/package.json index 33d714e..c4cf09a 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,6 @@ "@types/node": "^26.1.1", "tsx": "^4.23.1", "typescript": "^7.0.2", - "vitest": "^3.2.6" + "vitest": "^4.1.10" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9f9f19..3793385 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^7.0.2 version: 7.0.2 vitest: - specifier: ^3.2.6 - version: 3.2.7(@types/node@26.1.1)(tsx@4.23.1) + specifier: ^4.1.10 + version: 4.1.10(@types/node@26.1.1)(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1)) packages: @@ -48,28 +48,24 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [musl] '@biomejs/cli-linux-arm64@2.5.4': resolution: {integrity: sha512-pSEfW7B8kTsXUjUxC1xVVK+y85Ht3C5XxZ9gclmC7/3Ku9Vqz8jmI7k0p/BNIjQ6t4sFERI2sFeH73ybiZl6YQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [glibc] '@biomejs/cli-linux-x64-musl@2.5.4': resolution: {integrity: sha512-aby/PohmmgbShcHqFsZVzG8H6D98+P+A6xRWRrQcLW1pCjabcov5UUlke4UqNQBYTkDQav+jB4zyyDDeKB2GaA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [musl] '@biomejs/cli-linux-x64@2.5.4': resolution: {integrity: sha512-FNxojWJkL7EajAuzBgoLe0T2G0y112M4lBrDIFl/DomFTx8yqenYOIdsRLNXvOvBBofE8hJi85LjzLmBDpY7/Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [glibc] '@biomejs/cli-win32-arm64@2.5.4': resolution: {integrity: sha512-emoXexPZIPAZkz2RKmA95WJUqK3I5MJNYtwEbL5ESciRzhmFMMyekDhNG8hpeOaK+ZGRDxAU4wvGuA5IHQ0h0w==} @@ -276,79 +272,66 @@ packages: resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.62.2': resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.62.2': resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.62.2': resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.62.2': resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.62.2': resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.62.2': resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.62.2': resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.62.2': resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.62.2': resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.62.2': resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.62.2': resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.62.2': resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.62.2': resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} @@ -380,6 +363,9 @@ packages: cpu: [x64] os: [win32] + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -512,66 +498,48 @@ packages: cpu: [x64] os: [win32] - '@vitest/expect@3.2.7': - resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} + '@vitest/expect@4.1.10': + resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} - '@vitest/mocker@3.2.7': - resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} + '@vitest/mocker@4.1.10': + resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} peerDependencies: msw: ^2.4.9 - vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: msw: optional: true vite: optional: true - '@vitest/pretty-format@3.2.7': - resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + '@vitest/pretty-format@4.1.10': + resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} - '@vitest/runner@3.2.7': - resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} + '@vitest/runner@4.1.10': + resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} - '@vitest/snapshot@3.2.7': - resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} + '@vitest/snapshot@4.1.10': + resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} - '@vitest/spy@3.2.7': - resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} + '@vitest/spy@4.1.10': + resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} - '@vitest/utils@3.2.7': - resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + '@vitest/utils@4.1.10': + resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} - cac@6.7.14: - resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} - engines: {node: '>=8'} - - chai@5.3.3: - resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} - check-error@2.1.3: - resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} - engines: {node: '>= 16'} - - debug@4.4.3: - resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - deep-eql@5.0.2: - resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} - engines: {node: '>=6'} + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - es-module-lexer@1.7.0: - resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-module-lexer@2.3.1: + resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==} esbuild@0.28.1: resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} @@ -599,30 +567,21 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - js-tokens@9.0.1: - resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} - - loupe@3.2.1: - resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} - magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - nanoid@3.3.16: resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + obug@2.1.4: + resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} + engines: {node: '>=12.20.0'} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - pathval@2.0.1: - resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} - engines: {node: '>= 14.16'} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -630,8 +589,8 @@ packages: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} - postcss@8.5.19: - resolution: {integrity: sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==} + postcss@8.5.20: + resolution: {integrity: sha512-lW616l85ucIQL+FocMmL7pQFPqBmwejrCMg+iPxyImlrANNJG9NHq/RkyCZopDhd8C3LA03PHRJDjkbGu8vvug==} engines: {node: ^10 || ^12 || >=14} rollup@4.62.2: @@ -649,32 +608,22 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - std-env@3.10.0: - resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} - - strip-literal@3.1.0: - resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + std-env@4.2.0: + resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - tinyexec@0.3.2: - resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + tinyexec@1.2.4: + resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} + engines: {node: '>=18'} tinyglobby@0.2.17: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} - tinypool@1.1.1: - resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} - engines: {node: ^18.0.0 || >=20.0.0} - - tinyrainbow@2.0.0: - resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} - engines: {node: '>=14.0.0'} - - tinyspy@4.0.4: - resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} + tinyrainbow@3.1.0: + resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} tsx@4.23.1: @@ -690,11 +639,6 @@ packages: undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} - vite-node@3.2.4: - resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} - hasBin: true - vite@7.3.6: resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -735,26 +679,39 @@ packages: yaml: optional: true - vitest@3.2.7: - resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + vitest@4.1.10: + resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} hasBin: true peerDependencies: '@edge-runtime/vm': '*' - '@types/debug': ^4.1.12 - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 - '@vitest/browser': 3.2.7 - '@vitest/ui': 3.2.7 + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.1.10 + '@vitest/browser-preview': 4.1.10 + '@vitest/browser-webdriverio': 4.1.10 + '@vitest/coverage-istanbul': 4.1.10 + '@vitest/coverage-v8': 4.1.10 + '@vitest/ui': 4.1.10 happy-dom: '*' jsdom: '*' + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true - '@types/debug': + '@opentelemetry/api': optional: true '@types/node': optional: true - '@vitest/browser': + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/coverage-istanbul': + optional: true + '@vitest/coverage-v8': optional: true '@vitest/ui': optional: true @@ -960,6 +917,8 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@standard-schema/spec@1.1.0': {} + '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -1033,69 +992,54 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true - '@vitest/expect@3.2.7': + '@vitest/expect@4.1.10': dependencies: + '@standard-schema/spec': 1.1.0 '@types/chai': 5.2.3 - '@vitest/spy': 3.2.7 - '@vitest/utils': 3.2.7 - chai: 5.3.3 - tinyrainbow: 2.0.0 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + chai: 6.2.2 + tinyrainbow: 3.1.0 - '@vitest/mocker@3.2.7(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1))': + '@vitest/mocker@4.1.10(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1))': dependencies: - '@vitest/spy': 3.2.7 + '@vitest/spy': 4.1.10 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) - '@vitest/pretty-format@3.2.7': + '@vitest/pretty-format@4.1.10': dependencies: - tinyrainbow: 2.0.0 + tinyrainbow: 3.1.0 - '@vitest/runner@3.2.7': + '@vitest/runner@4.1.10': dependencies: - '@vitest/utils': 3.2.7 + '@vitest/utils': 4.1.10 pathe: 2.0.3 - strip-literal: 3.1.0 - '@vitest/snapshot@3.2.7': + '@vitest/snapshot@4.1.10': dependencies: - '@vitest/pretty-format': 3.2.7 + '@vitest/pretty-format': 4.1.10 + '@vitest/utils': 4.1.10 magic-string: 0.30.21 pathe: 2.0.3 - '@vitest/spy@3.2.7': - dependencies: - tinyspy: 4.0.4 + '@vitest/spy@4.1.10': {} - '@vitest/utils@3.2.7': + '@vitest/utils@4.1.10': dependencies: - '@vitest/pretty-format': 3.2.7 - loupe: 3.2.1 - tinyrainbow: 2.0.0 + '@vitest/pretty-format': 4.1.10 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 assertion-error@2.0.1: {} - cac@6.7.14: {} - - chai@5.3.3: - dependencies: - assertion-error: 2.0.1 - check-error: 2.1.3 - deep-eql: 5.0.2 - loupe: 3.2.1 - pathval: 2.0.1 - - check-error@2.1.3: {} + chai@6.2.2: {} - debug@4.4.3: - dependencies: - ms: 2.1.3 - - deep-eql@5.0.2: {} + convert-source-map@2.0.0: {} - es-module-lexer@1.7.0: {} + es-module-lexer@2.3.1: {} esbuild@0.28.1: optionalDependencies: @@ -1139,27 +1083,21 @@ snapshots: fsevents@2.3.3: optional: true - js-tokens@9.0.1: {} - - loupe@3.2.1: {} - magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 - ms@2.1.3: {} - nanoid@3.3.16: {} - pathe@2.0.3: {} + obug@2.1.4: {} - pathval@2.0.1: {} + pathe@2.0.3: {} picocolors@1.1.1: {} picomatch@4.0.5: {} - postcss@8.5.19: + postcss@8.5.20: dependencies: nanoid: 3.3.16 picocolors: 1.1.1 @@ -1202,26 +1140,18 @@ snapshots: stackback@0.0.2: {} - std-env@3.10.0: {} - - strip-literal@3.1.0: - dependencies: - js-tokens: 9.0.1 + std-env@4.2.0: {} tinybench@2.9.0: {} - tinyexec@0.3.2: {} + tinyexec@1.2.4: {} tinyglobby@0.2.17: dependencies: fdir: 6.5.0(picomatch@4.0.5) picomatch: 4.0.5 - tinypool@1.1.1: {} - - tinyrainbow@2.0.0: {} - - tinyspy@4.0.4: {} + tinyrainbow@3.1.0: {} tsx@4.23.1: dependencies: @@ -1254,33 +1184,12 @@ snapshots: undici-types@8.3.0: {} - vite-node@3.2.4(@types/node@26.1.1)(tsx@4.23.1): - dependencies: - cac: 6.7.14 - debug: 4.4.3 - es-module-lexer: 1.7.0 - pathe: 2.0.3 - vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) - transitivePeerDependencies: - - '@types/node' - - jiti - - less - - lightningcss - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml - vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1): dependencies: esbuild: 0.28.1 fdir: 6.5.0(picomatch@4.0.5) picomatch: 4.0.5 - postcss: 8.5.19 + postcss: 8.5.20 rollup: 4.62.2 tinyglobby: 0.2.17 optionalDependencies: @@ -1288,46 +1197,32 @@ snapshots: fsevents: 2.3.3 tsx: 4.23.1 - vitest@3.2.7(@types/node@26.1.1)(tsx@4.23.1): + vitest@4.1.10(@types/node@26.1.1)(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1)): dependencies: - '@types/chai': 5.2.3 - '@vitest/expect': 3.2.7 - '@vitest/mocker': 3.2.7(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1)) - '@vitest/pretty-format': 3.2.7 - '@vitest/runner': 3.2.7 - '@vitest/snapshot': 3.2.7 - '@vitest/spy': 3.2.7 - '@vitest/utils': 3.2.7 - chai: 5.3.3 - debug: 4.4.3 + '@vitest/expect': 4.1.10 + '@vitest/mocker': 4.1.10(vite@7.3.6(@types/node@26.1.1)(tsx@4.23.1)) + '@vitest/pretty-format': 4.1.10 + '@vitest/runner': 4.1.10 + '@vitest/snapshot': 4.1.10 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + es-module-lexer: 2.3.1 expect-type: 1.4.0 magic-string: 0.30.21 + obug: 2.1.4 pathe: 2.0.3 picomatch: 4.0.5 - std-env: 3.10.0 + std-env: 4.2.0 tinybench: 2.9.0 - tinyexec: 0.3.2 + tinyexec: 1.2.4 tinyglobby: 0.2.17 - tinypool: 1.1.1 - tinyrainbow: 2.0.0 + tinyrainbow: 3.1.0 vite: 7.3.6(@types/node@26.1.1)(tsx@4.23.1) - vite-node: 3.2.4(@types/node@26.1.1)(tsx@4.23.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 26.1.1 transitivePeerDependencies: - - jiti - - less - - lightningcss - msw - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - yaml why-is-node-running@2.3.0: dependencies: From 2e09bb182ebc1cb33460a184da4c0c88d1414aab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:09:12 -0700 Subject: [PATCH 18/18] chore(deps): Update harbor requirement in /harbor (#25) Updates the requirements on [harbor](https://github.com/harbor-framework/harbor-cookbook) to permit the latest version. - [Commits](https://github.com/harbor-framework/harbor-cookbook/commits) --- updated-dependencies: - dependency-name: harbor dependency-version: 0.20.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- harbor/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harbor/pyproject.toml b/harbor/pyproject.toml index 865bdcb..018bb29 100644 --- a/harbor/pyproject.toml +++ b/harbor/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ # Built and tested against 0.6.x; the range admits later 0.x releases the # spec-driven surface has stayed compatible with. Tighten if Harbor breaks API. dependencies = [ - "harbor>=0.6,<0.20", + "harbor>=0.6,<0.21", ] [project.optional-dependencies]