From 6cc8a979742e47f66c34c30cd2f616f66d8690be Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 21 Jul 2026 18:29:08 +0000 Subject: [PATCH 1/5] Redesign the public site as a sports wire box score Swap the marketing landing page for a data-first layout: sentence-case copy, denser standings tables, integrity as footnotes, adapters folded into Run, and a verdict-plus-ladder hero without CTA theater. Co-authored-by: Ned Cutler --- web/index.html | 16 +- web/public/favicon.svg | 4 +- web/src/App.tsx | 6 - web/src/components/Adapters.tsx | 58 -- web/src/components/Charts.tsx | 233 -------- web/src/components/Footer.tsx | 12 +- web/src/components/Hero.tsx | 42 +- web/src/components/HowItWorks.tsx | 70 +-- web/src/components/Integrity.tsx | 60 -- web/src/components/Ladder.tsx | 4 +- web/src/components/Leaderboard.tsx | 132 +++-- web/src/components/Nav.tsx | 14 +- web/src/components/Quickstart.tsx | 33 +- web/src/components/Results.tsx | 146 ----- web/src/index.css | 848 ++++++++++++----------------- web/src/lib.ts | 6 +- 16 files changed, 505 insertions(+), 1179 deletions(-) delete mode 100644 web/src/components/Adapters.tsx delete mode 100644 web/src/components/Charts.tsx delete mode 100644 web/src/components/Integrity.tsx delete mode 100644 web/src/components/Results.tsx diff --git a/web/index.html b/web/index.html index 5eae457..295dd81 100644 --- a/web/index.html +++ b/web/index.html @@ -4,31 +4,31 @@ - + - + - + - GM-Bench — Can a language model run a front office? + GM-Bench — phase one standings diff --git a/web/public/favicon.svg b/web/public/favicon.svg index fc4dfe2..18cdfe4 100644 --- a/web/public/favicon.svg +++ b/web/public/favicon.svg @@ -1,5 +1,5 @@ - + - + diff --git a/web/src/App.tsx b/web/src/App.tsx index 5845b2e..0528bef 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -4,10 +4,7 @@ import type { Leaderboard as LeaderboardData, Snapshot } from "./types"; import Nav from "./components/Nav"; import Hero from "./components/Hero"; import Leaderboard from "./components/Leaderboard"; -import Integrity from "./components/Integrity"; -import Results from "./components/Results"; import HowItWorks from "./components/HowItWorks"; -import Adapters from "./components/Adapters"; import Quickstart from "./components/Quickstart"; import Footer from "./components/Footer"; @@ -21,10 +18,7 @@ export default function App() {
- - -
diff --git a/web/src/components/Adapters.tsx b/web/src/components/Adapters.tsx deleted file mode 100644 index edd663a..0000000 --- a/web/src/components/Adapters.tsx +++ /dev/null @@ -1,58 +0,0 @@ -const ADAPTERS = [ - { - name: "Codex CLI", - body: "Read-only sandbox, ephemeral sessions, structured output against the shared GM action schema.", - snippet: "examples/codex_agent.py", - }, - { - name: "Claude Code", - body: "Single-shot prompting with no tools and no session persistence, plus an optional per-run budget cap.", - snippet: "examples/claude_agent.py", - }, - { - name: "Ollama", - body: "Fully local evaluation. Compact observation profiles keep prompts small enough for edge models.", - snippet: "examples/ollama_agent.py", - }, - { - name: "OpenAI-compatible", - body: "Point LLM_API_BASE at any chat-completions endpoint — OpenAI, or any provider that speaks the same API.", - snippet: "examples/openai_compatible_agent.py", - }, - { - name: "opencode", - body: "Route through opencode's configured provider and model catalog with a one-line environment switch.", - snippet: "examples/opencode_agent.py", - }, - { - name: "Any process", - body: "The protocol is just stdin/stdout JSON. If it can be launched as a subprocess, it can play GM.", - snippet: "--agent-cmd \"…\"", - }, -]; - -export default function Adapters() { - return ( -
-
-
-

Bring your own model

-

Adapters for the tools you already run.

-

- Every adapter speaks the same observation/action schema, so results are comparable - across providers — from frontier APIs to a laptop-hosted 4B model. -

-
-
- {ADAPTERS.map((adapter) => ( -
-

{adapter.name}

-

{adapter.body}

- {adapter.snippet} -
- ))} -
-
-
- ); -} diff --git a/web/src/components/Charts.tsx b/web/src/components/Charts.tsx deleted file mode 100644 index 74ff18d..0000000 --- a/web/src/components/Charts.tsx +++ /dev/null @@ -1,233 +0,0 @@ -import { useState } from "react"; -import type { Snapshot } from "../types"; -import { COLOR, fmt } from "../lib"; - -const W = 520; -const H = 250; -const PAD = { top: 18, right: 16, bottom: 34, left: 44 }; - -function yScale(value: number, max: number): number { - const inner = H - PAD.top - PAD.bottom; - return PAD.top + inner - (value / max) * inner; -} - -function niceMax(value: number): number { - const step = 10 ** Math.floor(Math.log10(value)); - return Math.ceil(value / step) * step; -} - -interface Tip { - x: number; - y: number; - lines: string[]; -} - -function GridLines({ max, ticks = 4 }: { max: number; ticks?: number }) { - return ( - - {Array.from({ length: ticks + 1 }, (_, index) => { - const value = (max / ticks) * index; - const y = yScale(value, max); - return ( - - - - {Math.round(value)} - - - ); - })} - - ); -} - -function TipBox({ tip }: { tip: Tip | null }) { - if (!tip) { - return null; - } - return ( -
- {tip.lines.map((line, index) => (index === 0 ? {line} :
{line}
))} -
- ); -} - -export function LiftChart({ snapshot }: { snapshot: Snapshot }) { - const [tip, setTip] = useState(null); - const rows = snapshot.paired.per_seed; - const max = niceMax(Math.max(...rows.map((row) => row.candidate_score)) * 1.08); - const band = (W - PAD.left - PAD.right) / rows.length; - const barWidth = Math.min(26, band / 3); - - return ( -
-
-

Per-seed paired lift

- same seeds, differenced -
-
- setTip(null)} - > - - {rows.map((row, index) => { - const cx = PAD.left + band * index + band / 2; - const candidateY = yScale(row.candidate_score, max); - const panelY = yScale(row.baseline_panel_score, max); - const baseY = yScale(0, max); - const showTip = () => - setTip({ - x: cx, - y: Math.min(candidateY, panelY), - lines: [ - `seed ${row.seed}`, - `candidate ${fmt(row.candidate_score, 1)}`, - `panel ${fmt(row.baseline_panel_score, 1)}`, - `lift ${row.lift >= 0 ? "+" : ""}${fmt(row.lift, 1)}`, - ], - }); - return ( - = 0 ? "+" : ""}${fmt(row.lift, 1)}`} - onMouseEnter={showTip} - onMouseLeave={() => setTip(null)} - onFocus={showTip} - onBlur={() => setTip(null)} - onClick={showTip} - onKeyDown={(event) => { - if (event.key === "Enter" || event.key === " ") { - event.preventDefault(); - showTip(); - } - }} - > - - - - - {row.lift >= 0 ? "+" : ""}{Math.round(row.lift)} - - - seed {row.seed} - - - ); - })} - - -
-
- - - value (candidate) - - - - baseline panel mean — the bar - - - mean lift +{fmt(snapshot.paired.paired_lift_mean, 1)} · σ {fmt(snapshot.paired.paired_lift_stddev, 1)} - -
-
- ); -} - -export function SeasonTraceChart({ snapshot }: { snapshot: Snapshot }) { - const [tip, setTip] = useState(null); - const trace = snapshot.season_trace; - const rows = trace.seasons; - const max = niceMax(Math.max(...rows.map((row) => row.score_after_season)) * 1.12); - const band = (W - PAD.left - PAD.right) / rows.length; - - const points = rows.map((row, index) => ({ - x: PAD.left + band * index + band / 2, - y: yScale(row.score_after_season, max), - row, - })); - const path = points.map((point, index) => `${index === 0 ? "M" : "L"}${point.x},${point.y}`).join(" "); - - return ( -
-
-

One franchise, five seasons

- - {trace.agent} agent · seed {trace.seed} - -
-
- setTip(null)} - > - - - {points.map(({ x, y, row }) => { - const showTip = () => - setTip({ - x, - y, - lines: [ - `season ${row.season} · ${row.wins}-${row.losses}`, - `score ${fmt(row.score_after_season, 1)}`, - `cap room $${fmt(row.cap_room, 1)}M${row.champion ? " · champion" : ""}`, - ], - }); - return ( - setTip(null)} - onFocus={showTip} - onBlur={() => setTip(null)} - onClick={showTip} - onKeyDown={(event) => { - if (event.key === "Enter" || event.key === " ") { - event.preventDefault(); - showTip(); - } - }} - > - - {row.champion && ( - - 🏆 - - )} - - - S{row.season} - - - {row.wins}-{row.losses} - - - ); - })} - - -
-
- - - objective score after season - - 🏆 championship - final cap room ${fmt(rows[rows.length - 1].cap_room, 1)}M -
-
- ); -} diff --git a/web/src/components/Footer.tsx b/web/src/components/Footer.tsx index 5ec7bf7..7f6903a 100644 --- a/web/src/components/Footer.tsx +++ b/web/src/components/Footer.tsx @@ -7,11 +7,11 @@ export default function Footer({ data }: { data: LeaderboardData }) {
diff --git a/web/src/components/Hero.tsx b/web/src/components/Hero.tsx index e4cc6f0..22c9ec0 100644 --- a/web/src/components/Hero.tsx +++ b/web/src/components/Hero.tsx @@ -5,45 +5,31 @@ export default function Hero({ data }: { data: LeaderboardData }) { const fingerprint = data.contract?.contract_fingerprint; const cap = data.publication.frozen_output_token_cap; const registryFrozen = data.publication.model_registry_frozen === true; + const modelCount = data.models.length; + return (
-
-

- A pre-registered front-office benchmark for LLM agents - By Ned Cutler +

+

+ GM-Bench.

- - Phase-one result - {data.models.length} models tested. 0 beat the scripted bar. - - -

- Can a language model out-manage a scripted front office? -

+ {/* LEARNING TODO: rewrite this h1 in your own wire voice (5–10 words). + Keep the finding; drop any leftover marketing tone. */} +

Phase one: 0 of {modelCount} models beat the scripted bar

- GM-Bench puts agents in charge of a franchise in a fictional hockey-style league — - contracts, trades, drafts, cap pressure — for seeded, replayable multi-season episodes. - Eight scripted heuristics set the bar and a hidden-information oracle sets the ceiling.{" "} - The board opens only when the pre-registered evidence is complete. + Agents run a franchise in a seeded hockey-style league — contracts, trades, drafts, cap + pressure — across multi-season episodes. Eight scripted heuristics set the bar; a + hidden-information oracle sets the ceiling.

{fingerprint && ( <> - frozen contract {fingerprint} ·{" "} + contract {fingerprint} ·{" "} )} {data.preset.seeds.length} seeds × {data.preset.seasons} seasons × 3 repeats @@ -56,7 +42,7 @@ export default function Hero({ data }: { data: LeaderboardData }) { {registryFrozen ? " · routes pinned" : " · routes pending smoke verification"}

-
+
diff --git a/web/src/components/HowItWorks.tsx b/web/src/components/HowItWorks.tsx index 8c3e9c3..1f78560 100644 --- a/web/src/components/HowItWorks.tsx +++ b/web/src/components/HowItWorks.tsx @@ -3,57 +3,37 @@ import type { Snapshot } from "../types"; const PHASES = [ { num: "01 · preseason", - title: "Build the roster", - body: "Sign free agents under a hard salary cap, dress an 18-player lineup, and balance veterans against prospects. Rivals compete for the same pool — a free agent visible now may be gone next phase.", + title: "Roster", + body: "Sign free agents under a hard salary cap, dress an 18-player lineup, balance veterans and prospects. Rivals share the same pool.", }, { num: "02 · trade deadline", - title: "Trade under pressure", - body: "Swap players with eleven AI rivals mid-season. Partners apply hidden valuation noise each season, so what looked fair in preseason may fail at the deadline. Illegal proposals are rejected and penalized.", + title: "Trades", + body: "Swap with eleven AI rivals mid-season. Partners apply hidden valuation noise each season. Illegal proposals are rejected and penalized.", }, { num: "03 · draft", - title: "Invest in the future", - body: "Spend draft capital on a seeded prospect class while opponents pick in inverse-standings order. Aging, development, and injuries play out across the season simulation and playoffs.", + title: "Draft", + body: "Spend capital on a seeded prospect class; opponents pick in inverse-standings order. Aging, development, and injuries play out through the season.", }, ]; const PROTO_POINTS = [ { title: "Observation on stdin", - body: "One JSON object per decision point: your team (roster, lineup, cap room), standings, free agents, draft class, trade market, recent transactions, and your memo scratchpad.", - icon: ( - - - - ), + body: "One JSON object per decision: team (roster, lineup, cap room), standings, free agents, draft class, trade market, recent transactions, memo scratchpad.", }, { title: "Actions on stdout", - body: "Reply with a JSON array of actions. Core verbs: sign_free_agent, trade, draft, set_lineup, and memo — plus release and noop when you need them.", - icon: ( - - - - ), + body: "JSON array of actions. Core verbs: sign_free_agent, trade, draft, set_lineup, memo — plus release and noop.", }, { title: "Deterministic replay", - body: "Leagues, development rolls, and injuries derive from the seed. The same agent on the same seed produces the same episode, every time.", - icon: ( - - - - ), + body: "Leagues, development rolls, and injuries derive from the seed. Same agent, same seed, same episode.", }, { title: "Scored beyond wins", - body: "The objective rewards wins, titles, future assets, prospect value, and cap health — and penalizes illegal or wasteful management.", - icon: ( - - - - ), + body: "Objective rewards wins, titles, future assets, prospect value, and cap health — and penalizes illegal or wasteful management.", }, ]; @@ -104,22 +84,21 @@ function CodeCard({ title, code }: { title: string; code: string }) { export default function HowItWorks({ snapshot }: { snapshot: Snapshot }) { return ( -
+
-

The decision loop

+

Protocol

Three decision points per season. Five seasons per episode.

- No browser automation, no memorized rosters — every player is fictional. Agents - face the same long-horizon trade-offs a real front office does, expressed as a - minimal JSON protocol any process can speak. + No browser automation, no memorized rosters — every player is fictional. Agents speak a + minimal JSON protocol any process can run.

-
+
{PHASES.map((phase) => ( -
-

{phase.num}

+
+

{phase.num}

{phase.title}

{phase.body}

@@ -130,25 +109,22 @@ export default function HowItWorks({ snapshot }: { snapshot: Snapshot }) {
{PROTO_POINTS.map((point) => (
- {point.icon} -
-

{point.title}

-

{point.body}

-
+

{point.title}

+

{point.body}

))}
-
+
-
+
-

Sample transaction audit

+

Transaction wire

- {snapshot.season_trace.agent} agent · seed {snapshot.season_trace.seed} · every action is logged + {snapshot.season_trace.agent} · seed {snapshot.season_trace.seed}
diff --git a/web/src/components/Integrity.tsx b/web/src/components/Integrity.tsx deleted file mode 100644 index eeaac36..0000000 --- a/web/src/components/Integrity.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import type { Leaderboard as LeaderboardData } from "../types"; - -export default function Integrity({ data }: { data: LeaderboardData }) { - const contract = data.contract; - const cap = data.publication.frozen_output_token_cap; - const registryFrozen = data.publication.model_registry_frozen === true; - return ( -
-
-
-

Integrity

-

The measurement is pre-registered.

-

- The score-affecting contract and statistical plan are frozen and fingerprinted before - official results. Route selection remains visibly pending until every registered smoke - clears the gate below. -

-
-
-
-

Frozen contract

-

- Simulator, scoring, presets, and action schemas hash to{" "} - {contract?.contract_fingerprint ?? "—"}. Any change to a score-affecting - source changes the fingerprint and invalidates comparability. -

-
-
-

Fixed compute policy

- {cap ? ( -

- One common {cap.toLocaleString("en-US")}-token output ceiling, reasoning off, JSON - mode on; exact provider routes are {registryFrozen ? "pinned" : "pending smoke verification"}. - The v1 table ranked output budgets, not models; v2 holds compute constant. -

- ) : ( -

The common compute policy is still pending and no fixed-ceiling claim is published.

- )} -
-
-

Tiers, not ranks

-

- The statistical plan froze before any data: Holm-corrected paired contrasts, per-model - p-values reported as descriptive only, and models with overlapping uncertainty - published as one tier. -

-
-
-

Evidence or nothing

-

- Paid runs are serial, checkpointed, and spend-capped; raw artifacts are hash-linked; - valid poor results are never re-run. When v1 failed this standard, it was withdrawn — - not patched. -

-
-
-
-
- ); -} diff --git a/web/src/components/Ladder.tsx b/web/src/components/Ladder.tsx index 3b70acf..ade4226 100644 --- a/web/src/components/Ladder.tsx +++ b/web/src/components/Ladder.tsx @@ -89,9 +89,9 @@ export default function Ladder({ data }: { data: LeaderboardData }) { return (
-

The score ladder

+

Score ladder

- objective score · {data.preset.seeds.length} seeds × {data.preset.seasons} seasons · scripted panel measured, model rows{" "} + objective score · {data.preset.seeds.length} seeds × {data.preset.seasons} seasons · panel measured · model rows{" "} {pending.publishable_ranking ? "published" : "withheld"}
diff --git a/web/src/components/Leaderboard.tsx b/web/src/components/Leaderboard.tsx index 05ae41e..16f8f2f 100644 --- a/web/src/components/Leaderboard.tsx +++ b/web/src/components/Leaderboard.tsx @@ -84,7 +84,7 @@ function ModelTable({ aria-expanded={showTelemetry} onClick={() => setShowTelemetry((visible) => !visible)} > - {showTelemetry ? "Hide full telemetry" : "Show full telemetry"} + {showTelemetry ? "Hide telemetry" : "Show telemetry"}
@@ -159,16 +159,14 @@ function ModelTable({
{withTiers ? ( <> - tiers group models whose paired-lift 95% intervals overlap — order inside a tier is display order, not a claim - per-model intervals are descriptive; family-wise claims follow the frozen Holm-corrected analysis plan + tiers group models whose paired-lift 95% intervals overlap + per-model intervals are descriptive; family-wise claims follow the frozen Holm plan ) : ( - observational lane — uncontrolled tool loops, context, and retries; no tiers, not comparable to the API lane + observational lane — uncontrolled tool loops; not comparable to the API lane )} - lift vs panel = paired per-seed difference against the full scripted-baseline mean - flags are visible protocol observations; eligible rows still passed every frozen publication gate - fallback = decisions answered by the adapter's fallback policy, not the model - cost from measured tokens × published prices; read score next to input/output tokens and cost, never alone + lift vs panel = paired per-seed difference against the scripted-baseline mean + fallback = decisions answered by the adapter policy, not the model
); @@ -188,9 +186,9 @@ function BarTable({ data }: { data: LeaderboardData }) { return (
-

The bar, measured

+

Scripted panel

- scripted panel · preset {data.preset.name} · {data.preset.seeds.length} seeds × {data.preset.seasons} seasons + preset {data.preset.name} · {data.preset.seeds.length} seeds × {data.preset.seasons} seasons {data.updated ? ` · updated ${data.updated}` : ""}
@@ -215,8 +213,8 @@ function BarTable({ data }: { data: LeaderboardData }) { - {row.kind === "bar" && the red line} - {row.kind === "ceiling" && hidden-information ceiling} + {row.kind === "bar" && red line} + {row.kind === "ceiling" && oracle ceiling} {row.kind === "baseline" && scripted} {fmt(row.mean_score, 1)} @@ -235,8 +233,8 @@ function BarTable({ data }: { data: LeaderboardData }) {
- every model row is paired against these exact seeds — the panel is the control group - the oracle sees hidden information no legal agent can; it bounds what the score can express + every model row is paired against these seeds + oracle sees hidden information no legal agent can
); @@ -255,23 +253,17 @@ function Gate({ data }: { data: LeaderboardData }) { { done: smokesDone, label: "every registered route smoke-verified" }, { done: rowsMet, - label: `≥${publication.minimum_headline_models} strictly eligible rows (${publication.eligible_headline_models} today)`, + label: `≥${publication.minimum_headline_models} eligible rows (${publication.eligible_headline_models} today)`, }, - { done: analysisDone, label: "Holm-adjusted panel analysis bound to the exact artifacts" }, + { done: analysisDone, label: "Holm-adjusted panel analysis bound to artifacts" }, ]; return (
-

No ranking yet — that is the protocol working

+

Ranking withheld until every gate clears

- The previous version of this table was withdrawn when a contract defect was found to - penalize some models and not others. The v2 board therefore publishes nothing until every - gate below is machine-verified: {publication.reason}. -

-

- Scores cannot influence the protocol — the contract, compute policy, model registry, and - statistical analysis plan freeze before official runs, and a valid poor result is - never re-run. + v1 was withdrawn after a contract defect penalized some models unevenly. v2 publishes + nothing until these checks pass: {publication.reason}.

{checks.map((check) => ( @@ -281,7 +273,7 @@ function Gate({ data }: { data: LeaderboardData }) { ))}
- Withheld by design + Withheld
); } @@ -290,20 +282,17 @@ function ArchiveNotice() { return (
-

Withdrawn: the sota-v1 results

+

Archive: sota-v1 withdrawn

retained as evidence · not a ranking

- Every score published under the previous sota-v1 contract has been withdrawn. - The scaffold prompt documented {`{"type":"scout","prospect_id":N}`} as a valid - action, but the simulator only ever read player_id and silently rejected the - documented form — with no protocol penalty, so the rejections never appeared in any summary. + Scores under the previous sota-v1 contract were withdrawn. The scaffold prompt + documented {`{"type":"scout","prospect_id":N}`} as valid, but the simulator + only read player_id and silently rejected the documented form.

- The defect did not fall evenly. It cost some candidates over a thousand silently-rejected - lookups and others none at all, while the scripted baselines were untouched. The v1 table was - therefore not a valid ranking of the models in it, and no caveat makes it one. The raw - artifacts remain in results/leaderboard/archive-v1/ as evidence of the defect. + The defect did not fall evenly across candidates. Raw artifacts remain in{" "} + results/leaderboard/archive-v1/.

); @@ -320,8 +309,8 @@ function MechanicBreakdown({ models }: { models: LeaderboardModel[] }) { return (
-

Protocol outcomes by mechanic

- accepted / rejected actions +

Accepted / rejected by mechanic

+ protocol outcomes
@@ -356,50 +345,91 @@ function MechanicBreakdown({ models }: { models: LeaderboardModel[] }) { ); } +function Footnotes({ data }: { data: LeaderboardData }) { + const contract = data.contract; + const cap = data.publication.frozen_output_token_cap; + const registryFrozen = data.publication.model_registry_frozen === true; + return ( +
+
+

Contract

+

+ Simulator, scoring, presets, and schemas hash to{" "} + {contract?.contract_fingerprint ?? "—"}. +

+
+
+

Compute

+ {cap ? ( +

+ Shared {cap.toLocaleString("en-US")}-token output ceiling, reasoning off, JSON mode on; + routes {registryFrozen ? "pinned" : "pending smoke verification"}. +

+ ) : ( +

Compute policy still pending — no fixed-ceiling claim published.

+ )} +
+
+

Tiers

+

+ Holm-corrected paired contrasts froze before data. Overlapping uncertainty publishes as + one tier — not an ordinal #1. +

+
+
+

Evidence

+

+ Runs are serial, checkpointed, spend-capped; artifacts hash-linked. Valid poor results are + not re-run. +

+
+
+ ); +} + export default function Leaderboard({ data }: { data: LeaderboardData }) { const publishable = data.publication.publishable_ranking; return (
-

The board

-

Tiers, not ranks. Evidence, not vibes.

+

Standings

+

Tiers from paired lifts, not a #1 ranking.

- Models manage the same franchises over the same {data.preset.seeds.length} seeds for{" "} - {data.preset.seasons} seasons ({data.preset.decision_points_per_episode} decisions per - franchise), paired per-seed against the scripted panel. The frozen analysis plan - publishes overlapping-uncertainty tiers — never an ordinal #1. + Same franchises, same {data.preset.seeds.length} seeds, {data.preset.seasons} seasons ( + {data.preset.decision_points_per_episode} decisions per franchise), paired per-seed + against the scripted panel.

{!publishable && } {publishable && ( )} {publishable && data.models.length > 0 && } -
+
{data.cli_harness_models.length > 0 && ( <> -

- Coding-harness rows run the same episodes through a CLI agent's own tool loop, context, - and retries. That harness is part of what gets measured, so these rows live in their own - observational table and never mix with the API lane. +

+ Coding-harness rows use each CLI agent’s own tool loop, context, and retries. Separate + table by design — not mixed with the API lane.

)} -
+ +
diff --git a/web/src/components/Nav.tsx b/web/src/components/Nav.tsx index 97b8152..ab3e3b7 100644 --- a/web/src/components/Nav.tsx +++ b/web/src/components/Nav.tsx @@ -1,19 +1,17 @@ -export function Logo({ size = 26 }: { size?: number }) { +export function Logo({ size = 24 }: { size?: number }) { return ( ); } const LINKS = [ - { href: "#leaderboard", label: "The board" }, - { href: "#integrity", label: "Integrity" }, + { href: "#leaderboard", label: "Standings" }, { href: "#protocol", label: "Protocol" }, - { href: "#reference", label: "Reference" }, + { href: "#quickstart", label: "Run" }, ]; export default function Nav() { @@ -32,7 +30,7 @@ export default function Nav() { ))} - Run the benchmark + Run locally
diff --git a/web/src/components/Quickstart.tsx b/web/src/components/Quickstart.tsx index 998bd79..65ef895 100644 --- a/web/src/components/Quickstart.tsx +++ b/web/src/components/Quickstart.tsx @@ -20,6 +20,15 @@ sqlite3 data/gm_bench.sqlite \\ 'select agent, seed, final_score from episodes order by final_score desc;'`; +const ADAPTERS = [ + { name: "Codex CLI", snippet: "examples/codex_agent.py" }, + { name: "Claude Code", snippet: "examples/claude_agent.py" }, + { name: "Ollama", snippet: "examples/ollama_agent.py" }, + { name: "OpenAI-compatible", snippet: "examples/openai_compatible_agent.py" }, + { name: "opencode", snippet: "examples/opencode_agent.py" }, + { name: "Any process", snippet: '--agent-cmd "…"' }, +]; + function CommandCard({ title, code }: { title: string; code: string }) { const [copied, setCopied] = useState(false); const copy = async () => { @@ -32,7 +41,7 @@ function CommandCard({ title, code }: { title: string; code: string }) {
{title} 
@@ -47,17 +56,27 @@ export default function Quickstart() {
     

       

         

-          
Quickstart

-          
From clone to scoreboard in two commands.

+          
Run

+          
Calibrate on baselines, then plug in an agent.

           

-            Start with the scripted baselines to calibrate, then plug in your own agent with
-             --agent-cmd. JSON Schemas for the observation and action protocol ship in{" "}
+            Use --agent-cmd for any subprocess. Observation and action schemas ship in{" "}
             schemas/.
           

         

         

-          
-          
+          
+          
+        

+        

+          Compatible with Codex CLI, Claude Code, Ollama, OpenAI-compatible
+          endpoints, opencode, or any stdin/stdout process:
+          

+            {ADAPTERS.map((adapter) => (
+              
+                {adapter.snippet}
+              
+            ))}
+          

         

       

     

diff --git a/web/src/components/Results.tsx b/web/src/components/Results.tsx
deleted file mode 100644
index f8be4f2..0000000
--- a/web/src/components/Results.tsx
+++ /dev/null
@@ -1,146 +0,0 @@
-import type { Snapshot } from "../types";
-import { agentColor, fmt, pct } from "../lib";
-import { LiftChart, SeasonTraceChart } from "./Charts";
-
-function StandingsTable({ snapshot }: { snapshot: Snapshot }) {
-  const { standings, config } = snapshot;
-  const maxScore = Math.max(...standings.map((row) => row.mean_score));
-  return (
-    

-      

-        
Agent standings

-        
-          seeds {config.seeds.join(" ")} · {config.seasons} seasons
-        
-      

-      

-
- - - - - - - - - - - - - {standings.map((row) => ( - - - - - - - - - - ))} - -
AgentMean scoreσWins/epTitlesIllegal
- - - {row.agent} - - {row.agent === config.candidate ? "candidate" : "baseline"} - - - {fmt(row.mean_score, 1)} -
-
-
-
{fmt(row.score_stddev, 1)}{fmt(row.mean_wins, 1)}{row.titles}{row.illegal_actions}
-
-
- ); -} - -function Verdict({ snapshot }: { snapshot: Snapshot }) { - const { normalized, paired } = snapshot; - return ( -
-
-

Paired evaluation verdict

- value vs panel · descriptive -
-
- +{fmt(paired.paired_lift_mean, 1)} - paired lift -
-
-
- Candidate mean score - {fmt(normalized.candidate_mean_score, 1)} -
-
- Baseline panel mean - {fmt(normalized.baseline_panel_mean_score, 1)} -
-
- Bootstrap 95% CI on lift - - [{fmt(paired.paired_lift_ci95[0], 1)}, {fmt(paired.paired_lift_ci95[1], 1)}] - -
-
- Seed win rate vs panel - {pct(paired.candidate_seed_win_rate)} -
- {paired.best_baseline && ( -
- vs best baseline ({paired.best_baseline.agent}) - - +{fmt(paired.best_baseline.paired_lift_mean, 1)} · {pct(paired.best_baseline.seed_win_rate)} seeds - -
- )} -
- Illegal actions (candidate) - {normalized.candidate_illegal_actions} -
-
-
- intervals are descriptive — the frozen analysis plan reserves significance language for the Holm-corrected family -
-
- ); -} - -export default function Results({ snapshot }: { snapshot: Snapshot }) { - return ( -
-
-
-

Reference run

-

How a result reads, demonstrated on scripted agents.

-

- Every agent plays the exact same league generations. Per-seed differencing cancels league - luck, and a deterministic bootstrap puts an interval on the lift. This reference run pits - the scripted value agent against the panel — the same report every model row - gets. -

-
-
- - -
-
- - -
-
-
- ); -} diff --git a/web/src/index.css b/web/src/index.css index f951aef..19d1602 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -1,24 +1,23 @@ -/* GM-Bench — "the officials' report from the rink". - Light ice surface, deep ink, rink-line hairlines. Red is reserved for one - meaning everywhere on the page: the bar to beat (the red line). Blue marks - measured candidates. Display type is condensed and uppercase like a - scoreboard; running text is a serif, like the report that gets filed. */ +/* GM-Bench — sports wire / box score. + Cold paper, ink, rink-line hairlines. Red means one thing: the bar to beat. + Blue marks measured candidates. Scores use condensed numerals; copy is + sentence-case grotesque, not campaign headlines. */ :root { - --ice: #f4f7f9; - --ice-2: #ecf1f5; + --paper: #eef2f5; + --paper-2: #e4eaef; --card: #ffffff; - --ink: #0c1d2e; - --steel: #51697e; - --faint: #5a7185; + --ink: #0a1620; + --steel: #3d5163; + --faint: #5c7386; --red: #c8102e; --red-soft: rgba(200, 16, 46, 0.07); - --blue: #155b9a; - --blue-soft: rgba(21, 91, 154, 0.08); - --line: #d9e4eb; - --line-strong: #b9cbd7; - --font-display: "Barlow Condensed", "Arial Narrow", sans-serif; - --font-body: "source serif 4", Georgia, serif; + --blue: #1a5f8f; + --blue-soft: rgba(26, 95, 143, 0.08); + --line: #d3dde6; + --line-strong: #c5d3de; + --font-score: "Barlow Condensed", "Arial Narrow", sans-serif; + --font-body: "IBM Plex Sans", "Helvetica Neue", sans-serif; --font-mono: "IBM Plex Mono", ui-monospace, monospace; } @@ -34,17 +33,17 @@ html { } body { - background: var(--ice); + background: var(--paper); color: var(--ink); font-family: var(--font-body); - font-size: 16.5px; - line-height: 1.65; + font-size: 15.5px; + line-height: 1.55; -webkit-font-smoothing: antialiased; overflow-x: clip; } ::selection { - background: rgba(21, 91, 154, 0.18); + background: rgba(26, 95, 143, 0.18); } a { @@ -73,22 +72,9 @@ code { } .shell { - max-width: 1120px; + max-width: 1080px; margin: 0 auto; - padding: 0 24px; -} - -/* ---------- display type ---------- */ - -.display, -h1, -h2, -.section-head h2 { - font-family: var(--font-display); - font-weight: 700; - text-transform: uppercase; - letter-spacing: 0.01em; - line-height: 0.98; + padding: 0 22px; } /* ---------- nav ---------- */ @@ -97,8 +83,8 @@ h2, position: sticky; top: 0; z-index: 40; - background: rgba(244, 247, 249, 0.92); - backdrop-filter: blur(10px); + background: rgba(238, 242, 245, 0.94); + backdrop-filter: blur(8px); border-bottom: 1px solid var(--line-strong); } @@ -106,43 +92,38 @@ h2, display: flex; align-items: center; justify-content: space-between; - height: 60px; + height: 52px; } .brand { display: flex; align-items: center; - gap: 10px; - font-family: var(--font-display); - font-weight: 700; - font-size: 21px; - text-transform: uppercase; - letter-spacing: 0.04em; + gap: 9px; + font-family: var(--font-body); + font-weight: 600; + font-size: 16px; + letter-spacing: -0.01em; } .brand-tag { font-family: var(--font-mono); font-size: 10px; font-weight: 500; - color: var(--blue); - border: 1px solid var(--blue); - border-radius: 3px; - padding: 1px 7px; + color: var(--steel); + border: 1px solid var(--line-strong); + border-radius: 2px; + padding: 1px 6px; margin-left: 2px; - letter-spacing: 0.1em; + letter-spacing: 0.06em; text-transform: uppercase; - transform: translateY(-1px); } .nav-links { display: flex; align-items: center; - gap: 26px; - font-family: var(--font-display); + gap: 22px; + font-size: 13.5px; font-weight: 500; - font-size: 16px; - text-transform: uppercase; - letter-spacing: 0.05em; color: var(--steel); } @@ -151,18 +132,13 @@ h2, } .nav-cta { - font-family: var(--font-display); - font-weight: 600; - color: var(--ice); - background: var(--ink); - padding: 7px 16px 8px; - border-radius: 4px; - transition: background 0.15s ease; + color: var(--ink); + border-bottom: 1px solid var(--line-strong); + padding-bottom: 1px; } .nav-cta:hover { - background: var(--blue); - color: var(--ice); + border-color: var(--ink); } @media (max-width: 800px) { @@ -173,18 +149,18 @@ h2, @media (max-width: 480px) { .shell { - padding-inline: 16px; + padding-inline: 14px; } .brand { gap: 7px; - font-size: 18px; + font-size: 15px; white-space: nowrap; } .brand svg { - width: 24px; - height: 24px; + width: 22px; + height: 22px; } .brand-tag { @@ -196,8 +172,7 @@ h2, } .nav-cta { - padding: 7px 10px 8px; - font-size: 14px; + font-size: 13px; white-space: nowrap; } } @@ -205,158 +180,92 @@ h2, /* ---------- hero ---------- */ .hero { - padding: 62px 0 40px; -} - -.hero-eyebrow { - display: inline-flex; - align-items: center; - gap: 10px; - font-family: var(--font-mono); - font-size: 12px; - letter-spacing: 0.12em; - text-transform: uppercase; - color: var(--steel); - margin-bottom: 16px; -} - -.hero-eyebrow::before { - content: ""; - width: 34px; - height: 3px; - background: var(--red); -} - -.hero-eyebrow a { - color: var(--ink); + padding: 48px 0 0; border-bottom: 1px solid var(--line-strong); + background: + linear-gradient(180deg, #f5f8fa 0%, var(--paper) 72%), + repeating-linear-gradient( + 90deg, + transparent, + transparent 47px, + rgba(197, 211, 222, 0.35) 47px, + rgba(197, 211, 222, 0.35) 48px + ); } -.hero-eyebrow a:hover { - color: var(--blue); - border-color: var(--blue); +.hero-top { + padding-bottom: 28px; } -.hero h1 { - font-size: clamp(52px, 7.6vw, 94px); - max-width: 17ch; - margin-bottom: 26px; -} - -.hero-result { - display: flex; - align-items: baseline; - width: fit-content; - gap: 10px 14px; - margin-bottom: 24px; - padding: 8px 12px 9px; - border-left: 3px solid var(--red); - background: var(--card); - box-shadow: inset 0 0 0 1px var(--line); - font-family: var(--font-mono); +.hero-brand { + font-family: var(--font-score); + font-weight: 700; + font-size: clamp(42px, 7vw, 68px); + letter-spacing: 0.02em; + line-height: 0.95; + text-transform: uppercase; + margin-bottom: 18px; } -.hero-result span, -.hero-result i { - color: var(--steel); - font-size: 10.5px; - font-style: normal; - letter-spacing: 0.08em; - text-transform: uppercase; +.hero-brand span { + color: var(--red); } -.hero-result strong { - font-family: var(--font-display); - font-size: 18px; - letter-spacing: 0.02em; - text-transform: uppercase; +.hero h1 { + font-family: var(--font-body); + font-weight: 600; + font-size: clamp(22px, 3.2vw, 28px); + letter-spacing: -0.02em; + line-height: 1.25; + max-width: 28ch; + margin-bottom: 12px; + text-transform: none; } -.hero-result:hover strong, -.hero-result:hover i { - color: var(--blue); +@media (prefers-reduced-motion: no-preference) { + .hero h1 { + text-decoration: underline; + text-decoration-color: var(--red); + text-decoration-thickness: 3px; + text-underline-offset: 6px; + animation: verdict-line 0.7s ease 0.15s both; + } } -.hero h1 .bar-word { - color: var(--red); +@keyframes verdict-line { + from { + text-decoration-color: transparent; + } + to { + text-decoration-color: var(--red); + } } .hero-sub { - font-size: 19px; + font-size: 16px; color: var(--steel); - max-width: 58ch; - margin-bottom: 30px; -} - -.hero-sub strong { - color: var(--ink); - font-weight: 600; + max-width: 52ch; + margin-bottom: 18px; } .hero-actions { display: flex; flex-wrap: wrap; - gap: 12px; - margin-bottom: 18px; + align-items: center; + gap: 16px; + margin-bottom: 14px; } -.btn-primary, -.btn-ghost { +.btn-primary { display: inline-flex; align-items: center; - gap: 9px; - font-family: var(--font-display); - font-weight: 600; - font-size: 18px; - text-transform: uppercase; - letter-spacing: 0.05em; - border-radius: 4px; - padding: 10px 22px 11px; -} - -.btn-text { - align-self: center; - padding: 8px 4px; - color: var(--steel); - border-bottom: 1px solid var(--line-strong); - font-family: var(--font-display); - font-size: 17px; + font-family: var(--font-body); font-weight: 600; - letter-spacing: 0.05em; - line-height: 1; - text-transform: uppercase; -} - -.btn-text:hover { - color: var(--blue); - border-color: var(--blue); -} - -@media (max-width: 620px) { - .hero { - padding-top: 44px; - } - - .hero-result { - align-items: flex-start; - flex-direction: column; - gap: 2px; - width: 100%; - } - - .hero-eyebrow { - align-items: flex-start; - flex-wrap: wrap; - } - - .hero-result strong { - font-size: 17px; - } -} - -.btn-primary { - color: var(--ice); + font-size: 14px; + color: var(--paper); background: var(--ink); + border-radius: 3px; + padding: 9px 16px; transition: background 0.15s ease; } @@ -364,22 +273,11 @@ h2, background: var(--blue); } -.btn-ghost { - color: var(--ink); - border: 1px solid var(--line-strong); - transition: border-color 0.15s ease, background 0.15s ease; -} - -.btn-ghost:hover { - border-color: var(--blue); - background: var(--blue-soft); -} - .hero-facts { font-family: var(--font-mono); - font-size: 12px; + font-size: 11.5px; color: var(--faint); - letter-spacing: 0.02em; + letter-spacing: 0.01em; } .hero-facts b { @@ -387,12 +285,19 @@ h2, font-weight: 500; } -/* ---------- score ladder (signature) ---------- */ +.hero-todo { + font-family: var(--font-mono); + font-size: 11px; + color: var(--faint); + margin-top: 8px; +} + +/* ---------- score ladder ---------- */ .ladder-panel { - margin: 44px 0 0; + margin: 0; border-top: 3px solid var(--red); - border-bottom: 1px solid var(--line-strong); + border-bottom: none; background: var(--card); } @@ -402,22 +307,27 @@ h2, justify-content: space-between; align-items: baseline; gap: 8px; - padding: 16px 22px 0; + padding: 14px 18px 0; } .ladder-head h2 { - font-size: 24px; + font-family: var(--font-body); + font-weight: 600; + font-size: 15px; + letter-spacing: -0.01em; + text-transform: none; + line-height: 1.2; } .ladder-head span { font-family: var(--font-mono); - font-size: 11.5px; + font-size: 11px; color: var(--faint); } .ladder-wrap { position: relative; - padding: 4px 10px 8px; + padding: 2px 8px 6px; } .ladder-svg { @@ -444,10 +354,11 @@ h2, } } -/* On narrow screens the scaled-down SVG can't fit every label; keep only the - semantically load-bearing ones (bar, ceiling, floor) — the full numbers - remain in the baselines table. */ @media (max-width: 620px) { + .hero { + padding-top: 36px; + } + .ladder-svg .lbl:not(.lbl-key) { display: none; } @@ -457,12 +368,12 @@ h2, position: absolute; pointer-events: none; background: var(--ink); - color: var(--ice); + color: var(--paper); font-family: var(--font-mono); - font-size: 11.5px; + font-size: 11px; line-height: 1.5; - padding: 7px 10px; - border-radius: 4px; + padding: 6px 9px; + border-radius: 3px; transform: translate(-50%, calc(-100% - 10px)); white-space: nowrap; z-index: 30; @@ -476,7 +387,7 @@ h2, /* ---------- sections ---------- */ .section { - padding: 76px 0; + padding: 52px 0; } .section + .section { @@ -484,41 +395,36 @@ h2, } .section-alt { - background: var(--ice-2); + background: var(--paper-2); } .kicker { - display: flex; - align-items: center; - gap: 12px; font-family: var(--font-mono); - font-size: 12px; - letter-spacing: 0.14em; + font-size: 11px; + letter-spacing: 0.1em; text-transform: uppercase; - color: var(--steel); - margin-bottom: 14px; -} - -.kicker::before { - content: ""; - width: 22px; - height: 1px; - background: var(--line-strong); + color: var(--faint); + margin-bottom: 8px; } .section-head { - max-width: 720px; - margin-bottom: 40px; + max-width: 640px; + margin-bottom: 28px; } .section-head h2 { - font-size: clamp(34px, 4.6vw, 52px); - margin-bottom: 14px; + font-family: var(--font-body); + font-weight: 600; + font-size: clamp(22px, 3vw, 28px); + letter-spacing: -0.02em; + line-height: 1.2; + text-transform: none; + margin-bottom: 10px; } .section-head p { color: var(--steel); - font-size: 17px; + font-size: 15px; } .section-head p code { @@ -530,14 +436,15 @@ h2, .panel { background: var(--card); border: 1px solid var(--line-strong); - border-radius: 6px; - padding: 22px 24px; + border-radius: 2px; + padding: 16px 18px; } .panel + .panel, .panel + .lane-note, -.lane-note + .panel { - margin-top: 20px; +.lane-note + .panel, +.panel + .footnotes { + margin-top: 14px; } .panel-title { @@ -546,15 +453,15 @@ h2, justify-content: space-between; flex-wrap: wrap; gap: 6px 12px; - margin-bottom: 16px; + margin-bottom: 12px; } .panel-title h3 { - font-family: var(--font-display); + font-family: var(--font-body); font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.03em; - font-size: 20px; + font-size: 14.5px; + letter-spacing: -0.01em; + text-transform: none; } .panel-title span { @@ -573,25 +480,24 @@ h2, .telemetry-toggle { color: var(--blue); - border-bottom: 1px solid var(--blue); + border-bottom: 1px solid transparent; font-family: var(--font-mono); font-size: 11px; line-height: 1.5; } .telemetry-toggle:hover { - color: var(--ink); - border-color: var(--ink); + border-color: var(--blue); } .panel > p { - font-size: 15.5px; + font-size: 14.5px; color: var(--steel); max-width: 70ch; } .panel > p + p { - margin-top: 10px; + margin-top: 8px; } .panel > p code { @@ -610,19 +516,19 @@ h2, table { width: 100%; border-collapse: collapse; - font-size: 14px; + font-size: 13.5px; } thead th { font-family: var(--font-mono); font-size: 10px; font-weight: 500; - letter-spacing: 0.09em; + letter-spacing: 0.06em; text-transform: uppercase; color: var(--faint); text-align: left; - padding: 8px 12px; - border-bottom: 1px solid var(--ink); + padding: 6px 10px; + border-bottom: 2px solid var(--ink); white-space: nowrap; } @@ -633,11 +539,19 @@ tbody td.num { } tbody td { - padding: 10px 12px; + padding: 8px 10px; border-bottom: 1px solid var(--line); white-space: nowrap; } +tbody tr { + transition: background 0.12s ease; +} + +tbody tr:hover { + background: rgba(26, 95, 143, 0.04); +} + tbody tr:last-child td { border-bottom: none; } @@ -665,24 +579,24 @@ tbody tr.tier-start td { .agent-cell { display: flex; align-items: center; - gap: 10px; + gap: 8px; font-weight: 600; } .agent-chip { - width: 10px; - height: 10px; - border-radius: 2px; + width: 8px; + height: 8px; + border-radius: 1px; flex-shrink: 0; } .tag { font-family: var(--font-mono); font-size: 9.5px; - letter-spacing: 0.07em; + letter-spacing: 0.05em; text-transform: uppercase; - border-radius: 3px; - padding: 2px 7px; + border-radius: 2px; + padding: 1px 6px; font-weight: 500; } @@ -702,7 +616,7 @@ tbody tr.tier-start td { } .tag-official { - color: var(--ice); + color: var(--paper); background: var(--blue); } @@ -714,15 +628,15 @@ tbody tr.tier-start td { .tier-chip { display: inline-block; - font-family: var(--font-display); - font-weight: 600; + font-family: var(--font-score); + font-weight: 700; font-size: 14px; - letter-spacing: 0.06em; + letter-spacing: 0.04em; text-transform: uppercase; color: var(--ink); border: 1px solid var(--ink); - border-radius: 3px; - padding: 1px 9px 2px; + border-radius: 2px; + padding: 0 7px 1px; } .status-stack { @@ -733,9 +647,10 @@ tbody tr.tier-start td { } .score-strong { - font-family: var(--font-display); + font-family: var(--font-score); font-weight: 700; - font-size: 17px; + font-size: 18px; + letter-spacing: 0.01em; } .mono-dim { @@ -746,9 +661,9 @@ tbody tr.tier-start td { .bar-track { position: relative; - height: 8px; - background: var(--ice-2); - min-width: 90px; + height: 6px; + background: var(--paper-2); + min-width: 80px; } .bar-fill { @@ -759,19 +674,19 @@ tbody tr.tier-start td { .legend { display: flex; flex-wrap: wrap; - gap: 6px 18px; - margin-top: 14px; + gap: 4px 16px; + margin-top: 12px; font-family: var(--font-mono); - font-size: 11.5px; + font-size: 11px; color: var(--steel); } .legend i { display: inline-block; - width: 10px; - height: 10px; - border-radius: 2px; - margin-right: 7px; + width: 8px; + height: 8px; + border-radius: 1px; + margin-right: 6px; vertical-align: -1px; } @@ -781,11 +696,11 @@ tbody tr.tier-start td { position: relative; background: var(--card); border: 1px solid var(--line-strong); - border-radius: 6px; - padding: 26px 28px; + border-radius: 2px; + padding: 18px 20px; display: grid; grid-template-columns: minmax(0, 1fr) auto; - gap: 24px; + gap: 18px; align-items: start; } @@ -796,46 +711,45 @@ tbody tr.tier-start td { } .gate h3 { - font-family: var(--font-display); - font-weight: 700; - text-transform: uppercase; - font-size: 27px; - margin-bottom: 10px; + font-family: var(--font-body); + font-weight: 600; + font-size: 16px; + letter-spacing: -0.01em; + text-transform: none; + margin-bottom: 8px; } .gate p { - font-size: 15.5px; + font-size: 14.5px; color: var(--steel); max-width: 62ch; } .gate p + p { - margin-top: 10px; + margin-top: 8px; } .stamp { - font-family: var(--font-display); - font-weight: 700; - font-size: 20px; - letter-spacing: 0.12em; + font-family: var(--font-mono); + font-weight: 600; + font-size: 11px; + letter-spacing: 0.08em; text-transform: uppercase; color: var(--steel); - border: 2px solid var(--steel); - border-radius: 4px; - padding: 8px 18px; - transform: rotate(3deg); + border: 1px solid var(--steel); + border-radius: 2px; + padding: 6px 12px; white-space: nowrap; align-self: center; - opacity: 0.85; } .gate-checks { - margin-top: 18px; + margin-top: 14px; display: flex; flex-wrap: wrap; - gap: 8px 22px; + gap: 6px 18px; font-family: var(--font-mono); - font-size: 12px; + font-size: 11.5px; color: var(--steel); } @@ -849,103 +763,123 @@ tbody tr.tier-start td { color: var(--faint); } -/* ---------- integrity facts ---------- */ +/* ---------- integrity footnotes ---------- */ -.fact-grid { +.footnotes { display: grid; grid-template-columns: repeat(4, 1fr); gap: 0; border: 1px solid var(--line-strong); - border-radius: 6px; + border-radius: 2px; background: var(--card); overflow: hidden; } @media (max-width: 900px) { - .fact-grid { + .footnotes { grid-template-columns: repeat(2, 1fr); } } @media (max-width: 540px) { - .fact-grid { + .footnotes { grid-template-columns: 1fr; } } -.fact { - padding: 22px 24px; +.footnote { + padding: 14px 16px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); } -.fact h4 { - font-family: var(--font-display); - font-weight: 600; +.footnote h4 { + font-family: var(--font-mono); + font-weight: 500; + font-size: 10.5px; + letter-spacing: 0.08em; text-transform: uppercase; - letter-spacing: 0.04em; - font-size: 18px; - margin-bottom: 6px; + color: var(--faint); + margin-bottom: 5px; } -.fact p { - font-size: 14px; +.footnote p { + font-size: 13px; color: var(--steel); + line-height: 1.45; } -.fact code { - font-size: 12px; +.footnote code { + font-size: 11.5px; color: var(--blue); } -/* ---------- decision loop ---------- */ +/* ---------- protocol (phases + IO) ---------- */ -.loop-grid { +.phase-wire { display: grid; grid-template-columns: repeat(3, 1fr); - gap: 20px; + gap: 0; + border: 1px solid var(--line-strong); + border-radius: 2px; + background: var(--card); + overflow: hidden; + margin-bottom: 18px; } @media (max-width: 900px) { - .loop-grid { + .phase-wire { grid-template-columns: 1fr; } } -.loop-card { - background: var(--card); - border: 1px solid var(--line-strong); - border-radius: 6px; - padding: 24px; +.phase-row { + padding: 18px 20px; + border-right: 1px solid var(--line); +} + +.phase-row:last-child { + border-right: none; +} + +@media (max-width: 900px) { + .phase-row { + border-right: none; + border-bottom: 1px solid var(--line); + } + + .phase-row:last-child { + border-bottom: none; + } } -.loop-num { +.phase-num { font-family: var(--font-mono); font-size: 11px; - letter-spacing: 0.1em; + letter-spacing: 0.08em; text-transform: uppercase; color: var(--blue); - margin-bottom: 12px; + margin-bottom: 8px; } -.loop-card h3 { - font-family: var(--font-display); +.phase-row h3 { + font-family: var(--font-body); font-weight: 600; - text-transform: uppercase; - font-size: 22px; - margin-bottom: 8px; + font-size: 15px; + letter-spacing: -0.01em; + text-transform: none; + margin-bottom: 6px; } -.loop-card p { - font-size: 14.5px; +.phase-row p { + font-size: 13.5px; color: var(--steel); } .protocol-grid { display: grid; grid-template-columns: minmax(0, 0.9fr) minmax(0, 1.1fr); - gap: 20px; - margin-top: 20px; + gap: 18px; align-items: start; } @@ -956,9 +890,9 @@ tbody tr.tier-start td { } .proto-point { - display: flex; - gap: 14px; - padding: 16px 4px; + display: grid; + gap: 2px; + padding: 12px 0; border-bottom: 1px solid var(--line); } @@ -966,33 +900,16 @@ tbody tr.tier-start td { border-bottom: none; } -.proto-icon { - flex-shrink: 0; - width: 36px; - height: 36px; - display: grid; - place-items: center; - border-radius: 4px; - border: 1px solid var(--line-strong); - color: var(--blue); - background: var(--card); -} - -.proto-icon svg { - width: 18px; - height: 18px; -} - .proto-point h4 { - font-family: var(--font-display); + font-family: var(--font-body); font-weight: 600; - text-transform: uppercase; - font-size: 17px; - margin-bottom: 3px; + font-size: 14px; + letter-spacing: -0.01em; + text-transform: none; } .proto-point p { - font-size: 14px; + font-size: 13.5px; color: var(--steel); } @@ -1000,7 +917,7 @@ tbody tr.tier-start td { .code-card { border: 1px solid var(--line-strong); - border-radius: 6px; + border-radius: 2px; background: var(--ink); color: #dce8f2; overflow: hidden; @@ -1010,8 +927,8 @@ tbody tr.tier-start td { display: flex; align-items: center; justify-content: space-between; - padding: 9px 16px; - border-bottom: 1px solid rgba(220, 232, 242, 0.15); + padding: 8px 14px; + border-bottom: 1px solid rgba(220, 232, 242, 0.12); font-family: var(--font-mono); font-size: 11px; color: #8fa8bd; @@ -1019,11 +936,11 @@ tbody tr.tier-start td { .code-card pre { margin: 0; - padding: 16px 18px; + padding: 14px 16px; overflow-x: auto; font-family: var(--font-mono); - font-size: 12.5px; - line-height: 1.7; + font-size: 12px; + line-height: 1.65; } .code-card code { @@ -1034,9 +951,9 @@ tbody tr.tier-start td { font-family: var(--font-mono); font-size: 11px; color: #8fa8bd; - border: 1px solid rgba(220, 232, 242, 0.25); - border-radius: 4px; - padding: 2px 10px; + border: 1px solid rgba(220, 232, 242, 0.22); + border-radius: 2px; + padding: 2px 9px; transition: color 0.15s ease, border-color 0.15s ease; } @@ -1045,54 +962,35 @@ tbody tr.tier-start td { border-color: #fff; } -/* ---------- adapters ---------- */ +/* ---------- adapters (inline in Run) ---------- */ -.adapter-grid { - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: 16px; -} - -@media (max-width: 900px) { - .adapter-grid { - grid-template-columns: repeat(2, 1fr); - } -} - -@media (max-width: 600px) { - .adapter-grid { - grid-template-columns: 1fr; - } -} - -.adapter-card { - background: var(--card); - border: 1px solid var(--line-strong); - border-radius: 6px; - padding: 20px 22px; +.adapter-line { + margin-top: 18px; + padding-top: 16px; + border-top: 1px solid var(--line); + font-size: 13.5px; + color: var(--steel); } -.adapter-card h4 { - font-family: var(--font-display); +.adapter-line strong { + color: var(--ink); font-weight: 600; - text-transform: uppercase; - font-size: 18px; - margin-bottom: 5px; } -.adapter-card p { - font-size: 13.5px; - color: var(--steel); - margin-bottom: 10px; +.adapter-chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 10px; } -.adapter-card code { +.adapter-chips code { font-size: 11.5px; color: var(--blue); border: 1px solid var(--line-strong); - border-radius: 4px; - padding: 2px 8px; - display: inline-block; + border-radius: 2px; + padding: 3px 8px; + background: var(--card); } /* ---------- transaction feed ---------- */ @@ -1100,17 +998,17 @@ tbody tr.tier-start td { .txn-list { display: flex; flex-direction: column; - max-height: 360px; + max-height: 320px; overflow-y: auto; } .txn-row { display: flex; align-items: baseline; - gap: 14px; - padding: 10px 4px; + gap: 12px; + padding: 8px 2px; border-bottom: 1px solid var(--line); - font-size: 13.5px; + font-size: 13px; } .txn-row:last-child { @@ -1120,12 +1018,12 @@ tbody tr.tier-start td { .txn-phase { font-family: var(--font-mono); font-size: 10px; - letter-spacing: 0.07em; + letter-spacing: 0.06em; text-transform: uppercase; color: var(--blue); border: 1px solid var(--line-strong); - border-radius: 3px; - padding: 1px 7px; + border-radius: 2px; + padding: 1px 6px; white-space: nowrap; } @@ -1151,45 +1049,10 @@ tbody tr.tier-start td { /* ---------- grids ---------- */ -.results-grid { - display: grid; - grid-template-columns: minmax(0, 1.25fr) minmax(0, 0.75fr); - gap: 20px; -} - -@media (max-width: 900px) { - .results-grid { - grid-template-columns: 1fr; - } -} - -.charts-grid { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 20px; - margin-top: 20px; -} - -@media (max-width: 900px) { - .charts-grid { - grid-template-columns: 1fr; - } -} - -.chart-svg { - width: 100%; - height: auto; - display: block; -} - -.chart-holder { - position: relative; -} - .quickstart-grid { display: grid; grid-template-columns: 1fr 1fr; - gap: 20px; + gap: 14px; } @media (max-width: 900px) { @@ -1198,87 +1061,44 @@ tbody tr.tier-start td { } } -/* ---------- verdict ---------- */ - -.verdict-big { - font-family: var(--font-display); - font-size: 46px; - font-weight: 700; - line-height: 1; - color: var(--blue); -} - -.verdict-big small { - font-size: 17px; - font-weight: 600; - color: var(--steel); - margin-left: 6px; - text-transform: uppercase; - letter-spacing: 0.04em; -} - -.verdict-row { - display: flex; - justify-content: space-between; - gap: 12px; - font-size: 14px; - padding: 9px 0; - border-bottom: 1px solid var(--line); -} - -.verdict-row:last-child { - border-bottom: none; -} - -.verdict-row span { - color: var(--steel); -} - -.verdict-row strong { - font-variant-numeric: tabular-nums; - font-weight: 600; -} - -/* ---------- lane note ---------- */ - .lane-note { - font-size: 15px; + font-size: 14px; color: var(--steel); max-width: 76ch; - padding: 4px 2px; + padding: 2px 0; } /* ---------- footer ---------- */ .footer { - border-top: 3px solid var(--ink); - padding: 40px 0 56px; + border-top: 2px solid var(--ink); + padding: 28px 0 40px; color: var(--steel); - font-size: 13.5px; + font-size: 13px; } .footer-inner { display: flex; flex-wrap: wrap; justify-content: space-between; - gap: 18px; + gap: 14px; align-items: center; } .footer-brand { display: grid; - gap: 5px; + gap: 4px; } .footer-brand > div { display: flex; align-items: center; - gap: 10px; + gap: 8px; } .footer .byline { width: fit-content; - margin-left: 32px; + margin-left: 30px; color: var(--ink); } @@ -1295,12 +1115,12 @@ tbody tr.tier-start td { .footer-links { display: flex; flex-wrap: wrap; - gap: 22px; + gap: 18px; } .footer .mono { font-family: var(--font-mono); - font-size: 11.5px; + font-size: 11px; color: var(--faint); } diff --git a/web/src/lib.ts b/web/src/lib.ts index cfa273f..4df4e61 100644 --- a/web/src/lib.ts +++ b/web/src/lib.ts @@ -5,11 +5,11 @@ // never rides on color alone — every mark sits next to its label. export const COLOR = { red: "#C8102E", - blue: "#155B9A", + blue: "#1A5F8F", steel: "#7E93A6", - ink: "#0C1D2E", + ink: "#0A1620", faint: "#8AA0B2", - grid: "#D9E4EB", + grid: "#C5D3DE", } as const; export function agentColor(agent: string): string { From 927c16bfefa538af39e84472f3e63ca9a94dcb00 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 21 Jul 2026 18:29:52 +0000 Subject: [PATCH 2/5] Remove unused hero-todo CSS rule Co-authored-by: Ned Cutler --- web/src/index.css | 7 ------- 1 file changed, 7 deletions(-) diff --git a/web/src/index.css b/web/src/index.css index 19d1602..d993113 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -285,13 +285,6 @@ code { font-weight: 500; } -.hero-todo { - font-family: var(--font-mono); - font-size: 11px; - color: var(--faint); - margin-top: 8px; -} - /* ---------- score ladder ---------- */ .ladder-panel { From 747f4fbe1a7edf05e7516b787a92d9f50dbbc1cd Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Wed, 22 Jul 2026 11:47:22 -0400 Subject: [PATCH 3/5] Clarify publication claims and refresh evidence dashboard --- tests/test_publication_claims.py | 7 +- web/index.html | 2 +- web/mockups/redesign-mockups.html | 252 ++++++++++++++++++++++ web/src/components/Footer.tsx | 3 + web/src/components/Hero.tsx | 150 ++++++++++---- web/src/components/HowItWorks.tsx | 14 +- web/src/components/Ladder.tsx | 35 ++-- web/src/components/Leaderboard.tsx | 13 +- web/src/components/Nav.tsx | 34 ++- web/src/index.css | 321 ++++++++++++++++++++++++++--- web/src/lib.ts | 11 +- web/src/theme.ts | 30 +++ 12 files changed, 779 insertions(+), 93 deletions(-) create mode 100644 web/mockups/redesign-mockups.html create mode 100644 web/src/theme.ts diff --git a/tests/test_publication_claims.py b/tests/test_publication_claims.py index 43e1340..aa336c9 100644 --- a/tests/test_publication_claims.py +++ b/tests/test_publication_claims.py @@ -53,9 +53,10 @@ def test_employer_facing_site_links_current_evidence_and_metadata() -> None: footer = Path("web/src/components/Footer.tsx").read_text() index = Path("web/index.html").read_text() - assert "0 beat the scripted bar" in hero - assert "sota-v2-phase-one-2026-07-19" in hero - assert "Built by Ned Cutler" in footer + assert "exceeded the scripted bar’s observed mean" in hero + assert "not an ordinal ranking" in hero + assert "sota-v2-phase-one-2026-07-19" in footer + assert "Ned Cutler" in footer assert "REPRODUCING_SOTA_V2_RELEASE.md" in footer assert 'href="%BASE_URL%favicon.svg"' in index assert 'rel="canonical" href="https://nedcut.github.io/gm-bench/"' in index diff --git a/web/index.html b/web/index.html index 295dd81..11e609e 100644 --- a/web/index.html +++ b/web/index.html @@ -28,7 +28,7 @@ diff --git a/web/mockups/redesign-mockups.html b/web/mockups/redesign-mockups.html new file mode 100644 index 0000000..aaf9871 --- /dev/null +++ b/web/mockups/redesign-mockups.html @@ -0,0 +1,252 @@ + + + + + +GM-Bench — redesign mockups + + + + + + + +

GM-Bench — redesign directions

+

Three benchmark-forward directions using the live 558e8f35ea1d66b9 data (8 models · 8 seeds × 5 seasons × 3 repeats · 4,096-token ceiling). The goal: demote marketing copy, promote the grid — closer to Artificial Analysis than a launch page.

+ + +
+
gm-bench.devA · Light dashboard
+
+
+
GM·Bench
+ +
+
+

Franchise-GM benchmark

+

Agents run a multi-season hockey franchise — drafts, trades, cap, lineups. Scored on paired lift vs a scripted panel. Sorted by mean score.

+
+
+
Models evaluated
8
all in one statistical tier
+
Beat the bar
0 / 8
none above pick-trader
+
Best model
231.9
muse-spark-1.1 · −41.9 lift
+
Human-scripted bar
411.6
oracle ceiling 431.2
+
+
+
+

Standings

+
+
+ + + + + + + + + + + + +
ModelScoreLift vs panel95% CICost/epTier
1muse-spark-1.1
meta · openrouter
231.9−41.9[−66.1, −16.3]$0.31Tier 1
2glm-5.2
z-ai · openrouter
217.5−56.3[−82.0, −30.5]$0.28Tier 1
3gemini-3.5-flash
google · openrouter
215.6−58.2[−89.6, −27.2]$0.19Tier 1
4hy3:free
tencent · openrouter
195.8−78.0[−91.5, −63.9]$0.00Tier 1
5qwen3.7-plus
qwen · openrouter
175.5−98.3[−116.8, −73.1]$0.22Tier 1
6gpt-5.6-luna
openai · openrouter
173.9−99.9[−114.2, −83.1]$0.35Tier 1
—pick-trader the bar to beat
scripted heuristic
411.6baseline—$0.00
▲oracle ceiling
sees hidden info
431.2———
+
+
+
+
A · Light dashboard (closest to Artificial Analysis). No verdict headline — a plain title, a 4-tile KPI strip, then the standings table with in-cell bars and lane tabs. The pick-trader "bar" and oracle "ceiling" live inside the same sorted table so the gap is obvious at a glance. Most neutral / most benchmarky.
+ + +
+
gm-bench.devB · Dark terminal
+
+
+
gm_bench
+
contract 558e8f35 · updated 2026-07-19 · registry frozen
+
+
+

Franchise-GM benchmark — phase one

+

8 models · 8 seeds × 5 seasons × 3 repeats · 4,096-token ceiling · reasoning off · paired lift vs scripted panel

+
+
+
models
8
+
beat bar
0/8
+
best score
231.9
+
scripted bar
411.6
+
oracle ceiling
431.2
+
+
+
standings · api lane · sorted by mean score
+ + + + + + + + + + + + + + +
#modelscoreliftci95vs oracle
01muse-spark-1.1231.9−41.9[−66, −16]
02glm-5.2217.5−56.3[−82, −30]
03gemini-3.5-flash215.6−58.2[−90, −27]
04hy3:free195.8−78.0[−92, −64]
05qwen3.7-plus175.5−98.3[−117, −73]
06gpt-5.6-luna173.9−99.9[−114, −83]
07claude-sonnet-5142.1−131.7[−154, −104]
08minimax-m3129.9−143.9[−167, −116]
—pick-trader · bar411.6—scripted
▲oracle · ceiling431.2—hidden info
+ □ ranking withheld until every publication gate clears +
+
+
+
B · Dark terminal. Same density as A but in a lab-notebook register — monospace body, contract hash in the chrome, KPIs as a hairline-divided stat line. Reads as "research artifact." Signals rigor hard; the full 8-row table + gate note make the withheld-ranking story native rather than a marketing gate card.
+ + +
+
gm-bench.devC · Gap-chart hero
+
+
+
GM·Bench
+ +
+
+
+

Eight frontier models. None beat a scripted GM.

+

Agents run a multi-season franchise against eight heuristics. The best model lands 180 points under the bar and 200 under an oracle that sees hidden information.

+
8 seeds × 5 seasons × 3 repeats · 4,096-token ceiling · contract 558e8f35
+
+
+
Mean score — model vs bar vs ceiling0 → 431
+
oracle
431.2
+
pick-trader
411.6
+
muse-spark-1.1
231.9
+
glm-5.2
217.5
+
gemini-3.5-flash
215.6
+
+ 5 more models
↓130
+
+
+
+
+
C · Gap-chart hero (a middle ground). Keeps one editorial headline but pays for it immediately with the chart that proves it — every model bar stops far short of the red line. If you want the story up top but still benchmark-first below the fold, this is the compromise. Warning: the serif headline is the most "markety" of the three — swap to Inter to cool it down.
+ + + diff --git a/web/src/components/Footer.tsx b/web/src/components/Footer.tsx index 7f6903a..d58ee55 100644 --- a/web/src/components/Footer.tsx +++ b/web/src/components/Footer.tsx @@ -25,6 +25,9 @@ export default function Footer({ data }: { data: LeaderboardData }) { Reproduce + + Decision log +
{data.contract diff --git a/web/src/components/Hero.tsx b/web/src/components/Hero.tsx index 22c9ec0..4f86579 100644 --- a/web/src/components/Hero.tsx +++ b/web/src/components/Hero.tsx @@ -1,49 +1,127 @@ import type { Leaderboard as LeaderboardData } from "../types"; -import Ladder from "./Ladder"; +import { fmt } from "../lib"; + +const COUNT_WORDS: Record = { + 1: "One", + 2: "Two", + 3: "Three", + 4: "Four", + 5: "Five", + 6: "Six", + 7: "Seven", + 8: "Eight", + 9: "Nine", + 10: "Ten", +}; + +function shortName(model: string): string { + return model.split("/").pop() ?? model; +} + +interface Bar { + name: string; + score: number; + kind: "ceiling" | "bar" | "model" | "rest"; +} export default function Hero({ data }: { data: LeaderboardData }) { - const fingerprint = data.contract?.contract_fingerprint; const cap = data.publication.frozen_output_token_cap; - const registryFrozen = data.publication.model_registry_frozen === true; + const fingerprint = data.contract?.contract_fingerprint; const modelCount = data.models.length; + const baselineCount = data.baselines.length; + const countWord = COUNT_WORDS[modelCount] ?? String(modelCount); + + const oracle = data.headroom.oracle; + const bar = data.baselines.find((b) => b.agent === "pick-trader")?.mean_score ?? oracle; + const ranked = [...data.models].sort((a, b) => b.mean_score - a.mean_score); + const best = ranked[0] ?? null; + const topModels = ranked.slice(0, 3); + const rest = ranked.slice(3); + const restMin = rest.length > 0 ? Math.min(...rest.map((m) => m.mean_score)) : null; + + // Oracle anchors the scale; the red line sits where the scripted bar falls, so + // every model bar visibly stops short of it — the gap is the whole message. + const scaleMax = oracle; + const width = (score: number) => `${(score / scaleMax) * 100}%`; + const redLineLeft = `${(bar / scaleMax) * 100}%`; + + const underBar = best ? Math.round(bar - best.mean_score) : 0; + const underOracle = best ? Math.round(oracle - best.mean_score) : 0; + + const bars: Bar[] = [ + { name: "oracle", score: oracle, kind: "ceiling" }, + { name: "pick-trader", score: bar, kind: "bar" }, + ...topModels.map((m) => ({ name: shortName(m.model), score: m.mean_score, kind: "model" })), + ]; return (
-
-

- GM-Bench. -

- {/* LEARNING TODO: rewrite this h1 in your own wire voice (5–10 words). - Keep the finding; drop any leftover marketing tone. */} -

Phase one: 0 of {modelCount} models beat the scripted bar

-

- Agents run a franchise in a seeded hockey-style league — contracts, trades, drafts, cap - pressure — across multi-season episodes. Eight scripted heuristics set the bar; a - hidden-information oracle sets the ceiling. -

-
- - Open standings - +
+
+

+ GM-Bench. +

+

+ {countWord} model-plus-scaffold systems. None exceeded the scripted bar’s observed mean. +

+

+ Under the frozen compact, fresh-spawn/memo-only, native-minimum-reasoning protocol, agents run a + multi-season franchise against {baselineCount} scripted reference policies. The best model lands {underBar} + points under the bar and {underOracle} under a partial oracle that sees hidden information. +

+

+ Observed means, not an ordinal ranking: all eight rows overlap in one tier and the full-family Holm test + does not reject at 0.05. Public seeds and committed baselines also permit benchmark-specific adaptation. +

+

+ {data.preset.seeds.length} seeds × {data.preset.seasons} seasons × 3 repeats + {cap && ( + <> + {" "} + · {cap.toLocaleString("en-US")}-token ceiling + + )} + {fingerprint && ( + <> + {" "} + · contract {fingerprint.slice(0, 8)} + + )} +

-

- {fingerprint && ( - <> - contract {fingerprint} ·{" "} - - )} - {data.preset.seeds.length} seeds × {data.preset.seasons} seasons × 3 repeats - {cap && ( - <> - {" "} - · {cap.toLocaleString("en-US")}-token output ceiling · reasoning off - + +

+
+ mean score — model vs bar vs ceiling + 0 → {Math.round(scaleMax)} +
+ {bars.map((b) => ( +
+ {b.name} +
+ + {b.kind !== "ceiling" && ( + + )} +
+ {fmt(b.score, 1)} +
+ ))} + {rest.length > 0 && ( +
+ + {rest.length} more models +
+ + +
+ {restMin === null ? "—" : `↓${Math.round(restMin)}`} +
)} - {registryFrozen ? " · routes pinned" : " · routes pending smoke verification"} -

-
-
- +

+ red line = pick-trader {fmt(bar, 1)}, the scripted bar to beat · oracle sees hidden + information +

+
); diff --git a/web/src/components/HowItWorks.tsx b/web/src/components/HowItWorks.tsx index 1f78560..4529659 100644 --- a/web/src/components/HowItWorks.tsx +++ b/web/src/components/HowItWorks.tsx @@ -9,7 +9,7 @@ const PHASES = [ { num: "02 · trade deadline", title: "Trades", - body: "Swap with eleven AI rivals mid-season. Partners apply hidden valuation noise each season. Illegal proposals are rejected and penalized.", + body: "Swap with eleven scripted opponent offices mid-season. Partners apply hidden valuation noise each season. Illegal proposals are rejected and penalized.", }, { num: "03 · draft", @@ -25,7 +25,7 @@ const PROTO_POINTS = [ }, { title: "Actions on stdout", - body: "JSON array of actions. Core verbs: sign_free_agent, trade, draft, set_lineup, memo — plus release and noop.", + body: "A production envelope with actions and optional usage telemetry. Core verbs: sign_free_agent, trade, draft, set_lineup, memo — plus release and noop.", }, { title: "Deterministic replay", @@ -55,7 +55,8 @@ const OBSERVATION_SNIPPET = `{ "standings": [ /* 12 teams */ ] }`; -const ACTIONS_SNIPPET = `[ +const ACTIONS_SNIPPET = `{ + "actions": [ { "type": "sign_free_agent", "player_id": 294, "years": 3, "salary": 4.1 }, { "type": "trade", "partner_team_id": 3, @@ -67,7 +68,9 @@ const ACTIONS_SNIPPET = `[ 11, 12, 13, 14, 15, 16, 17, 18] }, { "type": "memo", "text": "push for playoff spot; revisit D depth at deadline" } -]`; + ], + "usage": { "input_tokens": 1200, "output_tokens": 340 } +}`; function CodeCard({ title, code }: { title: string; code: string }) { return ( @@ -91,7 +94,8 @@ export default function HowItWorks({ snapshot }: { snapshot: Snapshot }) {

Three decision points per season. Five seasons per episode.

No browser automation, no memorized rosters — every player is fictional. Agents speak a - minimal JSON protocol any process can run. + minimal JSON protocol any process can run. Under the published scaffold, reading that JSON/API contract is + part of what the benchmark measures.

diff --git a/web/src/components/Ladder.tsx b/web/src/components/Ladder.tsx index ade4226..38b3391 100644 --- a/web/src/components/Ladder.tsx +++ b/web/src/components/Ladder.tsx @@ -1,6 +1,17 @@ import { useState } from "react"; import type { Leaderboard as LeaderboardData } from "../types"; -import { COLOR, fmt } from "../lib"; +import { fmt } from "../lib"; + +/* SVG marks read the same CSS variables as the tables, so the theme toggle + recolors the chart and the standings from one cascade. */ +const MARK = { + axis: "var(--mark-axis)", + ref: "var(--mark-ref)", + faint: "var(--mark-faint)", + grid: "var(--mark-grid)", + red: "var(--red)", + steel: "var(--steel)", +} as const; const W = 1080; const H = 260; @@ -106,21 +117,21 @@ export default function Ladder({ data }: { data: LeaderboardData }) { - + {/* axis + ticks */} - + {ticks.map((tick) => ( - + {tick} @@ -135,16 +146,15 @@ export default function Ladder({ data }: { data: LeaderboardData }) { width={x(domainMax) - x(0)} height={34} fill="url(#pending-hatch)" - stroke={COLOR.grid} + style={{ stroke: MARK.grid }} /> {pending.publishable_ranking ? "MODEL ROWS PUBLISHED ABOVE" @@ -155,7 +165,7 @@ export default function Ladder({ data }: { data: LeaderboardData }) { {placed.map((marker, index) => { const isBar = marker.kind === "bar"; const isCeiling = marker.kind === "ceiling"; - const stroke = isBar ? COLOR.red : isCeiling ? COLOR.ink : COLOR.steel; + const stroke = isBar ? MARK.red : isCeiling ? MARK.axis : MARK.ref; const key = isBar || isCeiling || marker.kind === "floor"; return ( @@ -193,9 +203,8 @@ export default function Ladder({ data }: { data: LeaderboardData }) { textAnchor={marker.anchor} fontSize={isBar ? "14" : "11.5"} fontWeight={isBar ? 700 : 500} - fill={stroke} + style={{ fill: stroke, textTransform: "uppercase", letterSpacing: "0.05em" }} fontFamily="Barlow Condensed, sans-serif" - style={{ textTransform: "uppercase", letterSpacing: "0.05em" }} > {marker.name} @@ -205,7 +214,7 @@ export default function Ladder({ data }: { data: LeaderboardData }) { y={marker.labelY + 3} textAnchor={marker.anchor} fontSize="10.5" - fill={isBar ? COLOR.red : COLOR.faint} + style={{ fill: isBar ? MARK.red : MARK.faint }} fontFamily="IBM Plex Mono, monospace" > {isBar ? `${fmt(marker.score, 1)} · THE RED LINE` : fmt(marker.score, 1)} diff --git a/web/src/components/Leaderboard.tsx b/web/src/components/Leaderboard.tsx index 16f8f2f..392fc32 100644 --- a/web/src/components/Leaderboard.tsx +++ b/web/src/components/Leaderboard.tsx @@ -95,6 +95,7 @@ function ModelTable({ {withTiers && Tier} Model Score + Output/dec Lift vs panel [95% CI] Cost/episode Status @@ -130,6 +131,9 @@ function ModelTable({ {fmt(model.mean_score, 1)} + + {model.output_tokens_per_decision === null ? "—" : fmt(model.output_tokens_per_decision, 0)} + {liftCell(model)} {cost(model)} {statusTag(model)} @@ -142,9 +146,6 @@ function ModelTable({ {model.input_tokens_per_decision === null ? "—" : fmt(model.input_tokens_per_decision, 0)} - - {model.output_tokens_per_decision === null ? "—" : fmt(model.output_tokens_per_decision, 0)} - {latency === null ? "—" : `${fmt(latency, 1)}s`} @@ -167,6 +168,7 @@ function ModelTable({ )} lift vs panel = paired per-seed difference against the scripted-baseline mean fallback = decisions answered by the adapter policy, not the model + output/dec stays beside score because same-lane ceilings do not imply equal effective compute
); @@ -401,6 +403,11 @@ export default function Leaderboard({ data }: { data: LeaderboardData }) { against the scripted panel.

+
+ Why there is no #1: this table is publishable evidence, not a resolved ranking. All eligible + models share an overlapping uncertainty tier; the unadjusted all-seed result is descriptive, while the + predeclared Holm-adjusted family result does not reject at 0.05. +
{!publishable && } {publishable && (