Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions .semgrep.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
rules:
# These three rules preserve the pinned upstream definitions and IDs. The
# pattern-not clauses are the narrow, reviewed exceptions from PR #67.
# These four rules preserve the pinned upstream definitions and IDs. The
# pattern-not clauses are narrow, reviewed exceptions to false positives.
- id: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected
patterns:
- pattern-not: urllib.$W("...")
Expand Down Expand Up @@ -89,3 +89,39 @@ rules:
...
- pattern: $CONNECTION.execute($QUERY, ...)
- pattern-not: connection.execute(f"ALTER TABLE episodes ADD COLUMN {name} {column_type}")

- id: html.security.audit.missing-integrity.missing-integrity
patterns:
- pattern-either:
- pattern: <script $...A >...</script>
- pattern: <link $...A >
- metavariable-pattern:
metavariable: $...A
patterns:
- pattern-either:
- pattern: src='... :// ...'
- pattern: src="... :// ..."
- pattern: href='... :// ...'
- pattern: href="... :// ..."
- pattern: src='//...'
- pattern: src="//..."
- pattern: href='//...'
- pattern: href="//..."
- pattern-not-regex: (?is).*integrity=.*
- pattern-not-regex: (google-analytics\.com|fonts\.googleapis\.com|fonts\.gstatic\.com|googletagmanager\.com)
- pattern-not-regex: .*rel\s*=\s*['"]?preconnect.*
# Canonical metadata identifies the document; it fetches no resource.
- pattern-not-regex: .*rel\s*=\s*['"]?canonical.*
paths:
include:
- "*.html"
message: >-
This tag is missing an 'integrity' subresource integrity attribute. The
'integrity' attribute allows for the browser to verify that externally hosted
files (for example from a CDN) are delivered without unexpected manipulation.
Without this attribute, if an attacker can modify the externally hosted resource,
this could lead to XSS and other types of attacks. To prevent this, include the
base64-encoded cryptographic hash of the resource (file) you're telling the
browser to fetch in the 'integrity' attribute for all externally hosted files.
severity: WARNING
languages: [generic]
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ front-office dynasty games. Agents manage a fictional hockey-style league over
multiple seasons by making roster, trade, free-agency, draft, and lineup
decisions through a JSON-compatible API.

Explore the [live leaderboard](https://nedcut.github.io/gm-bench/), read the
[phase-one findings](docs/blog/sota-v2-findings.md), or download the
[tagged evidence release](https://github.com/nedcut/gm-bench/releases/tag/sota-v2-phase-one-2026-07-19).

## Phase-one result

Eight pre-registered model systems produced strict `sota-v2` rows under one
Expand All @@ -19,7 +23,7 @@ Read the [phase-one findings](docs/blog/sota-v2-findings.md), inspect the
[generated analysis](results/analysis/publication-panel-analysis.json), or
follow the [clean-clone reproduction guide](docs/REPRODUCING_SOTA_V2_RELEASE.md).

The MVP includes:
GM-Bench includes:

- A seeded league simulator with aging, development, injuries, contracts,
salary-cap pressure, free agency, trades, drafts, playoffs, and team standings.
Expand Down
1 change: 1 addition & 0 deletions scripts/prepare_semgrep_ruleset.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected",
"python.lang.security.audit.formatted-sql-query.formatted-sql-query",
"python.sqlalchemy.security.sqlalchemy-execute-raw-query.sqlalchemy-execute-raw-query",
"html.security.audit.missing-integrity.missing-integrity",
}
)

Expand Down
15 changes: 15 additions & 0 deletions tests/test_publication_claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,18 @@ def test_public_claim_surfaces_use_the_frozen_lane_and_current_gate() -> None:
assert "1,024-token safety ceiling" not in blog
assert "`publishable_ranking: true`" in readme
assert "no ordinal \u201cbest model\u201d claim" in " ".join(readme.split())


def test_employer_facing_site_links_current_evidence_and_metadata() -> None:
hero = Path("web/src/components/Hero.tsx").read_text()
footer = Path("web/src/components/Footer.tsx").read_text()
index = Path("web/index.html").read_text()

assert "0 beat the scripted bar" in hero
assert "sota-v2-phase-one-2026-07-19" in hero
assert "Built by Ned Cutler" in footer
assert "REPRODUCING_SOTA_V2_RELEASE.md" in footer
assert 'href="%BASE_URL%favicon.svg"' in index
assert 'rel="canonical" href="https://nedcut.github.io/gm-bench/"' in index
assert 'property="og:title"' in index
assert "leaderboard that stays empty" not in index
19 changes: 17 additions & 2 deletions web/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,27 @@
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<link rel="icon" type="image/svg+xml" href="%BASE_URL%favicon.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="theme-color" content="#F4F7F9" />
<meta
name="description"
content="GM-Bench asks whether a language model can out-manage a scripted front office. A deterministic, pre-registered benchmark: seeded leagues, frozen contract, paired baseline scoring — and a leaderboard that stays empty until the evidence is complete."
content="GM-Bench is a deterministic, pre-registered test of long-horizon LLM planning. Eight models were evaluated under one frozen lane; none beat the strongest scripted heuristic."
/>
<!-- Canonical metadata is not a fetched subresource, so SRI does not apply. -->
<link rel="canonical" href="https://nedcut.github.io/gm-bench/" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://nedcut.github.io/gm-bench/" />
<meta property="og:title" content="GM-Bench — Can a language model run a front office?" />
<meta
property="og:description"
content="Eight models were evaluated in a deterministic, pre-registered front-office benchmark. None beat the strongest scripted heuristic."
/>
<meta name="twitter:card" content="summary" />
<meta name="twitter:title" content="GM-Bench — Can a language model run a front office?" />
<meta
name="twitter:description"
content="Eight models were evaluated under one frozen lane. None beat the strongest scripted heuristic."
/>
<title>GM-Bench — Can a language model run a front office?</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
Expand Down
23 changes: 17 additions & 6 deletions web/src/components/Footer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,26 @@ export default function Footer({ data }: { data: LeaderboardData }) {
return (
<footer className="footer">
<div className="shell footer-inner">
<div style={{ display: "flex", alignItems: "center", gap: 10 }}>
<Logo size={22} />
<span>GM-Bench · a pre-registered front-office benchmark for LLM agents</span>
<div className="footer-brand">
<div>
<Logo size={22} />
<span>GM-Bench · a pre-registered front-office benchmark for LLM agents</span>
</div>
<a className="byline" href="https://github.com/nedcut">
Built by Ned Cutler
</a>
</div>
<div className="footer-links">
<a href="https://github.com/nedcut/gm-bench">GitHub</a>
<a href="#leaderboard">The board</a>
<a href="#protocol">Protocol</a>
<a href="#quickstart">Quickstart</a>
<a href="https://github.com/nedcut/gm-bench/blob/main/docs/blog/sota-v2-findings.md">
Findings
</a>
<a href="https://github.com/nedcut/gm-bench/releases/tag/sota-v2-phase-one-2026-07-19">
Evidence release
</a>
<a href="https://github.com/nedcut/gm-bench/blob/main/docs/REPRODUCING_SOTA_V2_RELEASE.md">
Reproduce
</a>
</div>
<span className="mono">
{data.contract
Expand Down
20 changes: 17 additions & 3 deletions web/src/components/Hero.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,15 @@ export default function Hero({ data }: { data: LeaderboardData }) {
return (
<section className="hero" id="top">
<div className="shell">
<p className="hero-eyebrow">A pre-registered front-office benchmark for LLM agents</p>
<p className="hero-eyebrow">
<span>A pre-registered front-office benchmark for LLM agents</span>
<a href="https://github.com/nedcut">By Ned Cutler</a>
</p>
<a className="hero-result" href="#leaderboard">
<span>Phase-one result</span>
<strong>{data.models.length} models tested. 0 beat the scripted bar.</strong>
<i aria-hidden="true">View evidence →</i>
</a>
<h1>
Can a language model out-manage a <span className="bar-word">scripted front office</span>?
</h1>
Expand All @@ -22,8 +30,14 @@ export default function Hero({ data }: { data: LeaderboardData }) {
<a className="btn-primary" href="#leaderboard">
See the board
</a>
<a className="btn-ghost" href="#protocol">
Read the protocol
<a
className="btn-ghost"
href="https://github.com/nedcut/gm-bench/blob/main/docs/blog/sota-v2-findings.md"
>
Read the findings
</a>
<a className="btn-text" href="https://github.com/nedcut/gm-bench/releases/tag/sota-v2-phase-one-2026-07-19">
Open the evidence release
</a>
</div>
<p className="hero-facts">
Expand Down
87 changes: 50 additions & 37 deletions web/src/components/Leaderboard.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { useState } from "react";
import type { Leaderboard as LeaderboardData, LeaderboardModel, TieredLeaderboardModel } from "../types";
import { agentColor, COLOR, fmt, pct } from "../lib";
import { agentColor, fmt, pct } from "../lib";

const PROVIDER_LABELS: Record<string, string> = {
openai: "OpenAI API",
Expand Down Expand Up @@ -28,8 +29,12 @@ function statusTag(model: LeaderboardModel) {
sota-v2
</span>
{allIssues.length > 0 && (
<span className="tag tag-warn" title={issues}>
{allIssues.length} warning{allIssues.length === 1 ? "" : "s"}
<span
className="tag tag-warn"
title={issues}
aria-label={`${allIssues.length} protocol flags: ${issues}`}
>
{allIssues.length} flag{allIssues.length === 1 ? "" : "s"}
</span>
)}
</span>
Expand All @@ -55,42 +60,53 @@ function liftCell(model: LeaderboardModel): string {
}

function ModelTable({
data,
models,
title,
subtitle,
withTiers,
}: {
data: LeaderboardData;
models: Array<LeaderboardModel | TieredLeaderboardModel>;
title: string;
subtitle: string;
withTiers: boolean;
}) {
const maxScore = Math.max(data.headroom.oracle, ...models.map((model) => model.mean_score));
const [showTelemetry, setShowTelemetry] = useState(false);
let previousTier: number | undefined;
return (
<div className="panel">
<div className="panel-title">
<h3>{title}</h3>
<span>{subtitle}</span>
<div className="panel-title-actions">
<span>{subtitle}</span>
<button
className="telemetry-toggle"
type="button"
aria-expanded={showTelemetry}
onClick={() => setShowTelemetry((visible) => !visible)}
>
{showTelemetry ? "Hide full telemetry" : "Show full telemetry"}
</button>
</div>
</div>
<div className="table-wrap">
<div className="table-wrap" tabIndex={0} role="region" aria-label={`${title} table`}>
<table>
<thead>
<tr>
{withTiers && <th>Tier</th>}
<th>Model</th>
<th>Status</th>
<th className="num">Score</th>
<th></th>
<th className="num">Lift vs panel [95% CI]</th>
<th className="num">Fallback</th>
<th className="num">Failed queries</th>
<th className="num">Input/dec</th>
<th className="num">Output/dec</th>
<th className="num">Cost/episode</th>
<th className="num">Latency</th>
<th>Status</th>
{showTelemetry && (
<>
<th className="num">Fallback</th>
<th className="num">Failed queries</th>
<th className="num">Input/dec</th>
<th className="num">Output/dec</th>
<th className="num">Latency</th>
</>
)}
</tr>
</thead>
<tbody>
Expand All @@ -113,29 +129,27 @@ function ModelTable({
<span className="tag tag-candidate">{PROVIDER_LABELS[model.provider] ?? model.provider}</span>
</span>
</td>
<td>{statusTag(model)}</td>
<td className="num score-strong">{fmt(model.mean_score, 1)}</td>
<td>
<div className="bar-track">
<div
className="bar-fill"
style={{ width: `${Math.max(0, (model.mean_score / maxScore) * 100)}%`, background: COLOR.blue }}
/>
</div>
</td>
<td className="num mono-dim">{liftCell(model)}</td>
<td className="num mono-dim">{pct(model.fallback_rate)}</td>
<td className="num mono-dim">{model.failed_queries === undefined ? "—" : model.failed_queries}</td>
<td className="num mono-dim">
{model.input_tokens_per_decision === null ? "—" : fmt(model.input_tokens_per_decision, 0)}
</td>
<td className="num mono-dim">
{model.output_tokens_per_decision === null ? "—" : fmt(model.output_tokens_per_decision, 0)}
</td>
<td className="num mono-dim">{cost(model)}</td>
<td className="num mono-dim">
{latency === null ? "—" : `${fmt(latency, 1)}s`}
</td>
<td>{statusTag(model)}</td>
{showTelemetry && (
<>
<td className="num mono-dim">{pct(model.fallback_rate)}</td>
<td className="num mono-dim">
{model.failed_queries === undefined ? "—" : model.failed_queries}
</td>
<td className="num mono-dim">
{model.input_tokens_per_decision === null ? "—" : fmt(model.input_tokens_per_decision, 0)}
</td>
<td className="num mono-dim">
{model.output_tokens_per_decision === null ? "—" : fmt(model.output_tokens_per_decision, 0)}
</td>
<td className="num mono-dim">
{latency === null ? "—" : `${fmt(latency, 1)}s`}
</td>
</>
)}
</tr>
);
})}
Expand All @@ -152,6 +166,7 @@ function ModelTable({
<span>observational lane — uncontrolled tool loops, context, and retries; no tiers, not comparable to the API lane</span>
)}
<span>lift vs panel = paired per-seed difference against the full scripted-baseline mean</span>
<span>flags are visible protocol observations; eligible rows still passed every frozen publication gate</span>
<span>fallback = decisions answered by the adapter's fallback policy, not the model</span>
<span>cost from measured tokens × published prices; read score next to input/output tokens and cost, never alone</span>
</div>
Expand Down Expand Up @@ -359,7 +374,6 @@ export default function Leaderboard({ data }: { data: LeaderboardData }) {
{!publishable && <Gate data={data} />}
{publishable && (
<ModelTable
data={data}
models={data.models}
title="Official API leaderboard"
subtitle={`frozen ${data.publication.frozen_output_token_cap?.toLocaleString("en-US")}-token ceiling · reasoning off · ${data.updated ? `updated ${data.updated}` : "sota-v2"}`}
Expand All @@ -378,7 +392,6 @@ export default function Leaderboard({ data }: { data: LeaderboardData }) {
observational table and never mix with the API lane.
</p>
<ModelTable
data={data}
models={data.cli_harness_models}
title="Coding-harness lane"
subtitle="observational · separate table by design"
Expand Down
Loading