From 7523dd70dbddef502ada914c38ec20f958fd4965 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Sun, 19 Jul 2026 21:35:01 -0400 Subject: [PATCH] Package the phase-one benchmark release --- README.md | 28 +- docs/PUBLISH_READINESS.md | 34 +- docs/REPRODUCING_SOTA_V2_RELEASE.md | 88 +++++ docs/blog/sota-v2-findings.md | 162 ++++++--- .../sota-v2-phase-one-2026-07-19/README.md | 32 ++ .../SHA256SUMS.txt | 1 + .../manifest.json | 318 ++++++++++++++++++ scripts/package_publication_release.py | 241 +++++++++++++ tests/test_publication_claims.py | 48 +++ tests/test_publication_release.py | 96 ++++++ 10 files changed, 981 insertions(+), 67 deletions(-) create mode 100644 docs/REPRODUCING_SOTA_V2_RELEASE.md create mode 100644 releases/sota-v2-phase-one-2026-07-19/README.md create mode 100644 releases/sota-v2-phase-one-2026-07-19/SHA256SUMS.txt create mode 100644 releases/sota-v2-phase-one-2026-07-19/manifest.json create mode 100644 scripts/package_publication_release.py create mode 100644 tests/test_publication_claims.py create mode 100644 tests/test_publication_release.py diff --git a/README.md b/README.md index 3ece48e..21ccc07 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,24 @@ -# GM-Bench MVP +# GM-Bench GM-Bench is a small, deterministic sports general-manager benchmark inspired by front-office dynasty games. Agents manage a fictional hockey-style league over multiple seasons by making roster, trade, free-agency, draft, and lineup decisions through a JSON-compatible API. +## Phase-one result + +Eight pre-registered model systems produced strict `sota-v2` rows under one +frozen 4,096-token API lane. Every eligible model trailed the transparent +`pick-trader` heuristic (411.619), and all eight model uncertainty intervals +form one overlapping tier—so the evidence supports no ordinal “best model” +claim. Muse Spark 1.1 had the highest observed model mean at 231.851. Grok 4.5 +and Mistral Medium 3.5 completed but were excluded from the headline table for +incomplete usage/cost coverage. + +Read the [phase-one findings](docs/blog/sota-v2-findings.md), inspect the +[generated analysis](results/analysis/publication-panel-analysis.json), or +follow the [clean-clone reproduction guide](docs/REPRODUCING_SOTA_V2_RELEASE.md). + The MVP includes: - A seeded league simulator with aging, development, injuries, contracts, @@ -67,17 +81,17 @@ cd web && bun install && bun run dev # local site The site in `web/` deploys to GitHub Pages automatically on pushes to `main` (`.github/workflows/pages.yml`). -The provisional headline API lane uses the common 4,096-token safety ceiling in +The frozen phase-one headline API lane uses the common 4,096-token safety ceiling in [`config/sota_v2_lane.json`](config/sota_v2_lane.json), with native-minimum reasoning and actual token/cost efficiency reported beside score. Reasoning is disabled where optional and set to the lowest OpenRouter-supported effort where mandatory. The earlier four-cap study in [`config/output_budget_sweep.json`](config/output_budget_sweep.json) is -retained for auditability but retired as a publication prerequisite. Until all -registered routes pass the common smoke, the registry is frozen, and at least -eight strictly eligible fixed-cap rows exist, the generated site data says -`publishable_ranking: false`. Coding-harness rows are always emitted to a -separate table from the API headline lane. +retained for auditability but retired as a publication prerequisite. The common +smoke and eight-model minimum gates are now satisfied; generated site data +reports `publishable_ranking: true`. That flag permits the evidence table to be +shown, but the overlapping uncertainty tier still forbids an ordinal winner +claim. Coding-harness rows remain separate from the API headline lane. ### v2 changes diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 2bce012..f0bb121 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -51,7 +51,8 @@ It is that the project: That combination demonstrates benchmark design, simulation, agent evaluation, statistical reasoning, reliability engineering, and scientific judgment. The -remaining work is to finish one frozen experiment and present it clearly. +remaining work is to package, independently verify, and present the frozen +experiment clearly. ## What “publish-ready” means @@ -138,7 +139,7 @@ before any full-panel score is visible. scaffold, compact profile, one repair, and fresh-spawn execution. - [x] Freeze exact model routes and the registry only after all ten phase-one smokes are accepted. -- [ ] Validate the provisional common 4,096-token safety ceiling with +- [x] Validate the provisional common 4,096-token safety ceiling with native-minimum reasoning. - [x] Predeclare exact provider slugs/tags and the 75% cap-pressure rule: raise the entire lane to 8,192 before full results if any smoke call reaches 3,072 @@ -150,27 +151,29 @@ before any full-panel score is visible. - [x] Smoke all ten provisional phase-one models serially at 4,096 using the hardened scaffold. Start with dry-run and endpoint preflight; then run one paid model, inspect it, and approve the next rather than launching the set blindly. -- [ ] Verify each smoke's exact route, JSON behavior, registered reasoning provenance, +- [x] Verify each smoke's exact route, JSON behavior, registered reasoning provenance, repair/failure telemetry, cost coverage, per-call output distribution, and absence of truncation. Then record it with `run_publication_matrix.py record-smoke` so the machine-enforced manifest accepts it; the panel phase refuses to run until every registered model has an accepted entry. -- [ ] If the cap-pressure rule fires, amend the whole lane to 8,192 and repeat - affected smokes before any full-panel result. Do not make this decision after - seeing full scores. +- [x] Apply the cap-pressure rule before full results. It did not fire: the + maximum smoke output was 1,432 tokens, below the 3,072-token trigger, so the + lane remained at 4,096. - [x] Freeze the provisional model registry only after all ten phase-one routes pass. -- [ ] Re-estimate and record expected full-panel cost, runtime, serial - concurrency, and quota after the smokes. The existing cost artifact describes - the retired 12-cell plan and is not current spend guidance. -- [ ] Preserve raw artifacts and logs outside git; do not discard failed cells. +- [x] Re-estimate and record expected full-panel cost, runtime, serial + concurrency, and quota after the smokes. Final raw artifact cost was + $48.993235 across all ten completed cells; measured account spend was + approximately $49.06. +- [x] Preserve raw artifacts and final run metadata outside git; package them as + hash-linked release assets without discarding diagnostic cells. - [x] Confirm in automated tests that the retired analyzer still rejects missing, duplicate, mixed-provenance, wrong-route, wrong-lane, wrong-repeat, incomplete-telemetry, and invalid-contract cells. -- [ ] Update `config/sota_v2_lane.json` to `frozen-native-reasoning-cap` only +- [x] Update `config/sota_v2_lane.json` to `frozen-native-reasoning-cap` only after the ten phase-one smokes clear the cap-pressure audit. - [x] Record the rationale, decision date, pre-full-panel trigger, and known limitations in the decision log below. -- [ ] Regenerate the site and confirm it still refuses to publish a ranking if +- [x] Regenerate the site and confirm it still refuses to publish a ranking if any publication prerequisite is missing. **Exit condition:** the official API lane has a documented fixed-cap policy, @@ -437,15 +440,15 @@ to raw evidence without relying on trust in the author. ### Phase 6 — GitHub and portfolio presentation -- [ ] Replace “GM-Bench MVP” with a confident, accurate project name and one-line +- [x] Replace “GM-Bench MVP” with a confident, accurate project name and one-line description. - [ ] Add a concise GitHub repository description. - [ ] Add the deployed site as the repository homepage. - [ ] Add relevant topics such as `llm-evaluation`, `agents`, `benchmark`, `simulation`, `sports-analytics`, and `reproducible-research`. -- [ ] Put the primary result or honest “results pending” state near the top of the +- [x] Put the primary result or honest “results pending” state near the top of the README. -- [ ] Add a five-minute quickstart that works from a clean clone without provider +- [x] Add a five-minute release-verification path that works from a clean clone without provider credentials. - [ ] Add a separate provider-backed quickstart with explicit cost expectations. - [ ] Add an architecture or evaluation-flow diagram. @@ -568,6 +571,7 @@ decision and why. | 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Accepted-route artifact spend was $0.427613; total campaign spend was $0.728909 including the excluded Kimi diagnostic. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. | | 2026-07-18 | Settle successful serial-cell reservations against measured spend. | The runner retained every historical worst-case reservation, so a healthy panel could stop against cumulative hypothetical spend even after completed artifacts and the OpenRouter account established a much lower real cost. | Mark successful-cell reservations settled after post-cell spend measurement, keep failed/interrupted reservations active, and evaluate each next cell against measured spend plus only unresolved liabilities. | | 2026-07-18 | Amend GLM 5.2 from the unhealthy first-party Z.AI FP8 endpoint to Novita FP8. | The frozen `z-ai/fp8` endpoint remained at OpenRouter status `-2` across repeated launch preflights, while the exact dated Novita FP8 endpoint was healthy and advertised the common lane parameters. No full-panel GLM result existed. | Pin `novita/fp8`, replace rather than reuse the Z.AI smoke entry, refresh route pricing/runtime evidence, and require a clean exact-route smoke before restoring panel unlock. The replacement smoke completed 4/4 decisions with zero failures or truncations for $0.009225. | +| 2026-07-19 | Publish eight eligible phase-one rows without an ordinal winner claim. | Eight of ten registered cells passed exact-route and complete-cost gates. Grok recorded usage for 476/480 decisions and cost for 474/480; Mistral recorded cost for 479/480 after one fallback. All eight eligible seed-paired intervals form one connected tier, and every Holm-adjusted primary contrast is 0.078125. | Publish the eight eligible rows as one uncertainty tier, retain Grok and Mistral as diagnostics, do not rerun completed cells, and attach exact raw evidence plus checksums to the tagged release. | | 2026-07-18 | Reconcile the frozen publication protocol and reserve repair-call contingency before launch. | Independent Fable 5 review found that the runner and lane correctly enforced 4,096/3,072/8,192 native-minimum reasoning, but `publication_protocol.json` still described the retired 1,024/768/2,048 policy. It also noted that the prior reservation covered only primary calls even though one bounded repair is configured. No full-panel result existed. | Record the current lane as an explicit pre-data protocol amendment. Reserve one full-price call for every configured repair attempt and apply the committed 1.2x cost contingency before admitting each serial cell. Use a sub-$100 operator ceiling and monitor measured spend after every cell. | ## Experiment and release log diff --git a/docs/REPRODUCING_SOTA_V2_RELEASE.md b/docs/REPRODUCING_SOTA_V2_RELEASE.md new file mode 100644 index 0000000..e676597 --- /dev/null +++ b/docs/REPRODUCING_SOTA_V2_RELEASE.md @@ -0,0 +1,88 @@ +# Reproducing the `sota-v2` phase-one release + +This path verifies the published evidence without provider credentials or new +model spending. It checks the frozen contract, compact result rows, raw archive +hashes, compact-to-raw links, generated analysis, and website data. + +## 1. Check out the release + +```bash +git clone https://github.com/nedcut/gm-bench.git +cd gm-bench +git checkout sota-v2-phase-one-2026-07-19 +``` + +Use Python 3.11 or newer. Bun is required only for the website build. + +## 2. Download and verify the raw evidence + +```bash +mkdir -p /tmp/gm-bench-sota-v2-release +gh release download sota-v2-phase-one-2026-07-19 \ + --dir /tmp/gm-bench-sota-v2-release +cd /tmp/gm-bench-sota-v2-release +shasum -a 256 -c SHA256SUMS.txt +cd - +python3 scripts/package_publication_release.py \ + --verify /tmp/gm-bench-sota-v2-release/gm-bench-sota-v2-phase-one-raw-2026-07-19.zip +``` + +The verifier checks every archived byte hash and canonical raw JSON hash, then +proves that each committed compact artifact links to the corresponding raw +artifact where one exists. + +## 3. Re-run the publication analysis + +```bash +rm -rf /tmp/gm-bench-sota-v2-extracted +mkdir -p /tmp/gm-bench-sota-v2-extracted +unzip -q /tmp/gm-bench-sota-v2-release/gm-bench-sota-v2-phase-one-raw-2026-07-19.zip \ + -d /tmp/gm-bench-sota-v2-extracted +python3 scripts/analyze_publication_panel.py \ + --artifacts-dir /tmp/gm-bench-sota-v2-extracted/raw \ + --output /tmp/reproduced-publication-panel-analysis.json +cmp /tmp/reproduced-publication-panel-analysis.json \ + results/analysis/publication-panel-analysis.json +``` + +Expected outcome: eight eligible rows, with Grok 4.5 and Mistral Medium 3.5 +rejected for incomplete usage/cost telemetry. + +## 4. Validate the committed rows and generated site + +```bash +for artifact in results/leaderboard/*.json; do + python3 -m gm_bench validate-result "$artifact" --policy sota-v2 +done +python3 -m gm_bench validate-contract +python3 web/scripts/build_leaderboard.py +git diff --exit-code -- web/src/data/leaderboard.json +``` + +Expected site gate: + +- `eligible_headline_models`: 8 +- `minimum_headline_models`: 8 +- `publishable_ranking`: `true` +- `panel_analysis_ready`: `true` + +`publishable_ranking` means the table cleared its evidence gate. It does not +override the analysis: all eight model rows occupy one overlapping uncertainty +tier and every row trails `pick-trader`. + +## 5. Optional full repository verification + +```bash +python3 -m pytest -q +python3 -m ruff format --check gm_bench examples tests scripts +python3 -m ruff check gm_bench examples tests scripts +cd web +bun install --frozen-lockfile +bun run lint +bun run build +``` + +Please report the operating system, Python/Bun versions, release archive SHA, +and exact pass/fail output on the independent-reproduction issue. A successful +clean-clone run is external validation of packaging and reproducibility—not an +independent rerun of the paid model calls. diff --git a/docs/blog/sota-v2-findings.md b/docs/blog/sota-v2-findings.md index e9d9cb2..2e2b4cc 100644 --- a/docs/blog/sota-v2-findings.md +++ b/docs/blog/sota-v2-findings.md @@ -1,48 +1,120 @@ # Frontier models versus long-horizon asset management -> Draft. Publication is blocked until the fixed-cap API panel contains at least -> eight strictly eligible registered models. No v1 score below is a current -> ranking. - GM-Bench asks an agent to run the same procedurally generated hockey franchise -for five seasons: manage cap, negotiate trades, scout and draft prospects, and -trade current wins against future asset value. The benchmark is reproducible -in ways many model evaluations are not: its simulator and scoring contract are -fingerprinted, public and held-out seed panels are committed independently, -scripted policies provide absolute reference points, and raw run evidence is -hash-linked from compact published artifacts. - -The withdrawn v1 runs exposed two reasons to distrust an easy headline. First, -the documented prospect-scout action did not match the simulator, harming -models unevenly while leaving scripts untouched. Second, observed output ranged -from roughly 263 to 2,993 tokens per decision. The same nominal model scored -very differently through an API and a coding harness. That table mixed model -quality, output budget, and harness behavior, so it is evidence motivating the -new protocol—not a ranking. - -The `sota-v2` publication answers those failures directly. It fixes scouting, -separates API and coding-harness lanes, reports input and output tokens rather -than hiding output inside a total dominated by prompts, permits exactly one -measured JSON repair, and reports accepted and rejected actions by strategic -mechanic. The headline lane uses the same 1,024-token safety ceiling with -reasoning disabled for every model. Before any full result, all registered -models must pass a smoke audit; a call reaching 768 output tokens or showing -cap-induced truncation raises the whole lane to 2,048. The smoke audit selects -that single cap—1,024 by default, or 2,048 if the trigger fires—and the chosen -cap is then frozen for the entire published lane, so every headline number -shares one response budget rather than mixing 1,024- and 2,048-token results. -Actual token, cost, and latency efficiency are reported beside score rather than -folded into it. - -The scale will show four anchors: the hidden-information Oracle diagnostic, -the strongest honest scripted policy (`pick-trader`), the best eligible API -model, and `random`. This makes both model progress and remaining benchmark -headroom visible. The central question is not “can an LLM emit plausible GM -prose?” It is which model-plus-standardized-scaffold systems can beat a few -hundred lines of transparent heuristic at compounding, long-horizon asset -allocation under one fixed response budget. - -## Results - -Generated from `web/src/data/leaderboard.json` after the lane freeze. Do not -hand-copy provisional numbers here. +for five seasons: manage the cap, negotiate trades, scout and draft prospects, +and trade current wins against future asset value. In the frozen phase-one +study, eight publication-eligible frontier and open models all finished far +below a transparent scripted heuristic. + +That result is more interesting than a model leaderboard. It says that fluent +model output, even from expensive frontier systems, did not reliably compound +good decisions across this particular synthetic environment. It does **not** +say that one model is generally better than another, or that LLMs cannot manage +real sports teams. + +## Why the first ranking was withdrawn + +The earlier `sota-v1` runs exposed two reasons to distrust an easy headline. +First, the documented prospect-scout action did not match the simulator, +harming models unevenly while leaving scripts untouched. Second, observed +output ranged from roughly 263 to 2,993 tokens per decision, and the same +nominal model behaved very differently through an API and a coding harness. +That table mixed model quality, output budget, and harness behavior, so it is +archived as evidence motivating the new protocol—not as a ranking. + +The `sota-v2` protocol fixes scouting, separates API and coding-harness lanes, +reports input and output tokens, permits one measured JSON repair, and records +accepted and rejected actions by strategic mechanic. The phase-one API lane +was frozen before full scores were visible at a common 4,096-token total-output +safety ceiling with native-minimum reasoning: reasoning disabled where optional +and set to the lowest supported effort where mandatory. All ten exact routes +passed a real smoke audit without truncation or the predeclared 3,072-token +cap-pressure trigger. + +## The result + +Each registered model ran the same eight public seeds, five seasons, and three +candidate repeats: 24 episodes and 480 decision points. Repeats were averaged +within seed before comparison with that seed's deterministic `pick-trader` +score. The primary family contains all ten pre-registered models, including the +two technically ineligible rows. + +| Model | Mean score | Lift vs `pick-trader` | Tokens / decision | Cost | Illegal actions | +| --- | ---: | ---: | ---: | ---: | ---: | +| Muse Spark 1.1 | 231.851 | -179.768 | 10,084.2 | $7.5863 | 51 | +| GLM 5.2 | 217.539 | -194.080 | 10,764.4 | $1.2812 | 168 | +| Gemini 3.5 Flash | 215.624 | -195.995 | 8,770.7 | $6.3205 | 18 | +| Tencent HY3 | 195.841 | -215.778 | 7,743.3 | $0.0000 | 10 | +| Qwen 3.7 Plus | 175.520 | -236.099 | 8,039.6 | $1.2938 | 136 | +| GPT-5.6 Luna | 173.926 | -237.693 | 7,611.2 | $4.6802 | 126 | +| Claude Sonnet 5 | 142.143 | -269.475 | 9,944.4 | $11.0544 | 44 | +| MiniMax M3 | 129.880 | -281.739 | 8,728.2 | $1.2371 | 14 | + +`pick-trader` scored 411.619; the full scripted-baseline panel averaged +273.794. Even the highest observed model mean, Muse Spark's 231.851, remained +below both references. The eight eligible rows consumed $33.4535 in artifact- +reported API cost. Across all ten completed cells, including diagnostics, the +artifact total was $48.9932. + +The observed model means should not be read as ranks. Every eligible row lands +in one connected uncertainty tier because the seed-paired bootstrap intervals +overlap transitively. Each model's unadjusted exact sign-flip result is 0.0078, +reflecting a negative lift on every seed, but the predeclared Holm adjustment +uses the full ten-model family; every adjusted value is 0.078125. With only +eight public seeds, the study has limited resolution for model-versus-model +claims. “Muse had the highest observed mean” is supported. “Muse is the best +model” is not. + +## Two completed cells were excluded + +Grok 4.5 completed the simulator but recorded usage for 476/480 decisions and +cost for 474/480. Mistral Medium 3.5 recorded usage for 480/480 but cost for +479/480 after one adapter fallback. The frozen policy requires complete cost +and route telemetry for every headline decision, so neither row enters the +headline table and neither was rerun for a better result. Their raw evidence is +retained in the release archive; Mistral also has a compact diagnostic artifact. + +This distinction matters. Eligibility means that a result is complete and +comparable under the frozen lane. It does not mean that the model performed +well. Conversely, an excluded row can remain useful operational evidence +without being silently promoted into the comparison. + +## What appears to be hard + +The model rows accumulated between 10 and 168 illegal actions. Several also +issued many failed information queries: Muse recorded 453, GLM 209, and Claude +137. Those counts do not prove a single causal failure mode, but the traces show +that plausible local actions did not reliably become legal, state-aware plans +across drafts, contracts, lineups, and trades. The benchmark's strongest +scripted policies benefit from explicit knowledge of its mechanics, so beating +them is intentionally a demanding bar rather than a fair imitation of a human +front office. + +## Scope and limitations + +- GM-Bench is a synthetic hockey-style environment, not a real organization. +- Its score is hand-designed and encodes explicit value judgments. +- Scripted baselines were written with direct knowledge of the environment. +- Eight public seeds provide limited statistical resolution and may be exposed + to benchmark-specific adaptation. +- Native-minimum reasoning is operationally comparable, not identical compute: + actual model token use still ranged from 7,611 to 10,764 tokens per decision. +- The result evaluates model-plus-standardized-scaffold systems on this task; + it is not a claim about general intelligence or universal strategic ability. + +## Audit and reproduce + +The compact rows live in [`results/leaderboard/`](../../results/leaderboard/), +the seed-paired analysis in +[`results/analysis/publication-panel-analysis.json`](../../results/analysis/publication-panel-analysis.json), +and the release manifest in +[`releases/sota-v2-phase-one-2026-07-19/manifest.json`](../../releases/sota-v2-phase-one-2026-07-19/manifest.json). +The GitHub release attaches the exact raw public traces, frozen configs, final +run metadata, and checksums. See +[`docs/REPRODUCING_SOTA_V2_RELEASE.md`](../REPRODUCING_SOTA_V2_RELEASE.md) +for the clean-clone verification path. + +The durable conclusion is deliberately narrow: under GM-Bench's frozen +phase-one public protocol, none of the eight eligible model systems beat the +transparent `pick-trader` heuristic, and the sample does not support an ordinal +ranking among the models themselves. diff --git a/releases/sota-v2-phase-one-2026-07-19/README.md b/releases/sota-v2-phase-one-2026-07-19/README.md new file mode 100644 index 0000000..4f07f48 --- /dev/null +++ b/releases/sota-v2-phase-one-2026-07-19/README.md @@ -0,0 +1,32 @@ +# `sota-v2` phase-one release + +This release freezes the first public GM-Bench panel under the corrected +`sota-v2` contract and the common 4,096-token native-minimum-reasoning API lane. + +## Evidence state + +- 10 pre-registered model cells completed. +- 8 cells are strict, route-matched, cost-complete headline rows. +- Grok 4.5 is diagnostic because usage covers 476/480 decisions and cost covers + 474/480. +- Mistral Medium 3.5 is diagnostic because cost covers 479/480 decisions after + one adapter fallback. +- All eight eligible rows occupy one overlapping uncertainty tier. +- Every eligible model trails `pick-trader` (411.619). + +The release archive contains the exact ten raw public artifacts, frozen registry +and protocol files, generated panel analysis, and final run/reservation metadata. +[`manifest.json`](manifest.json) records byte and canonical JSON hashes plus +compact-to-raw links. [`SHA256SUMS.txt`](SHA256SUMS.txt) is committed and +attached beside the archive. + +## Interpretation + +Muse Spark 1.1 has the highest observed eligible mean (231.851), but the study +does not support an ordinal winner claim. The public panel contains eight seeds; +full-family Holm-adjusted p-values are 0.078125, and model intervals overlap. + +See [`docs/blog/sota-v2-findings.md`](../../docs/blog/sota-v2-findings.md) for +the narrative and +[`docs/REPRODUCING_SOTA_V2_RELEASE.md`](../../docs/REPRODUCING_SOTA_V2_RELEASE.md) +for the no-provider-cost verification path. diff --git a/releases/sota-v2-phase-one-2026-07-19/SHA256SUMS.txt b/releases/sota-v2-phase-one-2026-07-19/SHA256SUMS.txt new file mode 100644 index 0000000..dd17314 --- /dev/null +++ b/releases/sota-v2-phase-one-2026-07-19/SHA256SUMS.txt @@ -0,0 +1 @@ +cf0d033f0d4126a9389ecad57275ae050b8cdbafc882ccb661fdc9f85d8c9d4e gm-bench-sota-v2-phase-one-raw-2026-07-19.zip diff --git a/releases/sota-v2-phase-one-2026-07-19/manifest.json b/releases/sota-v2-phase-one-2026-07-19/manifest.json new file mode 100644 index 0000000..eec24a3 --- /dev/null +++ b/releases/sota-v2-phase-one-2026-07-19/manifest.json @@ -0,0 +1,318 @@ +{ + "archive_name": "gm-bench-sota-v2-phase-one-raw-2026-07-19.zip", + "artifacts": [ + { + "compact_artifact": "results/leaderboard/openrouter-gpt-5.6-luna-openai.json", + "compact_raw_artifact_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "cost_decisions": 480, + "cost_usd": 4.680175, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 173.926, + "model": "openai/gpt-5.6-luna", + "model_id": "openrouter-gpt-5.6-luna-openai", + "provider": "openrouter", + "raw_bytes": 4997542, + "raw_canonical_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "raw_path": "raw/openrouter-gpt-5.6-luna-openai--4096.json", + "raw_sha256": "88f6b7c4c3994b105214620d0512589f28ef1a39a1696fc2887a5bb2083aea03", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "OpenAI" + }, + { + "compact_artifact": "results/leaderboard/openrouter-claude-sonnet-5-bedrock.json", + "compact_raw_artifact_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "cost_decisions": 480, + "cost_usd": 11.05436, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 142.143, + "model": "anthropic/claude-sonnet-5", + "model_id": "openrouter-claude-sonnet-5-bedrock", + "provider": "openrouter", + "raw_bytes": 5192889, + "raw_canonical_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "raw_path": "raw/openrouter-claude-sonnet-5-bedrock--4096.json", + "raw_sha256": "0984245289740c083d4d454051b23fbbccccb0918cda1468b06dd8da25b8d39f", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Amazon Bedrock" + }, + { + "compact_artifact": "results/leaderboard/openrouter-gemini-3.5-flash-google-ai-studio.json", + "compact_raw_artifact_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "cost_decisions": 480, + "cost_usd": 6.320513, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 215.624, + "model": "google/gemini-3.5-flash", + "model_id": "openrouter-gemini-3.5-flash-google-ai-studio", + "provider": "openrouter", + "raw_bytes": 4650686, + "raw_canonical_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "raw_path": "raw/openrouter-gemini-3.5-flash-google-ai-studio--4096.json", + "raw_sha256": "e16be82e32007620fd7d144faa4f2312ab06eae1a5f0e2f03f4a66466284743e", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Google AI Studio" + }, + { + "compact_artifact": null, + "compact_raw_artifact_sha256": null, + "cost_decisions": 474, + "cost_usd": 8.561192, + "decisions": 480, + "decisions_with_usage": 476, + "mean_score": 176.081, + "model": "x-ai/grok-4.5", + "model_id": "openrouter-grok-4.5-xai", + "provider": "openrouter", + "raw_bytes": 5160010, + "raw_canonical_sha256": "0f2960ca86368c9f6f87d4d45adf4a42768544b08036127a5007e01c58a7dd35", + "raw_path": "raw/openrouter-grok-4.5-xai--4096.json", + "raw_sha256": "b1eb5857b23d11ce173847148580f84c8c186d94178fd41feed1dfa570df040d", + "rejection_reasons": [ + "candidate usage must cover every decision point", + "candidate cost telemetry must cover every decision point" + ], + "status": "diagnostic", + "upstream_provider": "xAI" + }, + { + "compact_artifact": "results/leaderboard/openrouter-muse-spark-1.1-meta.json", + "compact_raw_artifact_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "cost_decisions": 480, + "cost_usd": 7.586279, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 231.851, + "model": "meta/muse-spark-1.1", + "model_id": "openrouter-muse-spark-1.1-meta", + "provider": "openrouter", + "raw_bytes": 5281954, + "raw_canonical_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "raw_path": "raw/openrouter-muse-spark-1.1-meta--4096.json", + "raw_sha256": "5c4a10a44f5252d9cfd384d96ef2d058bcbdca01095c4b3e0245edb66dc0ba4c", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Meta" + }, + { + "compact_artifact": "results/leaderboard/openrouter-glm-5.2-novita.json", + "compact_raw_artifact_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "cost_decisions": 480, + "cost_usd": 1.281193, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 217.539, + "model": "z-ai/glm-5.2", + "model_id": "openrouter-glm-5.2-novita", + "provider": "openrouter", + "raw_bytes": 5714758, + "raw_canonical_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "raw_path": "raw/openrouter-glm-5.2-novita--4096.json", + "raw_sha256": "9c806be35410a4c44dc06c72c0d12d100738f57b62daa366fb584f8a54259f9f", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Novita" + }, + { + "compact_artifact": "results/leaderboard/openrouter-minimax-m3-minimax.json", + "compact_raw_artifact_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "cost_decisions": 480, + "cost_usd": 1.237149, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 129.88, + "model": "minimax/minimax-m3", + "model_id": "openrouter-minimax-m3-minimax", + "provider": "openrouter", + "raw_bytes": 4858524, + "raw_canonical_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "raw_path": "raw/openrouter-minimax-m3-minimax--4096.json", + "raw_sha256": "db3bb00f2d98bc22368438bd182975a3df69bc6d772f10877c3dc613e8fdba04", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Minimax" + }, + { + "compact_artifact": "results/leaderboard/openrouter-qwen3.7-plus-alibaba.json", + "compact_raw_artifact_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "cost_decisions": 480, + "cost_usd": 1.293793, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 175.52, + "model": "qwen/qwen3.7-plus", + "model_id": "openrouter-qwen3.7-plus-alibaba", + "provider": "openrouter", + "raw_bytes": 4907682, + "raw_canonical_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "raw_path": "raw/openrouter-qwen3.7-plus-alibaba--4096.json", + "raw_sha256": "13b31bf544eb93549c3f237d5db1c04e0ae72d0ade3bfc9e14d190d4ac10540e", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Alibaba" + }, + { + "compact_artifact": "results/diagnostics/openrouter-mistral-medium-3.5-mistral.json", + "compact_raw_artifact_sha256": "f04e5c6fc5d01d2368d1c52eb3c8cdfe43674ad476fbfc28b598d8f29690b6cc", + "cost_decisions": 479, + "cost_usd": 6.978581, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 138.362, + "model": "mistralai/mistral-medium-3-5", + "model_id": "openrouter-mistral-medium-3.5-mistral", + "provider": "openrouter", + "raw_bytes": 5060576, + "raw_canonical_sha256": "f04e5c6fc5d01d2368d1c52eb3c8cdfe43674ad476fbfc28b598d8f29690b6cc", + "raw_path": "raw/openrouter-mistral-medium-3.5-mistral--4096.json", + "raw_sha256": "43067518dd55d4914142b5ebcdb71b25385052000a44fb6951d597caf5dc31dd", + "rejection_reasons": [ + "candidate cost telemetry must cover every decision point" + ], + "status": "diagnostic", + "upstream_provider": "Mistral" + }, + { + "compact_artifact": "results/leaderboard/openrouter-tencent-hy3-free-novita.json", + "compact_raw_artifact_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "cost_decisions": 480, + "cost_usd": 0.0, + "decisions": 480, + "decisions_with_usage": 480, + "mean_score": 195.841, + "model": "tencent/hy3:free", + "model_id": "openrouter-tencent-hy3-free-novita", + "provider": "openrouter", + "raw_bytes": 4909835, + "raw_canonical_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "raw_path": "raw/openrouter-tencent-hy3-free-novita--4096.json", + "raw_sha256": "7c93f1d3bbe3e90e6a4aef03766c23e460ce15c43f5a92452c0b46d415fa2a52", + "rejection_reasons": [], + "status": "headline", + "upstream_provider": "Novita" + } + ], + "contract": "sota-v2", + "diagnostic_models": 2, + "eligible_headline_models": 8, + "files": [ + { + "bytes": 9869, + "path": "config/publication_protocol.json", + "role": "frozen-config", + "sha256": "fa549234d134f8c29536b23d463accbb568af13e81f1b4bebb26060c70df50ef" + }, + { + "bytes": 1521, + "path": "config/sota_v2_lane.json", + "role": "frozen-config", + "sha256": "e1f6afb733a0afa0ac287b2cabcaf39ddd0f9908780b6e0cb6071cdcde00cc06" + }, + { + "bytes": 10062, + "path": "config/sota_v2_models.json", + "role": "frozen-config", + "sha256": "3a401275e5658a7596dc897418083928c9a0200a1e78f46e26ac0428b0c24f91" + }, + { + "bytes": 11442, + "path": "config/sota_v2_smoke_manifest.json", + "role": "frozen-config", + "sha256": "ad55b3d07c2e772479a6647a1faf63a19412b70ac8ba9f4569a7d571d95e033f" + }, + { + "bytes": 5192889, + "path": "raw/openrouter-claude-sonnet-5-bedrock--4096.json", + "role": "raw-public-trace", + "sha256": "0984245289740c083d4d454051b23fbbccccb0918cda1468b06dd8da25b8d39f" + }, + { + "bytes": 4650686, + "path": "raw/openrouter-gemini-3.5-flash-google-ai-studio--4096.json", + "role": "raw-public-trace", + "sha256": "e16be82e32007620fd7d144faa4f2312ab06eae1a5f0e2f03f4a66466284743e" + }, + { + "bytes": 5714758, + "path": "raw/openrouter-glm-5.2-novita--4096.json", + "role": "raw-public-trace", + "sha256": "9c806be35410a4c44dc06c72c0d12d100738f57b62daa366fb584f8a54259f9f" + }, + { + "bytes": 4997542, + "path": "raw/openrouter-gpt-5.6-luna-openai--4096.json", + "role": "raw-public-trace", + "sha256": "88f6b7c4c3994b105214620d0512589f28ef1a39a1696fc2887a5bb2083aea03" + }, + { + "bytes": 5160010, + "path": "raw/openrouter-grok-4.5-xai--4096.json", + "role": "raw-public-trace", + "sha256": "b1eb5857b23d11ce173847148580f84c8c186d94178fd41feed1dfa570df040d" + }, + { + "bytes": 4858524, + "path": "raw/openrouter-minimax-m3-minimax--4096.json", + "role": "raw-public-trace", + "sha256": "db3bb00f2d98bc22368438bd182975a3df69bc6d772f10877c3dc613e8fdba04" + }, + { + "bytes": 5060576, + "path": "raw/openrouter-mistral-medium-3.5-mistral--4096.json", + "role": "raw-public-trace", + "sha256": "43067518dd55d4914142b5ebcdb71b25385052000a44fb6951d597caf5dc31dd" + }, + { + "bytes": 5281954, + "path": "raw/openrouter-muse-spark-1.1-meta--4096.json", + "role": "raw-public-trace", + "sha256": "5c4a10a44f5252d9cfd384d96ef2d058bcbdca01095c4b3e0245edb66dc0ba4c" + }, + { + "bytes": 4907682, + "path": "raw/openrouter-qwen3.7-plus-alibaba--4096.json", + "role": "raw-public-trace", + "sha256": "13b31bf544eb93549c3f237d5db1c04e0ae72d0ade3bfc9e14d190d4ac10540e" + }, + { + "bytes": 4909835, + "path": "raw/openrouter-tencent-hy3-free-novita--4096.json", + "role": "raw-public-trace", + "sha256": "7c93f1d3bbe3e90e6a4aef03766c23e460ce15c43f5a92452c0b46d415fa2a52" + }, + { + "bytes": 16418, + "path": "results/analysis/publication-panel-analysis.json", + "role": "panel-analysis", + "sha256": "c8d3ff83df153ea2f01553312db0ae0be80858403b7dee6a32a628ba6a6ca4bd" + }, + { + "bytes": 6872, + "path": "run-metadata/openrouter-reservations.json", + "role": "run-metadata", + "sha256": "35c145eea21e5253849d4bde3bcc080de6a6ab6b33a2c3096545edfe86cc0c94" + }, + { + "bytes": 1388, + "path": "run-metadata/run-state.json", + "role": "run-metadata", + "sha256": "ad9eb0ddd26005287e7a9f5a13f3ff0eb44122f83bf0ef82beecd9fcc7a203f7" + } + ], + "format": "gm-bench-publication-release-v1", + "notes": [ + "Raw artifacts are immutable evidence and hash-link to committed compact rows where available.", + "Raw operator diagnostics may contain machine-local cache paths; compact publication artifacts do not.", + "No ordinal model ranking is supported: all eligible rows occupy one overlapping uncertainty tier." + ], + "output_token_cap": 4096, + "registered_models": 10, + "release_date": "2026-07-19", + "release_id": "sota-v2-phase-one-2026-07-19", + "schema_version": 1 +} diff --git a/scripts/package_publication_release.py b/scripts/package_publication_release.py new file mode 100644 index 0000000..ec73880 --- /dev/null +++ b/scripts/package_publication_release.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Build a deterministic release archive for the frozen public panel.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import zipfile +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from gm_bench.publication import canonical_sha256 # noqa: E402 + +RELEASE_FORMAT = "gm-bench-publication-release-v1" +RELEASE_ID = "sota-v2-phase-one-2026-07-19" +FIXED_ZIP_TIMESTAMP = (2026, 7, 19, 0, 0, 0) +CONFIG_PATHS = ( + Path("config/sota_v2_models.json"), + Path("config/sota_v2_lane.json"), + Path("config/publication_protocol.json"), + Path("config/sota_v2_smoke_manifest.json"), +) +ANALYSIS_PATH = Path("results/analysis/publication-panel-analysis.json") +RUN_METADATA_NAMES = ("run-state.json", "openrouter-reservations.json") + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text()) + if not isinstance(payload, dict): + raise ValueError(f"{path} must contain a JSON object") + return payload + + +def _json_bytes(payload: dict[str, Any]) -> bytes: + return (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _file_record(path: str, data: bytes, role: str) -> dict[str, Any]: + return {"path": path, "role": role, "bytes": len(data), "sha256": _sha256(data)} + + +def _zip_bytes(entries: dict[str, bytes], output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for name in sorted(entries): + info = zipfile.ZipInfo(name, FIXED_ZIP_TIMESTAMP) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = 0o100644 << 16 + archive.writestr(info, entries[name]) + + +def build_release( + *, + repo_root: Path, + run_dir: Path, + archive_path: Path, + manifest_path: Path, + checksums_path: Path, +) -> dict[str, Any]: + registry = _read_json(repo_root / "config/sota_v2_models.json") + analysis = _read_json(repo_root / ANALYSIS_PATH) + cap = int(registry["output_token_cap"]) + eligible = {str(row["model_id"]): row for row in analysis.get("models") or []} + rejected = { + str(row["model_id"]): [str(reason) for reason in row.get("reasons") or []] + for row in analysis.get("rejected_artifacts") or [] + } + registered = [row for row in registry.get("models") or [] if isinstance(row, dict)] + registered_ids = {str(row.get("id")) for row in registered} + if set(eligible) | set(rejected) != registered_ids or set(eligible) & set(rejected): + raise ValueError("analysis must classify every registered model exactly once") + + entries: dict[str, bytes] = {} + files: list[dict[str, Any]] = [] + artifacts: list[dict[str, Any]] = [] + + def add_file(source: Path, archive_name: str, role: str) -> bytes: + data = source.read_bytes() + entries[archive_name] = data + files.append(_file_record(archive_name, data, role)) + return data + + for relative in CONFIG_PATHS: + add_file(repo_root / relative, relative.as_posix(), "frozen-config") + add_file(repo_root / ANALYSIS_PATH, ANALYSIS_PATH.as_posix(), "panel-analysis") + for name in RUN_METADATA_NAMES: + source = run_dir / name + if source.exists(): + add_file(source, f"run-metadata/{name}", "run-metadata") + + for spec in registered: + model_id = str(spec["id"]) + raw_path = run_dir / "raw" / f"{model_id}--{cap}.json" + archive_name = f"raw/{raw_path.name}" + raw_bytes = add_file(raw_path, archive_name, "raw-public-trace") + raw = json.loads(raw_bytes) + raw_canonical = canonical_sha256(raw) + status = "headline" if model_id in eligible else "diagnostic" + compact_relative = ( + Path("results/leaderboard") / f"{model_id}.json" + if status == "headline" + else Path("results/diagnostics") / f"{model_id}.json" + ) + compact_raw_hash = None + compact_path = None + if (repo_root / compact_relative).exists(): + compact = _read_json(repo_root / compact_relative) + compact_raw_hash = str((compact.get("publication") or {}).get("raw_artifact_sha256") or "") + if compact_raw_hash != raw_canonical: + raise ValueError(f"compact artifact does not hash-link to raw evidence for {model_id}") + compact_path = compact_relative.as_posix() + + candidate = raw.get("candidate") or {} + summary = candidate.get("summary") or {} + usage = summary.get("usage") or {} + artifacts.append( + { + "model_id": model_id, + "provider": spec.get("provider"), + "model": spec.get("model"), + "upstream_provider": spec.get("upstream_provider"), + "status": status, + "rejection_reasons": rejected.get(model_id, []), + "raw_path": archive_name, + "raw_bytes": len(raw_bytes), + "raw_sha256": _sha256(raw_bytes), + "raw_canonical_sha256": raw_canonical, + "compact_artifact": compact_path, + "compact_raw_artifact_sha256": compact_raw_hash, + "decisions": summary.get("decisions"), + "decisions_with_usage": usage.get("decisions_with_usage"), + "cost_decisions": usage.get("cost_decisions"), + "cost_usd": usage.get("cost_usd"), + "mean_score": summary.get("mean_score"), + } + ) + + manifest = { + "format": RELEASE_FORMAT, + "schema_version": 1, + "release_id": RELEASE_ID, + "release_date": "2026-07-19", + "contract": registry.get("contract"), + "output_token_cap": cap, + "registered_models": len(registered), + "eligible_headline_models": len(eligible), + "diagnostic_models": len(rejected), + "archive_name": archive_path.name, + "files": sorted(files, key=lambda row: str(row["path"])), + "artifacts": artifacts, + "notes": [ + "Raw artifacts are immutable evidence and hash-link to committed compact rows where available.", + "Raw operator diagnostics may contain machine-local cache paths; compact publication artifacts do not.", + "No ordinal model ranking is supported: all eligible rows occupy one overlapping uncertainty tier.", + ], + } + manifest_bytes = _json_bytes(manifest) + entries["manifest.json"] = manifest_bytes + _zip_bytes(entries, archive_path) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_bytes(manifest_bytes) + archive_sha = _sha256(archive_path.read_bytes()) + checksums_path.parent.mkdir(parents=True, exist_ok=True) + checksums_path.write_text(f"{archive_sha} {archive_path.name}\n") + return manifest + + +def verify_archive(archive_path: Path, *, repo_root: Path | None = None) -> dict[str, Any]: + with zipfile.ZipFile(archive_path) as archive: + names = archive.namelist() + if len(names) != len(set(names)) or "manifest.json" not in names: + raise ValueError("archive must contain one manifest and no duplicate paths") + if any(Path(name).is_absolute() or ".." in Path(name).parts for name in names): + raise ValueError("archive contains an unsafe path") + manifest = json.loads(archive.read("manifest.json")) + if manifest.get("format") != RELEASE_FORMAT: + raise ValueError("unsupported release manifest format") + expected_names = {str(row["path"]) for row in manifest.get("files") or []} | {"manifest.json"} + if set(names) != expected_names: + raise ValueError("archive members do not exactly match the manifest") + for row in manifest.get("files") or []: + data = archive.read(str(row["path"])) + if len(data) != row.get("bytes") or _sha256(data) != row.get("sha256"): + raise ValueError(f"file checksum mismatch: {row.get('path')}") + for artifact in manifest.get("artifacts") or []: + raw = json.loads(archive.read(str(artifact["raw_path"]))) + if canonical_sha256(raw) != artifact.get("raw_canonical_sha256"): + raise ValueError(f"canonical raw hash mismatch: {artifact.get('model_id')}") + compact_path = artifact.get("compact_artifact") + if compact_path and repo_root is not None: + compact = _read_json(repo_root / str(compact_path)) + linked = (compact.get("publication") or {}).get("raw_artifact_sha256") + if linked != artifact.get("raw_canonical_sha256"): + raise ValueError(f"committed compact link mismatch: {artifact.get('model_id')}") + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path) + parser.add_argument("--archive", type=Path) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--checksums", type=Path) + parser.add_argument("--verify", type=Path, help="verify an existing release archive instead of building") + args = parser.parse_args() + if args.verify: + manifest = verify_archive(args.verify.resolve(), repo_root=ROOT) + print( + f"ok: {args.verify} contains {manifest['eligible_headline_models']} headline and " + f"{manifest['diagnostic_models']} diagnostic model artifact(s)" + ) + return 0 + if not all((args.run_dir, args.archive, args.manifest, args.checksums)): + parser.error("build mode requires --run-dir, --archive, --manifest, and --checksums") + manifest = build_release( + repo_root=ROOT, + run_dir=args.run_dir.resolve(), + archive_path=args.archive.resolve(), + manifest_path=args.manifest.resolve(), + checksums_path=args.checksums.resolve(), + ) + print( + f"wrote {args.archive} with {manifest['eligible_headline_models']} headline and " + f"{manifest['diagnostic_models']} diagnostic model artifact(s)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_publication_claims.py b/tests/test_publication_claims.py new file mode 100644 index 0000000..a279d63 --- /dev/null +++ b/tests/test_publication_claims.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +DISPLAY_NAMES = { + "meta/muse-spark-1.1": "Muse Spark 1.1", + "z-ai/glm-5.2": "GLM 5.2", + "google/gemini-3.5-flash": "Gemini 3.5 Flash", + "tencent/hy3:free": "Tencent HY3", + "qwen/qwen3.7-plus": "Qwen 3.7 Plus", + "openai/gpt-5.6-luna": "GPT-5.6 Luna", + "anthropic/claude-sonnet-5": "Claude Sonnet 5", + "minimax/minimax-m3": "MiniMax M3", +} + + +def test_blog_result_table_matches_generated_leaderboard() -> None: + site = json.loads(Path("web/src/data/leaderboard.json").read_text()) + blog = Path("docs/blog/sota-v2-findings.md").read_text() + rows = {} + pattern = re.compile( + r"^\| (?P[^|]+?) \| (?P-?[0-9.]+) \| (?P-?[0-9.]+) " + r"\| (?P[0-9,.]+) \| \$(?P[0-9.]+) \| (?P[0-9]+) \|$" + ) + for line in blog.splitlines(): + match = pattern.match(line) + if match: + rows[match.group("name")] = match.groupdict() + + assert set(rows) == set(DISPLAY_NAMES.values()) + for model in site["models"]: + row = rows[DISPLAY_NAMES[model["model"]]] + assert row["mean"] == f"{model['mean_score']:.3f}" + assert row["lift"] == f"{model['lift_vs_best_baseline']:.3f}" + assert row["tokens"] == f"{model['tokens_per_decision']:,.1f}" + assert row["cost"] == f"{model['cost_usd']:.4f}" + assert row["illegal"] == str(model["illegal_actions"]) + + +def test_public_claim_surfaces_use_the_frozen_lane_and_current_gate() -> None: + blog = Path("docs/blog/sota-v2-findings.md").read_text() + readme = Path("README.md").read_text() + assert "4,096-token" in blog + assert "1,024-token safety ceiling" not in blog + assert "`publishable_ranking: true`" in readme + assert "no ordinal \u201cbest model\u201d claim" in " ".join(readme.split()) diff --git a/tests/test_publication_release.py b/tests/test_publication_release.py new file mode 100644 index 0000000..3c10163 --- /dev/null +++ b/tests/test_publication_release.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from gm_bench.publication import canonical_sha256 +from scripts.package_publication_release import build_release, verify_archive + + +def _write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) + + +def _release_fixture(tmp_path: Path) -> tuple[Path, Path]: + repo = tmp_path / "repo" + run_dir = tmp_path / "run" + registry = { + "contract": "sota-v2", + "output_token_cap": 4096, + "models": [ + { + "id": "demo", + "provider": "openrouter", + "model": "demo/model", + "upstream_provider": "Demo", + } + ], + } + raw = { + "candidate": { + "summary": { + "decisions": 4, + "mean_score": 12.5, + "usage": { + "decisions_with_usage": 4, + "cost_decisions": 4, + "cost_usd": 0.01, + }, + } + } + } + analysis = { + "models": [{"model_id": "demo"}], + "rejected_artifacts": [], + } + _write_json(repo / "config/sota_v2_models.json", registry) + for name in ("sota_v2_lane.json", "publication_protocol.json", "sota_v2_smoke_manifest.json"): + _write_json(repo / "config" / name, {"fixture": True}) + _write_json(repo / "results/analysis/publication-panel-analysis.json", analysis) + _write_json( + repo / "results/leaderboard/demo.json", + {"publication": {"raw_artifact_sha256": canonical_sha256(raw)}}, + ) + _write_json(run_dir / "raw/demo--4096.json", raw) + _write_json(run_dir / "run-state.json", {"phase": "panel"}) + return repo, run_dir + + +def test_release_archive_is_deterministic_and_verifiable(tmp_path: Path) -> None: + repo, run_dir = _release_fixture(tmp_path) + archive = tmp_path / "release.zip" + generated: list[bytes] = [] + for _ in range(2): + build_release( + repo_root=repo, + run_dir=run_dir, + archive_path=archive, + manifest_path=tmp_path / "manifest.json", + checksums_path=tmp_path / "SHA256SUMS.txt", + ) + generated.append(archive.read_bytes()) + + assert generated[0] == generated[1] + manifest = verify_archive(archive, repo_root=repo) + assert manifest["eligible_headline_models"] == 1 + assert manifest["diagnostic_models"] == 0 + assert manifest["artifacts"][0]["compact_artifact"] == "results/leaderboard/demo.json" + + +def test_release_build_rejects_compact_raw_hash_mismatch(tmp_path: Path) -> None: + repo, run_dir = _release_fixture(tmp_path) + _write_json(repo / "results/leaderboard/demo.json", {"publication": {"raw_artifact_sha256": "0" * 64}}) + + try: + build_release( + repo_root=repo, + run_dir=run_dir, + archive_path=tmp_path / "release.zip", + manifest_path=tmp_path / "manifest.json", + checksums_path=tmp_path / "SHA256SUMS.txt", + ) + except ValueError as exc: + assert "does not hash-link" in str(exc) + else: + raise AssertionError("expected compact/raw mismatch to fail")