From 54bb36c1320663c5470f2aae5853fd0277cdba36 Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Sun, 19 Jul 2026 17:32:22 -0400 Subject: [PATCH] Publish the phase-one frontier panel --- docs/PUBLISH_READINESS.md | 92 +- gm_bench/publication.py | 5 + .../analysis/publication-panel-analysis.json | 574 +++ results/benchmark_results.md | 25 +- ...openrouter-mistral-medium-3.5-mistral.json | 4149 ++++++++++++++++ .../openrouter-claude-sonnet-5-bedrock.json | 4149 ++++++++++++++++ ...ter-gemini-3.5-flash-google-ai-studio.json | 4150 +++++++++++++++++ .../openrouter-glm-5.2-novita.json | 4149 ++++++++++++++++ .../openrouter-gpt-5.6-luna-openai.json | 4149 ++++++++++++++++ .../openrouter-minimax-m3-minimax.json | 4149 ++++++++++++++++ .../openrouter-muse-spark-1.1-meta.json | 4150 +++++++++++++++++ .../openrouter-qwen3.7-plus-alibaba.json | 4149 ++++++++++++++++ .../openrouter-tencent-hy3-free-novita.json | 4149 ++++++++++++++++ tests/test_publication.py | 12 +- web/src/data/leaderboard.json | 705 ++- 15 files changed, 38700 insertions(+), 56 deletions(-) create mode 100644 results/analysis/publication-panel-analysis.json create mode 100644 results/diagnostics/openrouter-mistral-medium-3.5-mistral.json create mode 100644 results/leaderboard/openrouter-claude-sonnet-5-bedrock.json create mode 100644 results/leaderboard/openrouter-gemini-3.5-flash-google-ai-studio.json create mode 100644 results/leaderboard/openrouter-glm-5.2-novita.json create mode 100644 results/leaderboard/openrouter-gpt-5.6-luna-openai.json create mode 100644 results/leaderboard/openrouter-minimax-m3-minimax.json create mode 100644 results/leaderboard/openrouter-muse-spark-1.1-meta.json create mode 100644 results/leaderboard/openrouter-qwen3.7-plus-alibaba.json create mode 100644 results/leaderboard/openrouter-tencent-hy3-free-novita.json diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 14db71a..2bce012 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -7,17 +7,18 @@ > to preserve this first draft; the goal is to make it more accurate as the > project develops. -**Last reviewed:** 2026-07-18 +**Last reviewed:** 2026-07-19 **Current target:** Publish a validated `sota-v2` leaderboard, accompanying blog post, GitHub release, and public site. -**Current state:** The phase-one ten-model registry is frozen as amended on 2026-07-18, -uses exact OpenRouter endpoint tags, and completed its fresh 4,096-token -native-minimum-reasoning smoke gate; all ten registered routes are accepted with -zero failures, zero truncations, and no cap-pressure trigger. GLM 5.2 moved from -the unhealthy first-party Z.AI endpoint to an exact Novita FP8 route and passed -a fresh replacement smoke. Earlier 1,024-token -smokes are retained only as superseded operational evidence. The full panel run -is the remaining blocker before any eligible model row exists. +**Current state:** The frozen phase-one public panel is complete. Eight of ten +registered models produced route-matched, cost-complete `sota-v2` rows at the +shared 4,096-token native-minimum-reasoning cap, clearing the predeclared +minimum. Grok 4.5 and Mistral Medium 3.5 completed but remain diagnostic because +their artifacts lacked complete usage or cost coverage. The generated site now +exposes the eight eligible rows. All models overlap in one uncertainty tier, +and every eligible model trails `pick-trader`; the remaining release work is +claim review, raw public-trace packaging, presentation polish, and independent +reproduction rather than more phase-one model runs. **Current weekly focus:** [#63 — Publication sprint: freeze and ship GM-Bench `sota-v2`](https://github.com/nedcut/gm-bench/issues/63) **Broader roadmap:** [#60 — Roadmap to a publishable leaderboard + blog @@ -28,11 +29,12 @@ post](https://github.com/nedcut/gm-bench/issues/60) GM-Bench is viable as a focused benchmark for comparing models and agent scaffolds on synthetic, long-horizon resource allocation in a frozen sports management environment. It is already strong enough to be a flagship AI/ML -portfolio project. It is not yet ready for a headline model-ranking claim -because the current `sota-v2` leaderboard has no eligible model rows. The -compute policy is frozen at a 4,096-token native-minimum-reasoning ceiling, and -every registered route has passed the common smoke gate; the remaining blocker -is running the paid full panel itself. +portfolio project. The public `sota-v2` evidence gate is now satisfied with +eight eligible rows under the frozen 4,096-token native-minimum-reasoning lane. +The evidence does not support an ordinal model ranking: all eight rows occupy +one overlapping uncertainty tier, and none beats the transparent `pick-trader` +baseline. A public release still needs raw trace assets, claim review, +presentation polish, and independent reproduction. The strongest story is not merely that GM-Bench runs LLMs through a simulator. It is that the project: @@ -75,11 +77,11 @@ The project is publish-ready only when all four gates pass. | Reproducibility | Strong | Contract fingerprints, seed provenance, compact artifacts, and validators are in place. | | Benchmark validity | Strong but scoped | Scripted references, exploit canaries, oracle headroom, calibration, and mechanic coverage exist. | | Compute comparability | Frozen for phase one | The API lane has a common 4,096-token total-output ceiling, native-minimum reasoning, exact provider slugs and endpoint tags, a pre-full-panel 75% cap-pressure rule, and actual reasoning/token-efficiency reporting. All ten phase-one routes passed and were accepted; Kimi K3 and the unavailable Nemotron and DeepSeek routes are retained as exclusion evidence. | -| Current model evidence | Blocked | The active `sota-v2` leaderboard has no eligible model rows. | -| Statistical evidence | Partially ready | Paired analysis and power tooling exist; final model results do not. | +| Current model evidence | Public panel complete | Eight registered, route-matched, cost-complete `sota-v2` rows clear the publication floor; Grok and Mistral are retained as diagnostics. | +| Statistical evidence | Ready but low-resolution | Seed-paired intervals, exact sign-flip tests, full-family Holm adjustment, and overlap tiers are generated. All eight rows share one tier and trail `pick-trader`. | | External validation | Missing | No independent reproduction or third-party result has been recorded. | | GitHub presentation | Needs polish | Repository metadata, release packaging, README framing, and final site still need work. | -| Blog | Scaffolded | The honest narrative exists, but results must be generated from validated data. | +| Blog | Scaffolded | The validated results now exist; the narrative still needs its final evidence-backed update. | ## Critical path @@ -284,44 +286,46 @@ must be reported beside score. - [x] Keep the headline lane API-only, fresh-spawn, `compact`, and under the frozen output policy. - [x] Keep coding-agent CLI harnesses in a separate diagnostic table. -- [ ] Never parallelize Claude or another subscription/rate-limited CLI. +- [x] Never parallelize Claude or another subscription/rate-limited CLI; the + phase-one API cells ran serially with one worker. - [x] Verify all 10 provisional phase-one provider/model routes can accept the common privacy, parameter, JSON, registered reasoning, and bounded-output policy. -- [ ] Run a benchmark-level smoke for every provider/model combination at the +- [x] Run a benchmark-level smoke for every provider/model combination at the shared frozen cap immediately before the full panel. -- [ ] Use serial execution, fail-fast behavior, atomic checkpoints, and validated +- [x] Use serial execution, fail-fast behavior, atomic checkpoints, and validated resume rather than restarting completed episodes. -- [ ] Run all eight official seeds, five seasons, and three candidate repeats. -- [ ] Use the full official baseline panel. -- [ ] Require complete input/output token, latency, failure, repair, route, and - cost telemetry for every API decision. -- [ ] Reject or quarantine any row that does not pass strict `sota-v2` validation. +- [x] Run all eight official seeds, five seasons, and three candidate repeats. +- [x] Use the full official baseline panel. +- [x] Require complete input/output token, latency, failure, repair, route, and + cost telemetry for every headline API decision; quarantine rows that fail. +- [x] Reject or quarantine any row that does not pass strict `sota-v2` validation. - [ ] Put interesting but ineligible rows in `results/diagnostics/`, never in the - headline table. -- [ ] Compact only after strict validation and preserve the raw-artifact hash. -- [ ] Keep committed result artifacts under the CI size limit. + headline table. Mistral's compact diagnostic is committed; Grok's + non-compactable raw diagnostic still needs release-asset packaging. +- [x] Compact only after strict validation and preserve the raw-artifact hash. +- [x] Keep committed result artifacts under the CI size limit. - [ ] Publish raw public-panel traces as release assets so results are auditable. -- [ ] Preserve provider errors and incomplete attempts as diagnostic evidence. -- [ ] Regenerate the leaderboard from source artifacts; do not hand-copy scores. +- [x] Preserve provider errors and incomplete attempts as diagnostic evidence. +- [x] Regenerate the leaderboard from source artifacts; do not hand-copy scores. - [x] Require at least eight eligible, registered, route-matched, cost-complete headline rows before the generated JSON can expose a ranking. For every headline model, report at least: -- [ ] mean score and score standard deviation; -- [ ] an uncertainty interval for the mean or paired lift, with its method; -- [ ] lift versus the full baseline-panel mean; -- [ ] lift versus `pick-trader`; -- [ ] per-seed win rate and paired sign-flip p-value; -- [ ] input tokens per decision and output tokens per decision; -- [ ] total cost and cost per episode; -- [ ] illegal actions, failed queries, adapter failures, and repair attempts; -- [ ] execution lane, provider route, model snapshot, and scaffold fingerprint; -- [ ] result contract and seed-panel identity; and -- [ ] per-mechanic outcomes for drafting, trades, free agency, cap management, +- [x] mean score and score standard deviation; +- [x] an uncertainty interval for the mean or paired lift, with its method; +- [x] lift versus the full baseline-panel mean; +- [x] lift versus `pick-trader`; +- [x] per-seed win rate and paired sign-flip p-value; +- [x] input tokens per decision and output tokens per decision; +- [x] total cost and cost per episode; +- [x] illegal actions, failed queries, adapter failures, and repair attempts; +- [x] execution lane, provider route, model snapshot, and scaffold fingerprint; +- [x] result contract and seed-panel identity; and +- [x] per-mechanic outcomes for drafting, trades, free agency, cap management, scouting, and lineup decisions where supported. -- [ ] State how multiple model comparisons are handled. If adjusted inferential +- [x] State how multiple model comparisons are handled. If adjusted inferential claims are not justified at this sample size, label per-model p-values as descriptive and emphasize effect sizes and uncertainty instead. @@ -405,8 +409,8 @@ Claims to avoid: ### Phase 5 — blog, site, and durable artifacts -- [ ] Generate all tables and headline numbers from validated artifacts. -- [ ] Keep a visible “last updated” date and contract version on the site. +- [x] Generate all tables and headline numbers from validated artifacts. +- [x] Keep a visible “last updated” date and contract version on the site. - [ ] Lead the blog with the research question and the measurement problem, not with implementation history. - [ ] Explain the simulator and decision loop with one compact diagram. diff --git a/gm_bench/publication.py b/gm_bench/publication.py index ee2abc7..ca5c224 100644 --- a/gm_bench/publication.py +++ b/gm_bench/publication.py @@ -26,6 +26,11 @@ def compact_result(payload: dict[str, Any]) -> dict[str, Any]: if payload.get("publication"): raise ValueError("input already has publication metadata; compact the original raw artifact") result = copy.deepcopy(payload) + baseline_cache = result.get("baseline_cache") + if isinstance(baseline_cache, dict): + # The raw artifact may record an absolute local cache path for operator + # diagnostics. It is machine-specific and adds no publication evidence. + baseline_cache.pop("path", None) for label in ("candidate", "baselines"): blocks = [result.get(label)] if label == "candidate" else result.get(label, []) for block in blocks: diff --git a/results/analysis/publication-panel-analysis.json b/results/analysis/publication-panel-analysis.json new file mode 100644 index 0000000..9dba155 --- /dev/null +++ b/results/analysis/publication-panel-analysis.json @@ -0,0 +1,574 @@ +{ + "bootstrap": { + "iterations": 10000, + "label": "descriptive percentile 95% CI", + "seed": 20260716 + }, + "config_errors": [], + "eligible_model_count": 8, + "holm_family_size": 10, + "missing_models": [ + "openrouter-grok-4.5-xai", + "openrouter-mistral-medium-3.5-mistral" + ], + "models": [ + { + "artifact_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "bootstrap_ci95": [ + -209.560916, + -147.19875 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -179.768, + "model": "meta/muse-spark-1.1", + "model_id": "openrouter-muse-spark-1.1-meta", + "per_seed": [ + { + "candidate_mean_over_repeats": 248.494, + "lift": -207.231, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 305.737333, + "lift": -128.458667, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 274.606667, + "lift": -186.477333, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 222.253667, + "lift": -236.772333, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 188.619667, + "lift": -175.967333, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 181.748667, + "lift": -232.558333, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 207.929667, + "lift": -99.133333, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 225.417333, + "lift": -171.545667, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "bootstrap_ci95": [ + -239.071375, + -144.18575 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -194.079625, + "model": "z-ai/glm-5.2", + "model_id": "openrouter-glm-5.2-novita", + "per_seed": [ + { + "candidate_mean_over_repeats": 198.909, + "lift": -256.816, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 210.673, + "lift": -223.523, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 216.247, + "lift": -244.837, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 235.725, + "lift": -223.301, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 200.563333, + "lift": -164.023667, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 147.347333, + "lift": -266.959667, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 244.639667, + "lift": -62.423333, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 286.209667, + "lift": -110.753333, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "bootstrap_ci95": [ + -241.056333, + -143.75275 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -195.994875, + "model": "google/gemini-3.5-flash", + "model_id": "openrouter-gemini-3.5-flash-google-ai-studio", + "per_seed": [ + { + "candidate_mean_over_repeats": 179.596333, + "lift": -276.128667, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 228.073, + "lift": -206.123, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 218.597, + "lift": -242.487, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 187.538333, + "lift": -271.487667, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 161.742333, + "lift": -202.844667, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 249.007, + "lift": -165.3, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 259.947333, + "lift": -47.115667, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 240.490667, + "lift": -156.472333, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "bootstrap_ci95": [ + -252.634708, + -179.442375 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -215.777958, + "model": "tencent/hy3:free", + "model_id": "openrouter-tencent-hy3-free-novita", + "per_seed": [ + { + "candidate_mean_over_repeats": 175.823333, + "lift": -279.901667, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 206.049667, + "lift": -228.146333, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 169.768667, + "lift": -291.315333, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 213.246667, + "lift": -245.779333, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 188.015333, + "lift": -176.571667, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 218.39, + "lift": -195.917, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 184.724, + "lift": -122.339, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 210.709667, + "lift": -186.253333, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "bootstrap_ci95": [ + -277.546, + -187.278 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -236.099167, + "model": "qwen/qwen3.7-plus", + "model_id": "openrouter-qwen3.7-plus-alibaba", + "per_seed": [ + { + "candidate_mean_over_repeats": 147.156333, + "lift": -308.568667, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 162.474333, + "lift": -271.721667, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 165.137, + "lift": -295.947, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 213.920333, + "lift": -245.105667, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 175.608, + "lift": -188.979, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 140.678, + "lift": -273.629, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 217.295, + "lift": -89.768, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 181.888667, + "lift": -215.074333, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "bootstrap_ci95": [ + -273.144667, + -200.7945 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -237.693083, + "model": "openai/gpt-5.6-luna", + "model_id": "openrouter-gpt-5.6-luna-openai", + "per_seed": [ + { + "candidate_mean_over_repeats": 137.160333, + "lift": -318.564667, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 226.219667, + "lift": -207.976333, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 168.027, + "lift": -293.057, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 194.436333, + "lift": -264.589667, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 155.724, + "lift": -208.863, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 167.974667, + "lift": -246.332333, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 168.773, + "lift": -138.29, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 173.091333, + "lift": -223.871667, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "bootstrap_ci95": [ + -316.991292, + -217.256625 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -269.475375, + "model": "anthropic/claude-sonnet-5", + "model_id": "openrouter-claude-sonnet-5-bedrock", + "per_seed": [ + { + "candidate_mean_over_repeats": 116.985, + "lift": -338.74, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 172.292333, + "lift": -261.903667, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 112.787333, + "lift": -348.296667, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 124.900333, + "lift": -334.125667, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 166.97, + "lift": -197.617, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 118.29, + "lift": -296.017, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 178.447333, + "lift": -128.615667, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 146.475667, + "lift": -250.487333, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + }, + { + "artifact_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "bootstrap_ci95": [ + -329.425375, + -229.092667 + ], + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "mean_lift": -281.738875, + "model": "minimax/minimax-m3", + "model_id": "openrouter-minimax-m3-minimax", + "per_seed": [ + { + "candidate_mean_over_repeats": 98.365, + "lift": -357.36, + "pick_trader_score": 455.725, + "seed": 11 + }, + { + "candidate_mean_over_repeats": 144.612333, + "lift": -289.583667, + "pick_trader_score": 434.196, + "seed": 12 + }, + { + "candidate_mean_over_repeats": 107.610667, + "lift": -353.473333, + "pick_trader_score": 461.084, + "seed": 13 + }, + { + "candidate_mean_over_repeats": 120.417333, + "lift": -338.608667, + "pick_trader_score": 459.026, + "seed": 14 + }, + { + "candidate_mean_over_repeats": 177.757, + "lift": -186.83, + "pick_trader_score": 364.587, + "seed": 15 + }, + { + "candidate_mean_over_repeats": 99.804, + "lift": -314.503, + "pick_trader_score": 414.307, + "seed": 16 + }, + { + "candidate_mean_over_repeats": 159.197333, + "lift": -147.865667, + "pick_trader_score": 307.063, + "seed": 17 + }, + { + "candidate_mean_over_repeats": 131.276333, + "lift": -265.686667, + "pick_trader_score": 396.963, + "seed": 18 + } + ], + "provider": "openrouter", + "raw_artifact_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "seed_win_rate": 0.0, + "sign_flip_p_value": 0.0078125, + "tier": 1 + } + ], + "primary_contrast": "paired lift versus pick-trader", + "registered_model_count": 10, + "rejected_artifacts": [ + { + "artifact_index": 4, + "model_id": "openrouter-grok-4.5-xai", + "reasons": [ + "candidate usage must cover every decision point", + "candidate cost telemetry must cover every decision point" + ] + }, + { + "artifact_index": 6, + "model_id": "openrouter-mistral-medium-3.5-mistral", + "reasons": [ + "candidate cost telemetry must cover every decision point" + ] + } + ], + "schema_version": 1, + "sign_flip_inference": "descriptive; exact under the symmetry assumption", + "status": "partial" +} diff --git a/results/benchmark_results.md b/results/benchmark_results.md index 24bdaaf..458f49c 100644 --- a/results/benchmark_results.md +++ b/results/benchmark_results.md @@ -2,6 +2,30 @@ This snapshot records the benchmark results generated during the MVP build. +> **Phase-one public panel (2026-07-19, `sota-v2`):** Eight of ten +> pre-registered OpenRouter cells passed the frozen 4,096-token lane, exact-route, +> and complete-telemetry gates. Scores are shown with compute because output use +> is part of the comparison context. +> +> | Model | Mean | Stddev | Lift vs `pick-trader` | Tokens / decision | Cost | Illegal | +> | --- | ---: | ---: | ---: | ---: | ---: | ---: | +> | Muse Spark 1.1 | 231.851 | 39.764 | -179.768 | 10,084.2 | $7.5863 | 51 | +> | GLM 5.2 | 217.539 | 37.736 | -194.080 | 10,764.4 | $1.2812 | 168 | +> | Gemini 3.5 Flash | 215.624 | 33.263 | -195.995 | 8,770.7 | $6.3205 | 18 | +> | Tencent HY3 | 195.841 | 17.328 | -215.778 | 7,743.3 | $0.0000 | 10 | +> | Qwen 3.7 Plus | 175.520 | 26.355 | -236.099 | 8,039.6 | $1.2938 | 136 | +> | GPT-5.6 Luna | 173.926 | 24.839 | -237.693 | 7,611.2 | $4.6802 | 126 | +> | Claude Sonnet 5 | 142.143 | 25.552 | -269.475 | 9,944.4 | $11.0544 | 44 | +> | MiniMax M3 | 129.880 | 27.051 | -281.739 | 8,728.2 | $1.2371 | 14 | +> +> These rows are one overlapping uncertainty tier, not an ordinal model ranking. +> Each raw sign-flip result is descriptive; after Holm adjustment across the full +> ten-model registered family, every adjusted p-value is 0.078125. All eight +> eligible models trail `pick-trader` (411.619). Grok 4.5 and Mistral Medium 3.5 +> completed but were excluded from the headline table for incomplete usage/cost +> coverage; their evidence remains diagnostic. The machine-generated analysis is +> in `results/analysis/publication-panel-analysis.json`. + > **Note (2026-07-08/09, protocol v2 / contract `cf2607e59dba`):** Midseason, > multi-round interaction, functional injuries, and the full v1 baseline panel > (`strategic`, `pick-trader`) are now on `main`. SOTA-v1 claims must use this @@ -263,4 +287,3 @@ paid/external providers while creating the repository. | Claude Code | `CLAUDE_MODEL=opus CLAUDE_EFFORT=high` | 1 | 1 | 94.428 | 16 | 0 | 13 | Real model run using Claude CLI `opus` alias. | | Claude Code | `CLAUDE_MODEL=claude-opus-4.8 CLAUDE_EFFORT=high` | 1 | 1 | 110.534 | 14 | 0 | 0 | Did not reach model; Claude reported Opus 4 retired on 2026-06-15. | | Ollama | `OLLAMA_MODEL=gemma4:e4b` | 1 | 1 | 105.534 | 14 | 0 | 2 | Local model run, slow but model-driven. | - diff --git a/results/diagnostics/openrouter-mistral-medium-3.5-mistral.json b/results/diagnostics/openrouter-mistral-medium-3.5-mistral.json new file mode 100644 index 0000000..57aee94 --- /dev/null +++ b/results/diagnostics/openrouter-mistral-medium-3.5-mistral.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:mistralai/mistral-medium-3-5", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:mistralai/mistral-medium-3-5", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 23, + "final_score": 117.042, + "illegal_actions": 2, + "memo_writes": 5, + "protocol_penalty": 5.0, + "query_declines": 15, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 122.042, + "usage": { + "api_calls": 24, + "api_latency_ms": 55356.2, + "cached_input_tokens": 4504, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.30922, + "decisions_with_usage": 20, + "harness_latency_ms": 57296.6, + "input_tokens": 191983, + "max_output_tokens_per_call": 332, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2833, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 194816, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 211 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 90.793, + "illegal_actions": 10, + "memo_writes": 10, + "protocol_penalty": 25.0, + "query_declines": 10, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 115.793, + "usage": { + "api_calls": 24, + "api_latency_ms": 49252.5, + "cached_input_tokens": 50608, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.311606, + "decisions_with_usage": 20, + "harness_latency_ms": 51195.7, + "input_tokens": 190551, + "max_output_tokens_per_call": 412, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3437, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 193988, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 189 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 111.985, + "illegal_actions": 1, + "memo_writes": 6, + "protocol_penalty": 2.5, + "query_declines": 26, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 114.485, + "usage": { + "api_calls": 23, + "api_latency_ms": 40579.9, + "cached_input_tokens": 54920, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.28515, + "decisions_with_usage": 20, + "harness_latency_ms": 42442.9, + "input_tokens": 178410, + "max_output_tokens_per_call": 256, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2338, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 180748, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 181 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 169.944, + "illegal_actions": 6, + "memo_writes": 10, + "protocol_penalty": 15.0, + "query_declines": 7, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 184.944, + "usage": { + "api_calls": 23, + "api_latency_ms": 52809.0, + "cached_input_tokens": 18360, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.295978, + "decisions_with_usage": 20, + "harness_latency_ms": 54645.9, + "input_tokens": 182179, + "max_output_tokens_per_call": 314, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3028, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 185207, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 268 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 137.009, + "illegal_actions": 0, + "memo_writes": 4, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 137.009, + "usage": { + "api_calls": 20, + "api_latency_ms": 35594.8, + "cached_input_tokens": 52528, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.25099, + "decisions_with_usage": 20, + "harness_latency_ms": 37219.8, + "input_tokens": 156492, + "max_output_tokens_per_call": 355, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2167, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 158659, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 174.219, + "illegal_actions": 6, + "memo_writes": 11, + "protocol_penalty": 15.0, + "query_declines": 7, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 189.219, + "usage": { + "api_calls": 23, + "api_latency_ms": 70947.7, + "cached_input_tokens": 52552, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.303728, + "decisions_with_usage": 20, + "harness_latency_ms": 72814.3, + "input_tokens": 184190, + "max_output_tokens_per_call": 345, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3659, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 187849, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 264 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 112.033, + "illegal_actions": 7, + "memo_writes": 13, + "protocol_penalty": 17.5, + "query_declines": 8, + "rejected_offers": 5, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 129.533, + "usage": { + "api_calls": 27, + "api_latency_ms": 66403.2, + "cached_input_tokens": 24504, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.366905, + "decisions_with_usage": 20, + "harness_latency_ms": 68639.0, + "input_tokens": 222666, + "max_output_tokens_per_call": 386, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 4388, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 227054, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 141.622, + "illegal_actions": 4, + "memo_writes": 11, + "protocol_penalty": 10.0, + "query_declines": 6, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 151.622, + "usage": { + "api_calls": 22, + "api_latency_ms": 46755.9, + "cached_input_tokens": 41040, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.295459, + "decisions_with_usage": 20, + "harness_latency_ms": 48536.6, + "input_tokens": 181353, + "max_output_tokens_per_call": 282, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3124, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 184477, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 121.606, + "illegal_actions": 3, + "memo_writes": 11, + "protocol_penalty": 7.5, + "query_declines": 9, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 129.106, + "usage": { + "api_calls": 22, + "api_latency_ms": 58048.1, + "cached_input_tokens": 36448, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.29604, + "decisions_with_usage": 20, + "harness_latency_ms": 59839.7, + "input_tokens": 180556, + "max_output_tokens_per_call": 357, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3361, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 183917, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 208 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 126.4, + "illegal_actions": 2, + "memo_writes": 7, + "protocol_penalty": 5.0, + "query_declines": 9, + "rejected_offers": 3, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 131.4, + "usage": { + "api_calls": 22, + "api_latency_ms": 48255.5, + "cached_input_tokens": 16768, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.275715, + "decisions_with_usage": 20, + "harness_latency_ms": 50037.1, + "input_tokens": 169600, + "max_output_tokens_per_call": 349, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2842, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 172442, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 243 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 117.149, + "illegal_actions": 3, + "memo_writes": 8, + "protocol_penalty": 7.5, + "query_declines": 23, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 124.649, + "usage": { + "api_calls": 25, + "api_latency_ms": 50423.6, + "cached_input_tokens": 42792, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.323424, + "decisions_with_usage": 20, + "harness_latency_ms": 52540.1, + "input_tokens": 197971, + "max_output_tokens_per_call": 430, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3529, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 201500, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 262 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 89.568, + "illegal_actions": 4, + "memo_writes": 9, + "protocol_penalty": 10.0, + "query_declines": 8, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 99.568, + "usage": { + "api_calls": 21, + "api_latency_ms": 47520.1, + "cached_input_tokens": 62984, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.271766, + "decisions_with_usage": 20, + "harness_latency_ms": 49241.6, + "input_tokens": 165622, + "max_output_tokens_per_call": 293, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3111, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 168733, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 255 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 164.755, + "illegal_actions": 0, + "memo_writes": 10, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 164.755, + "usage": { + "api_calls": 21, + "api_latency_ms": 43010.5, + "cached_input_tokens": 18968, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.278553, + "decisions_with_usage": 20, + "harness_latency_ms": 44705.2, + "input_tokens": 169576, + "max_output_tokens_per_call": 336, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3225, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 172801, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 122.85, + "illegal_actions": 6, + "memo_writes": 9, + "protocol_penalty": 15.0, + "query_declines": 7, + "rejected_offers": 3, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 137.85, + "usage": { + "api_calls": 21, + "api_latency_ms": 48767.1, + "cached_input_tokens": 47720, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.285232, + "decisions_with_usage": 20, + "harness_latency_ms": 50450.2, + "input_tokens": 171923, + "max_output_tokens_per_call": 446, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3646, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 175569, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 205 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 138.107, + "illegal_actions": 5, + "memo_writes": 10, + "protocol_penalty": 12.5, + "query_declines": 11, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 150.607, + "usage": { + "api_calls": 24, + "api_latency_ms": 53303.6, + "cached_input_tokens": 33984, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.318792, + "decisions_with_usage": 20, + "harness_latency_ms": 55210.6, + "input_tokens": 194969, + "max_output_tokens_per_call": 373, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3512, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 198481, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 215 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 121.915, + "illegal_actions": 7, + "memo_writes": 7, + "protocol_penalty": 17.5, + "query_declines": 6, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 139.415, + "usage": { + "api_calls": 23, + "api_latency_ms": 48595.6, + "cached_input_tokens": 17960, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.285779, + "decisions_with_usage": 20, + "harness_latency_ms": 50467.1, + "input_tokens": 176874, + "max_output_tokens_per_call": 391, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2729, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 179603, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 174 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 125.089, + "illegal_actions": 1, + "memo_writes": 8, + "protocol_penalty": 2.5, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 127.589, + "usage": { + "api_calls": 21, + "api_latency_ms": 47176.3, + "cached_input_tokens": 48520, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.268703, + "decisions_with_usage": 20, + "harness_latency_ms": 48885.5, + "input_tokens": 163656, + "max_output_tokens_per_call": 388, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166752, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 190 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 9, + "final_score": 225.01, + "illegal_actions": 9, + "memo_writes": 11, + "protocol_penalty": 22.5, + "query_declines": 11, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 247.51, + "usage": { + "api_calls": 23, + "api_latency_ms": 48854.3, + "cached_input_tokens": 61304, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.300373, + "decisions_with_usage": 20, + "harness_latency_ms": 50716.9, + "input_tokens": 182320, + "max_output_tokens_per_call": 439, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3586, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 185906, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 210 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 10, + "final_score": 137.596, + "illegal_actions": 2, + "memo_writes": 7, + "protocol_penalty": 5.0, + "query_declines": 6, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 142.596, + "usage": { + "api_calls": 21, + "api_latency_ms": 45644.9, + "cached_input_tokens": 19320, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.265447, + "decisions_with_usage": 20, + "harness_latency_ms": 47391.6, + "input_tokens": 165370, + "max_output_tokens_per_call": 305, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2319, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167689, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 195 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 127.886, + "illegal_actions": 5, + "memo_writes": 6, + "protocol_penalty": 12.5, + "query_declines": 9, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 140.386, + "usage": { + "api_calls": 21, + "api_latency_ms": 41162.7, + "cached_input_tokens": 38888, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.266629, + "decisions_with_usage": 20, + "harness_latency_ms": 42860.9, + "input_tokens": 164671, + "max_output_tokens_per_call": 385, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2616, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167287, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 247 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 1, + "final_score": 158.697, + "illegal_actions": 5, + "memo_writes": 9, + "protocol_penalty": 12.5, + "query_declines": 5, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 171.197, + "usage": { + "api_calls": 22, + "api_latency_ms": 52018.2, + "cached_input_tokens": 40552, + "calls_with_finish_reason": 21, + "cost_decisions": 19, + "cost_usd": 0.266892, + "decisions_with_usage": 20, + "harness_latency_ms": 53798.4, + "input_tokens": 163872, + "max_output_tokens_per_call": 392, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2811, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166683, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 167.537, + "illegal_actions": 4, + "memo_writes": 11, + "protocol_penalty": 10.0, + "query_declines": 18, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 177.537, + "usage": { + "api_calls": 26, + "api_latency_ms": 52865.4, + "cached_input_tokens": 21968, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.3282, + "decisions_with_usage": 20, + "harness_latency_ms": 55011.6, + "input_tokens": 202346, + "max_output_tokens_per_call": 250, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3291, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 205637, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 256 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 175.09, + "illegal_actions": 3, + "memo_writes": 4, + "protocol_penalty": 7.5, + "query_declines": 16, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 182.59, + "usage": { + "api_calls": 22, + "api_latency_ms": 36588.2, + "cached_input_tokens": 46784, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.273301, + "decisions_with_usage": 20, + "harness_latency_ms": 38379.8, + "input_tokens": 170758, + "max_output_tokens_per_call": 253, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 2289, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 173047, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 249 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 146.793, + "illegal_actions": 5, + "memo_writes": 8, + "protocol_penalty": 12.5, + "query_declines": 13, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 159.293, + "usage": { + "api_calls": 20, + "api_latency_ms": 42483.1, + "cached_input_tokens": 40832, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.254699, + "decisions_with_usage": 20, + "harness_latency_ms": 44130.7, + "input_tokens": 153874, + "max_output_tokens_per_call": 355, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 3185, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157059, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "wins": 246 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.002, + "decisions": 480, + "failed_decisions": 1, + "failed_queries": 139, + "illegal_actions": 100, + "mean_score": 138.362, + "mean_strategy_score": 148.779, + "mean_total_wins": 223.375, + "memo_writes": 205, + "query_declines": 248, + "rejected_offers": 17, + "score_stddev": 20.652, + "total_protocol_penalty": 250.0, + "usage": { + "api_calls": 541, + "api_latency_ms": 1182416.4, + "cached_input_tokens": 894808, + "calls_with_finish_reason": 540, + "cost_decisions": 479, + "cost_usd": 6.978581, + "decisions_with_usage": 480, + "harness_latency_ms": 1226457.8, + "input_tokens": 4281782, + "max_output_tokens_per_call": 446, + "mean_input_tokens_per_decision": 8920.4, + "mean_output_tokens_per_decision": 154.4, + "mean_tokens_per_decision": 9074.8, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 74122, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4355904, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "within_seed_score_stddev": 18.248 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.002, + "candidate_decisions": 480, + "candidate_failed_decisions": 1, + "candidate_failed_queries": 139, + "candidate_illegal_actions": 100, + "candidate_mean_score": 138.362, + "candidate_mean_strategy_score": 148.779, + "candidate_mean_tokens_per_decision": 9074.8, + "candidate_memo_writes": 205, + "candidate_protocol_penalty": 250.0, + "candidate_query_declines": 248, + "candidate_rejected_offers": 17, + "candidate_usage": { + "api_calls": 541, + "api_latency_ms": 1182416.4, + "cached_input_tokens": 894808, + "calls_with_finish_reason": 540, + "cost_decisions": 479, + "cost_usd": 6.978581, + "decisions_with_usage": 480, + "harness_latency_ms": 1226457.8, + "input_tokens": 4281782, + "max_output_tokens_per_call": 446, + "mean_input_tokens_per_decision": 8920.4, + "mean_output_tokens_per_decision": 154.4, + "mean_tokens_per_decision": 9074.8, + "model": "mistralai/mistral-medium-3-5", + "output_tokens": 74122, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4355904, + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_providers": [ + "Mistral" + ] + }, + "score_lift": -135.432, + "score_lift_pct": -49.46 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -273.257, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -158.41, + -115.842 + ], + "paired_lift_mean": -135.432, + "paired_lift_stddev": 30.086, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 106.607, + "lift": -160.163, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 160.391, + "lift": -125.039, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 125.087, + "lift": -156.356, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 111.039, + "lift": -193.111, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 141.904, + "lift": -129.014, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 157.338, + "lift": -105.984, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 141.393, + "lift": -96.411, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 163.14, + "lift": -117.378, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 113, + "rejected": 38 + }, + "draft": { + "accepted": 1748, + "rejected": 251 + }, + "information_memory": { + "accepted": 471, + "rejected": 138 + }, + "lineup": { + "accepted": 118, + "rejected": 50 + }, + "trades": { + "accepted": 543, + "rejected": 27 + } + }, + "raw_artifact_sha256": "f04e5c6fc5d01d2368d1c52eb3c8cdfe43674ad476fbfc28b598d8f29690b6cc", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:mistralai/mistral-medium-3-5", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "mistralai/mistral-medium-3-5", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Mistral | mistralai/mistral-medium-3.5-20260430", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Mistral", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "mistral", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T21:21:37+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-claude-sonnet-5-bedrock.json b/results/leaderboard/openrouter-claude-sonnet-5-bedrock.json new file mode 100644 index 0000000..990b4cc --- /dev/null +++ b/results/leaderboard/openrouter-claude-sonnet-5-bedrock.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:anthropic/claude-sonnet-5", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:anthropic/claude-sonnet-5", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 100.102, + "illegal_actions": 0, + "memo_writes": 15, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 100.102, + "usage": { + "api_calls": 23, + "api_latency_ms": 133736.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.475196, + "decisions_with_usage": 20, + "harness_latency_ms": 135638.2, + "input_tokens": 197928, + "max_output_tokens_per_call": 772, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7934, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 205862, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 185 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 161.778, + "illegal_actions": 5, + "memo_writes": 17, + "protocol_penalty": 12.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 174.278, + "usage": { + "api_calls": 23, + "api_latency_ms": 140971.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.497272, + "decisions_with_usage": 20, + "harness_latency_ms": 142871.1, + "input_tokens": 203831, + "max_output_tokens_per_call": 703, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8961, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 212792, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 188 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 89.075, + "illegal_actions": 10, + "memo_writes": 14, + "protocol_penalty": 25.0, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 114.075, + "usage": { + "api_calls": 21, + "api_latency_ms": 123392.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.440344, + "decisions_with_usage": 20, + "harness_latency_ms": 125082.0, + "input_tokens": 179722, + "max_output_tokens_per_call": 1043, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8090, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 187812, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 193 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 155.512, + "illegal_actions": 1, + "memo_writes": 16, + "protocol_penalty": 2.5, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 158.012, + "usage": { + "api_calls": 22, + "api_latency_ms": 123948.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.459256, + "decisions_with_usage": 20, + "harness_latency_ms": 125752.4, + "input_tokens": 192413, + "max_output_tokens_per_call": 654, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7443, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 199856, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 280 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 164.975, + "illegal_actions": 1, + "memo_writes": 14, + "protocol_penalty": 2.5, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 167.475, + "usage": { + "api_calls": 22, + "api_latency_ms": 143575.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.479832, + "decisions_with_usage": 20, + "harness_latency_ms": 145450.2, + "input_tokens": 193141, + "max_output_tokens_per_call": 833, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 9355, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 202496, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 298 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 196.39, + "illegal_actions": 0, + "memo_writes": 17, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 196.39, + "usage": { + "api_calls": 22, + "api_latency_ms": 135151.4, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.468082, + "decisions_with_usage": 20, + "harness_latency_ms": 136942.6, + "input_tokens": 191886, + "max_output_tokens_per_call": 750, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8431, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 200317, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 294 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 120.278, + "illegal_actions": 2, + "memo_writes": 16, + "protocol_penalty": 5.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 125.278, + "usage": { + "api_calls": 21, + "api_latency_ms": 122321.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.44587, + "decisions_with_usage": 20, + "harness_latency_ms": 124040.0, + "input_tokens": 185490, + "max_output_tokens_per_call": 706, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7489, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 192979, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 172 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 76.068, + "illegal_actions": 1, + "memo_writes": 14, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 78.568, + "usage": { + "api_calls": 21, + "api_latency_ms": 122195.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.4428, + "decisions_with_usage": 20, + "harness_latency_ms": 123887.0, + "input_tokens": 184850, + "max_output_tokens_per_call": 842, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7310, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 192160, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 142.016, + "illegal_actions": 3, + "memo_writes": 17, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 149.516, + "usage": { + "api_calls": 24, + "api_latency_ms": 146582.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.533578, + "decisions_with_usage": 20, + "harness_latency_ms": 148551.0, + "input_tokens": 219534, + "max_output_tokens_per_call": 786, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 9451, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 228985, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 211 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 128.114, + "illegal_actions": 1, + "memo_writes": 13, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 130.614, + "usage": { + "api_calls": 22, + "api_latency_ms": 108108.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.436428, + "decisions_with_usage": 20, + "harness_latency_ms": 109947.8, + "input_tokens": 185474, + "max_output_tokens_per_call": 784, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 6548, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 192022, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 243 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 126.486, + "illegal_actions": 1, + "memo_writes": 15, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 128.986, + "usage": { + "api_calls": 21, + "api_latency_ms": 106667.5, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.4132, + "decisions_with_usage": 20, + "harness_latency_ms": 108424.3, + "input_tokens": 175310, + "max_output_tokens_per_call": 540, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 6258, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 181568, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 262 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 120.101, + "illegal_actions": 2, + "memo_writes": 16, + "protocol_penalty": 5.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 125.101, + "usage": { + "api_calls": 21, + "api_latency_ms": 120688.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.42457, + "decisions_with_usage": 20, + "harness_latency_ms": 122490.8, + "input_tokens": 176315, + "max_output_tokens_per_call": 779, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7194, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 183509, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 240 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 119.791, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 119.791, + "usage": { + "api_calls": 20, + "api_latency_ms": 121935.7, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.423204, + "decisions_with_usage": 20, + "harness_latency_ms": 123576.6, + "input_tokens": 174782, + "max_output_tokens_per_call": 795, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7364, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 182146, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 265.475, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 265.475, + "usage": { + "api_calls": 21, + "api_latency_ms": 127918.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.448736, + "decisions_with_usage": 20, + "harness_latency_ms": 129646.0, + "input_tokens": 184818, + "max_output_tokens_per_call": 695, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7910, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 192728, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 235 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 115.644, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 115.644, + "usage": { + "api_calls": 21, + "api_latency_ms": 130611.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.45201, + "decisions_with_usage": 20, + "harness_latency_ms": 132358.9, + "input_tokens": 185825, + "max_output_tokens_per_call": 895, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8036, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 193861, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 220 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 125.86, + "illegal_actions": 5, + "memo_writes": 15, + "protocol_penalty": 12.5, + "query_declines": 7, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 138.36, + "usage": { + "api_calls": 21, + "api_latency_ms": 115390.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.425368, + "decisions_with_usage": 20, + "harness_latency_ms": 117136.7, + "input_tokens": 177064, + "max_output_tokens_per_call": 736, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7124, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 184188, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 218 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 80.708, + "illegal_actions": 2, + "memo_writes": 10, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 85.708, + "usage": { + "api_calls": 23, + "api_latency_ms": 107117.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.451766, + "decisions_with_usage": 20, + "harness_latency_ms": 109061.4, + "input_tokens": 194953, + "max_output_tokens_per_call": 552, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 6186, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 201139, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 196 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 148.302, + "illegal_actions": 0, + "memo_writes": 15, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 148.302, + "usage": { + "api_calls": 22, + "api_latency_ms": 133112.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.456258, + "decisions_with_usage": 20, + "harness_latency_ms": 134917.7, + "input_tokens": 186764, + "max_output_tokens_per_call": 826, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8273, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 195037, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 211 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 185.141, + "illegal_actions": 2, + "memo_writes": 18, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 190.141, + "usage": { + "api_calls": 21, + "api_latency_ms": 142621.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.45519, + "decisions_with_usage": 20, + "harness_latency_ms": 144364.6, + "input_tokens": 182940, + "max_output_tokens_per_call": 677, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 8931, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 191871, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 234 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 12, + "final_score": 183.195, + "illegal_actions": 2, + "memo_writes": 19, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 188.195, + "usage": { + "api_calls": 26, + "api_latency_ms": 162337.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.565836, + "decisions_with_usage": 20, + "harness_latency_ms": 164529.6, + "input_tokens": 232588, + "max_output_tokens_per_call": 651, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 10066, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 242654, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 224 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 167.006, + "illegal_actions": 3, + "memo_writes": 12, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 174.506, + "usage": { + "api_calls": 23, + "api_latency_ms": 129043.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.482296, + "decisions_with_usage": 20, + "harness_latency_ms": 130974.4, + "input_tokens": 202543, + "max_output_tokens_per_call": 792, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7721, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 210264, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 238 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 153.768, + "illegal_actions": 1, + "memo_writes": 17, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 156.268, + "usage": { + "api_calls": 22, + "api_latency_ms": 119060.5, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.447806, + "decisions_with_usage": 20, + "harness_latency_ms": 120803.2, + "input_tokens": 189463, + "max_output_tokens_per_call": 701, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 6888, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 196351, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 264 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 121.141, + "illegal_actions": 0, + "memo_writes": 17, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 121.141, + "usage": { + "api_calls": 23, + "api_latency_ms": 126840.4, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.473638, + "decisions_with_usage": 20, + "harness_latency_ms": 128760.1, + "input_tokens": 197469, + "max_output_tokens_per_call": 647, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7870, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 205339, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 250 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 164.518, + "illegal_actions": 2, + "memo_writes": 13, + "protocol_penalty": 5.0, + "query_declines": 3, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 169.518, + "usage": { + "api_calls": 22, + "api_latency_ms": 129241.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.455824, + "decisions_with_usage": 20, + "harness_latency_ms": 131034.0, + "input_tokens": 189762, + "max_output_tokens_per_call": 1009, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 7630, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 197392, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "wins": 282 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 4, + "decision_failure_rate": 0.0, + "decisions": 480, + "failed_decisions": 0, + "failed_queries": 137, + "illegal_actions": 44, + "mean_score": 142.143, + "mean_strategy_score": 146.727, + "mean_total_wins": 229.75, + "memo_writes": 361, + "query_declines": 27, + "rejected_offers": 4, + "score_stddev": 25.552, + "total_protocol_penalty": 110.0, + "usage": { + "api_calls": 528, + "api_latency_ms": 3072570.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 528, + "cost_decisions": 480, + "cost_usd": 11.05436, + "decisions_with_usage": 480, + "harness_latency_ms": 3116240.6, + "input_tokens": 4584865, + "max_output_tokens_per_call": 1043, + "mean_input_tokens_per_decision": 9551.8, + "mean_output_tokens_per_decision": 392.6, + "mean_tokens_per_decision": 9944.4, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 188463, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4773328, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "within_seed_score_stddev": 25.59 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.0, + "candidate_decisions": 480, + "candidate_failed_decisions": 0, + "candidate_failed_queries": 137, + "candidate_illegal_actions": 44, + "candidate_mean_score": 142.143, + "candidate_mean_strategy_score": 146.727, + "candidate_mean_tokens_per_decision": 9944.4, + "candidate_memo_writes": 361, + "candidate_protocol_penalty": 110.0, + "candidate_query_declines": 27, + "candidate_rejected_offers": 4, + "candidate_usage": { + "api_calls": 528, + "api_latency_ms": 3072570.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 528, + "cost_decisions": 480, + "cost_usd": 11.05436, + "decisions_with_usage": 480, + "harness_latency_ms": 3116240.6, + "input_tokens": 4584865, + "max_output_tokens_per_call": 1043, + "mean_input_tokens_per_decision": 9551.8, + "mean_output_tokens_per_decision": 392.6, + "mean_tokens_per_decision": 9944.4, + "model": "anthropic/claude-sonnet-5", + "output_tokens": 188463, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4773328, + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_providers": [ + "Amazon Bedrock" + ] + }, + "score_lift": -131.651, + "score_lift_pct": -48.08 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -269.475, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -154.031, + -104.212 + ], + "paired_lift_mean": -131.651, + "paired_lift_stddev": 36.203, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 116.985, + "lift": -149.784, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 172.292, + "lift": -113.137, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 112.787, + "lift": -168.656, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 124.9, + "lift": -179.25, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 166.97, + "lift": -103.948, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 118.29, + "lift": -145.032, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 178.447, + "lift": -59.357, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 146.476, + "lift": -134.042, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 85, + "rejected": 9 + }, + "draft": { + "accepted": 1699, + "rejected": 27 + }, + "information_memory": { + "accepted": 743, + "rejected": 137 + }, + "lineup": { + "accepted": 138, + "rejected": 39 + }, + "trades": { + "accepted": 588, + "rejected": 0 + } + }, + "raw_artifact_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:anthropic/claude-sonnet-5", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "anthropic/claude-sonnet-5", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Amazon Bedrock", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "amazon-bedrock/global", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T00:00:01+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-gemini-3.5-flash-google-ai-studio.json b/results/leaderboard/openrouter-gemini-3.5-flash-google-ai-studio.json new file mode 100644 index 0000000..a333e7f --- /dev/null +++ b/results/leaderboard/openrouter-gemini-3.5-flash-google-ai-studio.json @@ -0,0 +1,4150 @@ +{ + "agent": "openrouter:google/gemini-3.5-flash", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:google/gemini-3.5-flash", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 238.252, + "illegal_actions": 3, + "memo_writes": 0, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 245.752, + "usage": { + "api_calls": 25, + "api_latency_ms": 31425.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.295316, + "decisions_with_usage": 20, + "harness_latency_ms": 33485.0, + "input_tokens": 190955, + "max_output_tokens_per_call": 134, + "model": "google/gemini-3.5-flash", + "output_tokens": 987, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 191942, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 223 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 177.532, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 180.032, + "usage": { + "api_calls": 27, + "api_latency_ms": 32706.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.311114, + "decisions_with_usage": 20, + "harness_latency_ms": 34820.0, + "input_tokens": 200137, + "max_output_tokens_per_call": 167, + "model": "google/gemini-3.5-flash", + "output_tokens": 1212, + "protocol_repair_attempts": 5, + "protocol_repairs_succeeded": 5, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 201349, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 182 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 123.005, + "illegal_actions": 1, + "memo_writes": 2, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 125.505, + "usage": { + "api_calls": 23, + "api_latency_ms": 27259.5, + "cached_input_tokens": 12121, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.252006, + "decisions_with_usage": 20, + "harness_latency_ms": 29117.3, + "input_tokens": 171778, + "max_output_tokens_per_call": 143, + "model": "google/gemini-3.5-flash", + "output_tokens": 1189, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 172967, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 170 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 200.407, + "illegal_actions": 0, + "memo_writes": 1, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 200.407, + "usage": { + "api_calls": 22, + "api_latency_ms": 27675.4, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.25882, + "decisions_with_usage": 20, + "harness_latency_ms": 29405.8, + "input_tokens": 165688, + "max_output_tokens_per_call": 178, + "model": "google/gemini-3.5-flash", + "output_tokens": 1143, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166831, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 299 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 2, + "final_score": 227.501, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 227.501, + "usage": { + "api_calls": 22, + "api_latency_ms": 27179.8, + "cached_input_tokens": 12124, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.237208, + "decisions_with_usage": 20, + "harness_latency_ms": 28942.4, + "input_tokens": 163140, + "max_output_tokens_per_call": 175, + "model": "google/gemini-3.5-flash", + "output_tokens": 985, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 164125, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 295 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 256.311, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 256.311, + "usage": { + "api_calls": 23, + "api_latency_ms": 29741.4, + "cached_input_tokens": 20206, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.247648, + "decisions_with_usage": 20, + "harness_latency_ms": 31587.2, + "input_tokens": 175627, + "max_output_tokens_per_call": 239, + "model": "google/gemini-3.5-flash", + "output_tokens": 1276, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 176903, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 290 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 316.549, + "illegal_actions": 4, + "memo_writes": 0, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 326.549, + "usage": { + "api_calls": 25, + "api_latency_ms": 32436.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.308378, + "decisions_with_usage": 20, + "harness_latency_ms": 34720.4, + "input_tokens": 197030, + "max_output_tokens_per_call": 556, + "model": "google/gemini-3.5-flash", + "output_tokens": 1426, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 198456, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 195 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 202.994, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 202.994, + "usage": { + "api_calls": 21, + "api_latency_ms": 25921.8, + "cached_input_tokens": 4042, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.245734, + "decisions_with_usage": 20, + "harness_latency_ms": 27601.7, + "input_tokens": 162823, + "max_output_tokens_per_call": 97, + "model": "google/gemini-3.5-flash", + "output_tokens": 773, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 163596, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 219 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 136.248, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 138.748, + "usage": { + "api_calls": 25, + "api_latency_ms": 32990.1, + "cached_input_tokens": 0, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.302255, + "decisions_with_usage": 20, + "harness_latency_ms": 35426.6, + "input_tokens": 192431, + "max_output_tokens_per_call": 536, + "model": "google/gemini-3.5-flash", + "output_tokens": 1512, + "protocol_repair_attempts": 4, + "protocol_repairs_succeeded": 4, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 193943, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 223 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 0, + "final_score": 179.244, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 179.244, + "usage": { + "api_calls": 22, + "api_latency_ms": 26617.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.25523, + "decisions_with_usage": 20, + "harness_latency_ms": 28645.8, + "input_tokens": 163870, + "max_output_tokens_per_call": 142, + "model": "google/gemini-3.5-flash", + "output_tokens": 1047, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 164917, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 311 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 219.096, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 221.596, + "usage": { + "api_calls": 20, + "api_latency_ms": 26316.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.236701, + "decisions_with_usage": 20, + "harness_latency_ms": 28010.1, + "input_tokens": 147851, + "max_output_tokens_per_call": 539, + "model": "google/gemini-3.5-flash", + "output_tokens": 1658, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 149509, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 164.275, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 164.275, + "usage": { + "api_calls": 22, + "api_latency_ms": 29178.4, + "cached_input_tokens": 32325, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.213135, + "decisions_with_usage": 20, + "harness_latency_ms": 30959.5, + "input_tokens": 162621, + "max_output_tokens_per_call": 510, + "model": "google/gemini-3.5-flash", + "output_tokens": 1427, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 164048, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 270 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 150.947, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 150.947, + "usage": { + "api_calls": 20, + "api_latency_ms": 29357.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.248546, + "decisions_with_usage": 20, + "harness_latency_ms": 30959.4, + "input_tokens": 155334, + "max_output_tokens_per_call": 397, + "model": "google/gemini-3.5-flash", + "output_tokens": 1727, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157061, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 233 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 159.082, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 161.582, + "usage": { + "api_calls": 20, + "api_latency_ms": 27813.1, + "cached_input_tokens": 12124, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.226894, + "decisions_with_usage": 20, + "harness_latency_ms": 29437.8, + "input_tokens": 156154, + "max_output_tokens_per_call": 172, + "model": "google/gemini-3.5-flash", + "output_tokens": 1003, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157157, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 221 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.198, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 175.198, + "usage": { + "api_calls": 20, + "api_latency_ms": 29234.6, + "cached_input_tokens": 12125, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.229911, + "decisions_with_usage": 20, + "harness_latency_ms": 30853.0, + "input_tokens": 154587, + "max_output_tokens_per_call": 409, + "model": "google/gemini-3.5-flash", + "output_tokens": 1600, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 156187, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 235 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 297.684, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 297.684, + "usage": { + "api_calls": 20, + "api_latency_ms": 29569.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.241208, + "decisions_with_usage": 20, + "harness_latency_ms": 31142.5, + "input_tokens": 153407, + "max_output_tokens_per_call": 181, + "model": "google/gemini-3.5-flash", + "output_tokens": 1233, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 154640, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 219.73, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 219.73, + "usage": { + "api_calls": 22, + "api_latency_ms": 29114.9, + "cached_input_tokens": 4040, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.251798, + "decisions_with_usage": 20, + "harness_latency_ms": 30874.2, + "input_tokens": 166869, + "max_output_tokens_per_call": 134, + "model": "google/gemini-3.5-flash", + "output_tokens": 772, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167641, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 259 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 2, + "failed_queries": 0, + "final_score": 229.607, + "illegal_actions": 2, + "memo_writes": 0, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 234.607, + "usage": { + "api_calls": 27, + "api_latency_ms": 35192.5, + "cached_input_tokens": 8082, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.29742, + "decisions_with_usage": 20, + "harness_latency_ms": 37399.8, + "input_tokens": 200032, + "max_output_tokens_per_call": 100, + "model": "google/gemini-3.5-flash", + "output_tokens": 920, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 200952, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 247 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 221.668, + "illegal_actions": 2, + "memo_writes": 0, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 226.668, + "usage": { + "api_calls": 23, + "api_latency_ms": 28679.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.271442, + "decisions_with_usage": 20, + "harness_latency_ms": 30533.9, + "input_tokens": 174834, + "max_output_tokens_per_call": 145, + "model": "google/gemini-3.5-flash", + "output_tokens": 1021, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 175855, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 224.59, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 227.09, + "usage": { + "api_calls": 23, + "api_latency_ms": 30358.7, + "cached_input_tokens": 12124, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.257938, + "decisions_with_usage": 20, + "harness_latency_ms": 32182.6, + "input_tokens": 175572, + "max_output_tokens_per_call": 166, + "model": "google/gemini-3.5-flash", + "output_tokens": 1216, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 176788, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 260 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 0, + "final_score": 333.584, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 333.584, + "usage": { + "api_calls": 24, + "api_latency_ms": 30212.6, + "cached_input_tokens": 8081, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.270121, + "decisions_with_usage": 20, + "harness_latency_ms": 32143.5, + "input_tokens": 180064, + "max_output_tokens_per_call": 229, + "model": "google/gemini-3.5-flash", + "output_tokens": 1215, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 181279, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 256 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 165.745, + "illegal_actions": 1, + "memo_writes": 0, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 168.245, + "usage": { + "api_calls": 28, + "api_latency_ms": 34558.7, + "cached_input_tokens": 0, + "calls_with_finish_reason": 28, + "cost_decisions": 20, + "cost_usd": 0.316889, + "decisions_with_usage": 20, + "harness_latency_ms": 36829.4, + "input_tokens": 203590, + "max_output_tokens_per_call": 416, + "model": "google/gemini-3.5-flash", + "output_tokens": 1278, + "protocol_repair_attempts": 6, + "protocol_repairs_succeeded": 6, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 204868, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 260 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 222.108, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 222.108, + "usage": { + "api_calls": 23, + "api_latency_ms": 30323.5, + "cached_input_tokens": 4042, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.266824, + "decisions_with_usage": 20, + "harness_latency_ms": 32314.3, + "input_tokens": 171097, + "max_output_tokens_per_call": 474, + "model": "google/gemini-3.5-flash", + "output_tokens": 1737, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 172834, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 281 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 0, + "final_score": 333.619, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 333.619, + "usage": { + "api_calls": 26, + "api_latency_ms": 33626.3, + "cached_input_tokens": 20206, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.277977, + "decisions_with_usage": 20, + "harness_latency_ms": 35662.4, + "input_tokens": 194596, + "max_output_tokens_per_call": 498, + "model": "google/gemini-3.5-flash", + "output_tokens": 1485, + "protocol_repair_attempts": 4, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 196081, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "wins": 281 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 8, + "decision_failure_rate": 0.013, + "decisions": 480, + "failed_decisions": 6, + "failed_queries": 5, + "illegal_actions": 18, + "mean_score": 215.624, + "mean_strategy_score": 217.499, + "mean_total_wins": 251.792, + "memo_writes": 3, + "query_declines": 2, + "rejected_offers": 2, + "score_stddev": 33.263, + "total_protocol_penalty": 45.0, + "usage": { + "api_calls": 553, + "api_latency_ms": 717491.3, + "cached_input_tokens": 161642, + "calls_with_finish_reason": 553, + "cost_decisions": 480, + "cost_usd": 6.320513, + "decisions_with_usage": 480, + "harness_latency_ms": 763054.6, + "input_tokens": 4180087, + "max_output_tokens_per_call": 556, + "mean_input_tokens_per_decision": 8708.5, + "mean_output_tokens_per_decision": 62.2, + "mean_tokens_per_decision": 8770.7, + "model": "google/gemini-3.5-flash", + "output_tokens": 29842, + "protocol_repair_attempts": 52, + "protocol_repairs_succeeded": 46, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4209929, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "within_seed_score_stddev": 41.754 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.013, + "candidate_decisions": 480, + "candidate_failed_decisions": 6, + "candidate_failed_queries": 5, + "candidate_illegal_actions": 18, + "candidate_mean_score": 215.624, + "candidate_mean_strategy_score": 217.499, + "candidate_mean_tokens_per_decision": 8770.7, + "candidate_memo_writes": 3, + "candidate_protocol_penalty": 45.0, + "candidate_query_declines": 2, + "candidate_rejected_offers": 2, + "candidate_usage": { + "api_calls": 553, + "api_latency_ms": 717491.3, + "cached_input_tokens": 161642, + "calls_with_finish_reason": 553, + "cost_decisions": 480, + "cost_usd": 6.320513, + "decisions_with_usage": 480, + "harness_latency_ms": 763054.6, + "input_tokens": 4180087, + "max_output_tokens_per_call": 556, + "mean_input_tokens_per_decision": 8708.5, + "mean_output_tokens_per_decision": 62.2, + "mean_tokens_per_decision": 8770.7, + "model": "google/gemini-3.5-flash", + "output_tokens": 29842, + "protocol_repair_attempts": 52, + "protocol_repairs_succeeded": 46, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4209929, + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_providers": [ + "Google AI Studio" + ] + }, + "score_lift": -58.17, + "score_lift_pct": -21.25 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -195.995, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.125, + "num_seeds": 8, + "paired_lift_ci95": [ + -89.586, + -27.162 + ], + "paired_lift_mean": -58.17, + "paired_lift_stddev": 44.135, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 179.596, + "lift": -87.173, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 228.073, + "lift": -57.357, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 218.597, + "lift": -62.846, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 187.538, + "lift": -116.612, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 161.742, + "lift": -109.176, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 249.007, + "lift": -14.315, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 259.947, + "lift": 22.143, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 240.491, + "lift": -40.027, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0234, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 195, + "rejected": 3 + }, + "draft": { + "accepted": 1507, + "rejected": 2 + }, + "information_memory": { + "accepted": 7, + "rejected": 5 + }, + "lineup": { + "accepted": 183, + "rejected": 16 + }, + "trades": { + "accepted": 571, + "rejected": 0 + } + }, + "raw_artifact_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:google/gemini-3.5-flash", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "google/gemini-3.5-flash", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Google AI Studio | google/gemini-3.5-flash-20260519", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Google AI Studio", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "google-ai-studio", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_EFFORT": "minimal", + "OPENROUTER_REASONING_ENABLED": "true", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T00:12:49+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-glm-5.2-novita.json b/results/leaderboard/openrouter-glm-5.2-novita.json new file mode 100644 index 0000000..2ed9e72 --- /dev/null +++ b/results/leaderboard/openrouter-glm-5.2-novita.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:z-ai/glm-5.2", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:z-ai/glm-5.2", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 28, + "final_score": 207.715, + "illegal_actions": 12, + "memo_writes": 26, + "protocol_penalty": 30.0, + "query_declines": 0, + "rejected_offers": 4, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 237.715, + "usage": { + "api_calls": 34, + "api_latency_ms": 289300.6, + "cached_input_tokens": 6720, + "calls_with_finish_reason": 34, + "cost_decisions": 20, + "cost_usd": 0.068599, + "decisions_with_usage": 20, + "harness_latency_ms": 292299.3, + "input_tokens": 248557, + "max_output_tokens_per_call": 423, + "model": "z-ai/glm-5.2", + "output_tokens": 8789, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 257346, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 192 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 13, + "final_score": 188.148, + "illegal_actions": 7, + "memo_writes": 23, + "protocol_penalty": 17.5, + "query_declines": 2, + "rejected_offers": 2, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 205.648, + "usage": { + "api_calls": 26, + "api_latency_ms": 231540.5, + "cached_input_tokens": 19776, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.048143, + "decisions_with_usage": 20, + "harness_latency_ms": 233847.1, + "input_tokens": 184568, + "max_output_tokens_per_call": 422, + "model": "z-ai/glm-5.2", + "output_tokens": 6849, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 191417, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 191 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 22, + "final_score": 200.864, + "illegal_actions": 10, + "memo_writes": 28, + "protocol_penalty": 25.0, + "query_declines": 1, + "rejected_offers": 8, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 225.864, + "usage": { + "api_calls": 35, + "api_latency_ms": 322441.2, + "cached_input_tokens": 33216, + "calls_with_finish_reason": 35, + "cost_decisions": 20, + "cost_usd": 0.065311, + "decisions_with_usage": 20, + "harness_latency_ms": 325364.0, + "input_tokens": 253926, + "max_output_tokens_per_call": 541, + "model": "z-ai/glm-5.2", + "output_tokens": 9819, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 263745, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 227 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 179.4, + "illegal_actions": 3, + "memo_writes": 24, + "protocol_penalty": 7.5, + "query_declines": 2, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 186.9, + "usage": { + "api_calls": 28, + "api_latency_ms": 212369.0, + "cached_input_tokens": 17536, + "calls_with_finish_reason": 28, + "cost_decisions": 20, + "cost_usd": 0.053396, + "decisions_with_usage": 20, + "harness_latency_ms": 214693.8, + "input_tokens": 201988, + "max_output_tokens_per_call": 585, + "model": "z-ai/glm-5.2", + "output_tokens": 7320, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 209308, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 278 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 158.332, + "illegal_actions": 3, + "memo_writes": 18, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 165.832, + "usage": { + "api_calls": 23, + "api_latency_ms": 162079.5, + "cached_input_tokens": 20352, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.040958, + "decisions_with_usage": 20, + "harness_latency_ms": 164023.9, + "input_tokens": 161625, + "max_output_tokens_per_call": 388, + "model": "z-ai/glm-5.2", + "output_tokens": 5326, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166951, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 294.287, + "illegal_actions": 3, + "memo_writes": 28, + "protocol_penalty": 7.5, + "query_declines": 4, + "rejected_offers": 5, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 301.787, + "usage": { + "api_calls": 30, + "api_latency_ms": 278724.4, + "cached_input_tokens": 27584, + "calls_with_finish_reason": 30, + "cost_decisions": 20, + "cost_usd": 0.058191, + "decisions_with_usage": 20, + "harness_latency_ms": 281287.9, + "input_tokens": 223253, + "max_output_tokens_per_call": 461, + "model": "z-ai/glm-5.2", + "output_tokens": 9397, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 232650, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 321 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 11, + "final_score": 190.344, + "illegal_actions": 2, + "memo_writes": 24, + "protocol_penalty": 5.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 195.344, + "usage": { + "api_calls": 28, + "api_latency_ms": 263721.8, + "cached_input_tokens": 16576, + "calls_with_finish_reason": 28, + "cost_decisions": 20, + "cost_usd": 0.053555, + "decisions_with_usage": 20, + "harness_latency_ms": 266054.2, + "input_tokens": 200699, + "max_output_tokens_per_call": 504, + "model": "z-ai/glm-5.2", + "output_tokens": 8054, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 208753, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 233 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 261.198, + "illegal_actions": 16, + "memo_writes": 25, + "protocol_penalty": 40.0, + "query_declines": 7, + "rejected_offers": 6, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 301.198, + "usage": { + "api_calls": 27, + "api_latency_ms": 274276.7, + "cached_input_tokens": 24512, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.052579, + "decisions_with_usage": 20, + "harness_latency_ms": 276596.8, + "input_tokens": 202545, + "max_output_tokens_per_call": 591, + "model": "z-ai/glm-5.2", + "output_tokens": 8290, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 210835, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 215 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 2, + "final_score": 197.199, + "illegal_actions": 8, + "memo_writes": 22, + "protocol_penalty": 20.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 217.199, + "usage": { + "api_calls": 27, + "api_latency_ms": 247849.3, + "cached_input_tokens": 25280, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.050519, + "decisions_with_usage": 20, + "harness_latency_ms": 250198.3, + "input_tokens": 198872, + "max_output_tokens_per_call": 453, + "model": "z-ai/glm-5.2", + "output_tokens": 7062, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 205934, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 216 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 148.921, + "illegal_actions": 5, + "memo_writes": 26, + "protocol_penalty": 12.5, + "query_declines": 5, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 161.421, + "usage": { + "api_calls": 32, + "api_latency_ms": 271182.0, + "cached_input_tokens": 22464, + "calls_with_finish_reason": 32, + "cost_decisions": 20, + "cost_usd": 0.060832, + "decisions_with_usage": 20, + "harness_latency_ms": 273923.1, + "input_tokens": 233737, + "max_output_tokens_per_call": 419, + "model": "z-ai/glm-5.2", + "output_tokens": 8259, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 241996, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 275 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 7, + "final_score": 237.752, + "illegal_actions": 4, + "memo_writes": 22, + "protocol_penalty": 10.0, + "query_declines": 2, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 247.752, + "usage": { + "api_calls": 26, + "api_latency_ms": 230620.3, + "cached_input_tokens": 29376, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.046497, + "decisions_with_usage": 20, + "harness_latency_ms": 232865.3, + "input_tokens": 185117, + "max_output_tokens_per_call": 493, + "model": "z-ai/glm-5.2", + "output_tokens": 7418, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 192535, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 281 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 5, + "final_score": 320.502, + "illegal_actions": 4, + "memo_writes": 21, + "protocol_penalty": 10.0, + "query_declines": 6, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 330.502, + "usage": { + "api_calls": 28, + "api_latency_ms": 247121.6, + "cached_input_tokens": 26112, + "calls_with_finish_reason": 28, + "cost_decisions": 20, + "cost_usd": 0.052431, + "decisions_with_usage": 20, + "harness_latency_ms": 249472.4, + "input_tokens": 203118, + "max_output_tokens_per_call": 507, + "model": "z-ai/glm-5.2", + "output_tokens": 8364, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 211482, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 301 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 141.521, + "illegal_actions": 5, + "memo_writes": 22, + "protocol_penalty": 12.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 154.021, + "usage": { + "api_calls": 27, + "api_latency_ms": 225837.3, + "cached_input_tokens": 21632, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.051324, + "decisions_with_usage": 20, + "harness_latency_ms": 228169.6, + "input_tokens": 199043, + "max_output_tokens_per_call": 459, + "model": "z-ai/glm-5.2", + "output_tokens": 7217, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 206260, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 232 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 3, + "final_score": 291.742, + "illegal_actions": 6, + "memo_writes": 28, + "protocol_penalty": 15.0, + "query_declines": 1, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 306.742, + "usage": { + "api_calls": 32, + "api_latency_ms": 387265.1, + "cached_input_tokens": 27264, + "calls_with_finish_reason": 32, + "cost_decisions": 20, + "cost_usd": 0.062845, + "decisions_with_usage": 20, + "harness_latency_ms": 389896.8, + "input_tokens": 241172, + "max_output_tokens_per_call": 519, + "model": "z-ai/glm-5.2", + "output_tokens": 10095, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 251267, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 240 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 168.427, + "illegal_actions": 1, + "memo_writes": 22, + "protocol_penalty": 2.5, + "query_declines": 2, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 170.927, + "usage": { + "api_calls": 27, + "api_latency_ms": 253225.3, + "cached_input_tokens": 28480, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.050403, + "decisions_with_usage": 20, + "harness_latency_ms": 255556.8, + "input_tokens": 200382, + "max_output_tokens_per_call": 563, + "model": "z-ai/glm-5.2", + "output_tokens": 7617, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 207999, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 228 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 156.251, + "illegal_actions": 10, + "memo_writes": 21, + "protocol_penalty": 25.0, + "query_declines": 8, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 181.251, + "usage": { + "api_calls": 27, + "api_latency_ms": 230362.5, + "cached_input_tokens": 20992, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.049691, + "decisions_with_usage": 20, + "harness_latency_ms": 232679.5, + "input_tokens": 194190, + "max_output_tokens_per_call": 412, + "model": "z-ai/glm-5.2", + "output_tokens": 6743, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 200933, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 213 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 131.26, + "illegal_actions": 4, + "memo_writes": 20, + "protocol_penalty": 10.0, + "query_declines": 3, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 141.26, + "usage": { + "api_calls": 22, + "api_latency_ms": 190750.8, + "cached_input_tokens": 36160, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.034886, + "decisions_with_usage": 20, + "harness_latency_ms": 192606.9, + "input_tokens": 152401, + "max_output_tokens_per_call": 341, + "model": "z-ai/glm-5.2", + "output_tokens": 5171, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157572, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 177 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 34, + "final_score": 154.531, + "illegal_actions": 7, + "memo_writes": 22, + "protocol_penalty": 17.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 172.031, + "usage": { + "api_calls": 26, + "api_latency_ms": 203313.0, + "cached_input_tokens": 38848, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.044033, + "decisions_with_usage": 20, + "harness_latency_ms": 205483.0, + "input_tokens": 187261, + "max_output_tokens_per_call": 426, + "model": "z-ai/glm-5.2", + "output_tokens": 6391, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 193652, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 203 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 8, + "final_score": 244.73, + "illegal_actions": 13, + "memo_writes": 26, + "protocol_penalty": 32.5, + "query_declines": 5, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 277.23, + "usage": { + "api_calls": 27, + "api_latency_ms": 278304.6, + "cached_input_tokens": 22464, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.052688, + "decisions_with_usage": 20, + "harness_latency_ms": 305751.9, + "input_tokens": 201504, + "max_output_tokens_per_call": 497, + "model": "z-ai/glm-5.2", + "output_tokens": 8993, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 210497, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 254 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 238.94, + "illegal_actions": 7, + "memo_writes": 23, + "protocol_penalty": 17.5, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 256.44, + "usage": { + "api_calls": 26, + "api_latency_ms": 267734.5, + "cached_input_tokens": 30528, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.047229, + "decisions_with_usage": 20, + "harness_latency_ms": 269960.8, + "input_tokens": 189417, + "max_output_tokens_per_call": 465, + "model": "z-ai/glm-5.2", + "output_tokens": 8285, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 197702, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 287 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 250.249, + "illegal_actions": 10, + "memo_writes": 23, + "protocol_penalty": 25.0, + "query_declines": 4, + "rejected_offers": 4, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 275.249, + "usage": { + "api_calls": 30, + "api_latency_ms": 298286.5, + "cached_input_tokens": 31424, + "calls_with_finish_reason": 30, + "cost_decisions": 20, + "cost_usd": 0.056394, + "decisions_with_usage": 20, + "harness_latency_ms": 300823.5, + "input_tokens": 225088, + "max_output_tokens_per_call": 587, + "model": "z-ai/glm-5.2", + "output_tokens": 8933, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 234021, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 258 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 5, + "final_score": 302.903, + "illegal_actions": 16, + "memo_writes": 26, + "protocol_penalty": 40.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 342.903, + "usage": { + "api_calls": 33, + "api_latency_ms": 318539.7, + "cached_input_tokens": 25856, + "calls_with_finish_reason": 33, + "cost_decisions": 20, + "cost_usd": 0.064094, + "decisions_with_usage": 20, + "harness_latency_ms": 321329.2, + "input_tokens": 246808, + "max_output_tokens_per_call": 565, + "model": "z-ai/glm-5.2", + "output_tokens": 10464, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 257272, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 310 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 208.193, + "illegal_actions": 10, + "memo_writes": 24, + "protocol_penalty": 25.0, + "query_declines": 8, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 233.193, + "usage": { + "api_calls": 27, + "api_latency_ms": 248540.6, + "cached_input_tokens": 28736, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.05308, + "decisions_with_usage": 20, + "harness_latency_ms": 250821.2, + "input_tokens": 202541, + "max_output_tokens_per_call": 507, + "model": "z-ai/glm-5.2", + "output_tokens": 8417, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 210958, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 319 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 11, + "final_score": 347.533, + "illegal_actions": 2, + "memo_writes": 24, + "protocol_penalty": 5.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 352.533, + "usage": { + "api_calls": 31, + "api_latency_ms": 321339.9, + "cached_input_tokens": 40512, + "calls_with_finish_reason": 31, + "cost_decisions": 20, + "cost_usd": 0.063515, + "decisions_with_usage": 20, + "harness_latency_ms": 323995.7, + "input_tokens": 232316, + "max_output_tokens_per_call": 4096, + "model": "z-ai/glm-5.2", + "output_tokens": 13506, + "protocol_repair_attempts": 2, + "protocol_repairs_succeeded": 2, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 245822, + "truncated_calls": 1, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 311 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 11, + "decision_failure_rate": 0.017, + "decisions": 480, + "failed_decisions": 8, + "failed_queries": 209, + "illegal_actions": 168, + "mean_score": 217.539, + "mean_strategy_score": 235.039, + "mean_total_wins": 251.542, + "memo_writes": 568, + "query_declines": 79, + "rejected_offers": 42, + "score_stddev": 37.736, + "total_protocol_penalty": 420.0, + "usage": { + "api_calls": 679, + "api_latency_ms": 6254726.7, + "cached_input_tokens": 622400, + "calls_with_finish_reason": 679, + "cost_decisions": 480, + "cost_usd": 1.281193, + "decisions_with_usage": 480, + "harness_latency_ms": 6337701.0, + "input_tokens": 4970128, + "max_output_tokens_per_call": 4096, + "mean_input_tokens_per_decision": 10354.4, + "mean_output_tokens_per_decision": 410.0, + "mean_tokens_per_decision": 10764.4, + "model": "z-ai/glm-5.2", + "output_tokens": 196779, + "protocol_repair_attempts": 29, + "protocol_repairs_succeeded": 21, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 5166907, + "truncated_calls": 1, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "within_seed_score_stddev": 38.667 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.017, + "candidate_decisions": 480, + "candidate_failed_decisions": 8, + "candidate_failed_queries": 209, + "candidate_illegal_actions": 168, + "candidate_mean_score": 217.539, + "candidate_mean_strategy_score": 235.039, + "candidate_mean_tokens_per_decision": 10764.4, + "candidate_memo_writes": 568, + "candidate_protocol_penalty": 420.0, + "candidate_query_declines": 79, + "candidate_rejected_offers": 42, + "candidate_usage": { + "api_calls": 679, + "api_latency_ms": 6254726.7, + "cached_input_tokens": 622400, + "calls_with_finish_reason": 679, + "cost_decisions": 480, + "cost_usd": 1.281193, + "decisions_with_usage": 480, + "harness_latency_ms": 6337701.0, + "input_tokens": 4970128, + "max_output_tokens_per_call": 4096, + "mean_input_tokens_per_decision": 10354.4, + "mean_output_tokens_per_decision": 410.0, + "mean_tokens_per_decision": 10764.4, + "model": "z-ai/glm-5.2", + "output_tokens": 196779, + "protocol_repair_attempts": 29, + "protocol_repairs_succeeded": 21, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 5166907, + "truncated_calls": 1, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "score_lift": -56.255, + "score_lift_pct": -20.55 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -194.08, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.25, + "num_seeds": 8, + "paired_lift_ci95": [ + -81.974, + -30.464 + ], + "paired_lift_mean": -56.255, + "paired_lift_stddev": 39.192, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 198.909, + "lift": -67.861, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 210.673, + "lift": -74.757, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 216.247, + "lift": -65.196, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 235.725, + "lift": -68.425, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 200.563, + "lift": -70.355, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 147.347, + "lift": -115.975, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 244.64, + "lift": 6.835, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 286.21, + "lift": 5.692, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0312, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 229, + "rejected": 76 + }, + "draft": { + "accepted": 1741, + "rejected": 92 + }, + "information_memory": { + "accepted": 1042, + "rejected": 199 + }, + "lineup": { + "accepted": 195, + "rejected": 76 + }, + "trades": { + "accepted": 595, + "rejected": 55 + } + }, + "raw_artifact_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:z-ai/glm-5.2", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "z-ai/glm-5.2", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Novita | z-ai/glm-5.2-20260616", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Novita", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "novita/fp8", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T05:54:03+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-gpt-5.6-luna-openai.json b/results/leaderboard/openrouter-gpt-5.6-luna-openai.json new file mode 100644 index 0000000..23e189f --- /dev/null +++ b/results/leaderboard/openrouter-gpt-5.6-luna-openai.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:openai/gpt-5.6-luna", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:openai/gpt-5.6-luna", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 158.324, + "illegal_actions": 11, + "memo_writes": 14, + "protocol_penalty": 27.5, + "query_declines": 12, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 185.824, + "usage": { + "api_calls": 23, + "api_latency_ms": 27475.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.210005, + "decisions_with_usage": 20, + "harness_latency_ms": 29377.8, + "input_tokens": 158116, + "max_output_tokens_per_call": 213, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2063, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 160179, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 195 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 145.751, + "illegal_actions": 12, + "memo_writes": 14, + "protocol_penalty": 30.0, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 175.751, + "usage": { + "api_calls": 22, + "api_latency_ms": 33786.9, + "cached_input_tokens": 7053, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.193085, + "decisions_with_usage": 20, + "harness_latency_ms": 36777.3, + "input_tokens": 151119, + "max_output_tokens_per_call": 211, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2052, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153171, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 196 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 107.406, + "illegal_actions": 9, + "memo_writes": 10, + "protocol_penalty": 22.5, + "query_declines": 9, + "rejected_offers": 2, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 129.906, + "usage": { + "api_calls": 21, + "api_latency_ms": 26000.7, + "cached_input_tokens": 7053, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.180196, + "decisions_with_usage": 20, + "harness_latency_ms": 27865.3, + "input_tokens": 142338, + "max_output_tokens_per_call": 159, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1733, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 144071, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 189 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 219.36, + "illegal_actions": 3, + "memo_writes": 14, + "protocol_penalty": 7.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 226.86, + "usage": { + "api_calls": 23, + "api_latency_ms": 28883.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.215455, + "decisions_with_usage": 20, + "harness_latency_ms": 31009.0, + "input_tokens": 162457, + "max_output_tokens_per_call": 152, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2067, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 164524, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 318 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 257.543, + "illegal_actions": 5, + "memo_writes": 12, + "protocol_penalty": 12.5, + "query_declines": 12, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 270.043, + "usage": { + "api_calls": 21, + "api_latency_ms": 27543.7, + "cached_input_tokens": 7062, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.186561, + "decisions_with_usage": 20, + "harness_latency_ms": 29413.0, + "input_tokens": 145773, + "max_output_tokens_per_call": 157, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2080, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147853, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 302 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 201.756, + "illegal_actions": 9, + "memo_writes": 14, + "protocol_penalty": 22.5, + "query_declines": 8, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 224.256, + "usage": { + "api_calls": 22, + "api_latency_ms": 31522.5, + "cached_input_tokens": 7062, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.198075, + "decisions_with_usage": 20, + "harness_latency_ms": 33680.8, + "input_tokens": 153498, + "max_output_tokens_per_call": 222, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2390, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 155888, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 299 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 258.893, + "illegal_actions": 5, + "memo_writes": 16, + "protocol_penalty": 12.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 271.393, + "usage": { + "api_calls": 22, + "api_latency_ms": 26930.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.212612, + "decisions_with_usage": 20, + "harness_latency_ms": 28829.8, + "input_tokens": 158982, + "max_output_tokens_per_call": 216, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2317, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 161299, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 123.144, + "illegal_actions": 4, + "memo_writes": 13, + "protocol_penalty": 10.0, + "query_declines": 5, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 133.144, + "usage": { + "api_calls": 21, + "api_latency_ms": 26484.8, + "cached_input_tokens": 7060, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.190113, + "decisions_with_usage": 20, + "harness_latency_ms": 28298.4, + "input_tokens": 149393, + "max_output_tokens_per_call": 197, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1918, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 151311, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 185 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 122.044, + "illegal_actions": 3, + "memo_writes": 16, + "protocol_penalty": 7.5, + "query_declines": 7, + "rejected_offers": 2, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 129.544, + "usage": { + "api_calls": 23, + "api_latency_ms": 27032.8, + "cached_input_tokens": 7060, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.209086, + "decisions_with_usage": 20, + "harness_latency_ms": 29001.3, + "input_tokens": 163324, + "max_output_tokens_per_call": 151, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2178, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 165502, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 183.676, + "illegal_actions": 2, + "memo_writes": 17, + "protocol_penalty": 5.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 188.676, + "usage": { + "api_calls": 21, + "api_latency_ms": 26033.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.197472, + "decisions_with_usage": 20, + "harness_latency_ms": 27747.8, + "input_tokens": 147492, + "max_output_tokens_per_call": 152, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2187, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 149679, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 327 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 190.585, + "illegal_actions": 5, + "memo_writes": 17, + "protocol_penalty": 12.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 203.085, + "usage": { + "api_calls": 23, + "api_latency_ms": 33874.9, + "cached_input_tokens": 7058, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.20779, + "decisions_with_usage": 20, + "harness_latency_ms": 35709.5, + "input_tokens": 160676, + "max_output_tokens_per_call": 153, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2513, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 163189, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 209.048, + "illegal_actions": 4, + "memo_writes": 15, + "protocol_penalty": 10.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 219.048, + "usage": { + "api_calls": 22, + "api_latency_ms": 32213.7, + "cached_input_tokens": 7058, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.193392, + "decisions_with_usage": 20, + "harness_latency_ms": 34068.7, + "input_tokens": 152013, + "max_output_tokens_per_call": 159, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1918, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153931, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 271 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.905, + "illegal_actions": 2, + "memo_writes": 14, + "protocol_penalty": 5.0, + "query_declines": 7, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 144.905, + "usage": { + "api_calls": 22, + "api_latency_ms": 26719.1, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.207064, + "decisions_with_usage": 20, + "harness_latency_ms": 28531.6, + "input_tokens": 155756, + "max_output_tokens_per_call": 168, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2064, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157820, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 210 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 185.076, + "illegal_actions": 4, + "memo_writes": 14, + "protocol_penalty": 10.0, + "query_declines": 7, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 195.076, + "usage": { + "api_calls": 23, + "api_latency_ms": 31921.7, + "cached_input_tokens": 7054, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.210377, + "decisions_with_usage": 20, + "harness_latency_ms": 33832.7, + "input_tokens": 163972, + "max_output_tokens_per_call": 172, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2257, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166229, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 217 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 142.191, + "illegal_actions": 2, + "memo_writes": 13, + "protocol_penalty": 5.0, + "query_declines": 5, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 147.191, + "usage": { + "api_calls": 20, + "api_latency_ms": 25925.1, + "cached_input_tokens": 7054, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.178259, + "decisions_with_usage": 20, + "harness_latency_ms": 27569.9, + "input_tokens": 140292, + "max_output_tokens_per_call": 156, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1837, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 142129, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 197 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 197.317, + "illegal_actions": 1, + "memo_writes": 14, + "protocol_penalty": 2.5, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 199.817, + "usage": { + "api_calls": 22, + "api_latency_ms": 36864.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.198893, + "decisions_with_usage": 20, + "harness_latency_ms": 38715.6, + "input_tokens": 150935, + "max_output_tokens_per_call": 145, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1707, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 152642, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 235 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.834, + "illegal_actions": 5, + "memo_writes": 13, + "protocol_penalty": 12.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 142.334, + "usage": { + "api_calls": 21, + "api_latency_ms": 26600.5, + "cached_input_tokens": 7061, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.180452, + "decisions_with_usage": 20, + "harness_latency_ms": 28322.8, + "input_tokens": 142787, + "max_output_tokens_per_call": 146, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1684, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 144471, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 215 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 176.773, + "illegal_actions": 5, + "memo_writes": 12, + "protocol_penalty": 12.5, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 189.273, + "usage": { + "api_calls": 21, + "api_latency_ms": 25405.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.188953, + "decisions_with_usage": 20, + "harness_latency_ms": 27172.1, + "input_tokens": 142751, + "max_output_tokens_per_call": 212, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1755, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 144506, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 183 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 194.279, + "illegal_actions": 6, + "memo_writes": 14, + "protocol_penalty": 15.0, + "query_declines": 5, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 209.279, + "usage": { + "api_calls": 22, + "api_latency_ms": 28723.9, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.200928, + "decisions_with_usage": 20, + "harness_latency_ms": 30526.9, + "input_tokens": 151507, + "max_output_tokens_per_call": 147, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1927, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153434, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 234 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.157, + "illegal_actions": 7, + "memo_writes": 10, + "protocol_penalty": 17.5, + "query_declines": 9, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 131.657, + "usage": { + "api_calls": 21, + "api_latency_ms": 28288.5, + "cached_input_tokens": 7061, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.184565, + "decisions_with_usage": 20, + "harness_latency_ms": 30051.6, + "input_tokens": 145385, + "max_output_tokens_per_call": 206, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1828, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147213, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 231 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 197.883, + "illegal_actions": 6, + "memo_writes": 13, + "protocol_penalty": 15.0, + "query_declines": 2, + "rejected_offers": 3, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 212.883, + "usage": { + "api_calls": 21, + "api_latency_ms": 25629.8, + "cached_input_tokens": 7061, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.185192, + "decisions_with_usage": 20, + "harness_latency_ms": 27469.0, + "input_tokens": 145769, + "max_output_tokens_per_call": 153, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1853, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147622, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 248 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 283.895, + "illegal_actions": 10, + "memo_writes": 15, + "protocol_penalty": 25.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 308.895, + "usage": { + "api_calls": 22, + "api_latency_ms": 36480.7, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.205222, + "decisions_with_usage": 20, + "harness_latency_ms": 38370.9, + "input_tokens": 152679, + "max_output_tokens_per_call": 184, + "model": "openai/gpt-5.6-luna", + "output_tokens": 2398, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 155077, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 271 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 110.118, + "illegal_actions": 3, + "memo_writes": 11, + "protocol_penalty": 7.5, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 117.618, + "usage": { + "api_calls": 20, + "api_latency_ms": 31043.3, + "cached_input_tokens": 7060, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.169225, + "decisions_with_usage": 20, + "harness_latency_ms": 32761.4, + "input_tokens": 134611, + "max_output_tokens_per_call": 153, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1516, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 136127, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 245 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 125.261, + "illegal_actions": 3, + "memo_writes": 11, + "protocol_penalty": 7.5, + "query_declines": 9, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 132.761, + "usage": { + "api_calls": 20, + "api_latency_ms": 24051.8, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.177203, + "decisions_with_usage": 20, + "harness_latency_ms": 25782.4, + "input_tokens": 133887, + "max_output_tokens_per_call": 169, + "model": "openai/gpt-5.6-luna", + "output_tokens": 1643, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 135530, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "wins": 248 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 5, + "decision_failure_rate": 0.0, + "decisions": 480, + "failed_decisions": 0, + "failed_queries": 5, + "illegal_actions": 126, + "mean_score": 173.926, + "mean_strategy_score": 187.051, + "mean_total_wins": 237.542, + "memo_writes": 326, + "query_declines": 133, + "rejected_offers": 9, + "score_stddev": 24.839, + "total_protocol_penalty": 315.0, + "usage": { + "api_calls": 519, + "api_latency_ms": 695438.0, + "cached_input_tokens": 98817, + "calls_with_finish_reason": 519, + "cost_decisions": 480, + "cost_usd": 4.680175, + "decisions_with_usage": 480, + "harness_latency_ms": 740885.6, + "input_tokens": 3605512, + "max_output_tokens_per_call": 222, + "mean_input_tokens_per_decision": 7511.5, + "mean_output_tokens_per_decision": 99.8, + "mean_tokens_per_decision": 7611.2, + "model": "openai/gpt-5.6-luna", + "output_tokens": 47885, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3653397, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "within_seed_score_stddev": 35.771 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.0, + "candidate_decisions": 480, + "candidate_failed_decisions": 0, + "candidate_failed_queries": 5, + "candidate_illegal_actions": 126, + "candidate_mean_score": 173.926, + "candidate_mean_strategy_score": 187.051, + "candidate_mean_tokens_per_decision": 7611.2, + "candidate_memo_writes": 326, + "candidate_protocol_penalty": 315.0, + "candidate_query_declines": 133, + "candidate_rejected_offers": 9, + "candidate_usage": { + "api_calls": 519, + "api_latency_ms": 695438.0, + "cached_input_tokens": 98817, + "calls_with_finish_reason": 519, + "cost_decisions": 480, + "cost_usd": 4.680175, + "decisions_with_usage": 480, + "harness_latency_ms": 740885.6, + "input_tokens": 3605512, + "max_output_tokens_per_call": 222, + "mean_input_tokens_per_decision": 7511.5, + "mean_output_tokens_per_decision": 99.8, + "mean_tokens_per_decision": 7611.2, + "model": "openai/gpt-5.6-luna", + "output_tokens": 47885, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3653397, + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_providers": [ + "OpenAI" + ] + }, + "score_lift": -99.869, + "score_lift_pct": -36.48 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -237.693, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -114.217, + -83.091 + ], + "paired_lift_mean": -99.869, + "paired_lift_stddev": 22.587, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 137.16, + "lift": -129.609, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 226.22, + "lift": -59.21, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 168.027, + "lift": -113.416, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 194.436, + "lift": -109.714, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 155.724, + "lift": -115.194, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 167.975, + "lift": -95.348, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 168.773, + "lift": -69.031, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 173.091, + "lift": -107.427, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 94, + "rejected": 12 + }, + "draft": { + "accepted": 1764, + "rejected": 133 + }, + "information_memory": { + "accepted": 339, + "rejected": 5 + }, + "lineup": { + "accepted": 197, + "rejected": 123 + }, + "trades": { + "accepted": 583, + "rejected": 0 + } + }, + "raw_artifact_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:openai/gpt-5.6-luna", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "openai/gpt-5.6-luna", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "OpenAI | openai/gpt-5.6-luna-20260709", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "OpenAI", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "openai", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-18T23:07:58+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-minimax-m3-minimax.json b/results/leaderboard/openrouter-minimax-m3-minimax.json new file mode 100644 index 0000000..ac3d706 --- /dev/null +++ b/results/leaderboard/openrouter-minimax-m3-minimax.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:minimax/minimax-m3", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:minimax/minimax-m3", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 108.493, + "illegal_actions": 1, + "memo_writes": 5, + "protocol_penalty": 2.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 110.993, + "usage": { + "api_calls": 24, + "api_latency_ms": 59139.0, + "cached_input_tokens": 3632, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.05131, + "decisions_with_usage": 20, + "harness_latency_ms": 61040.5, + "input_tokens": 168729, + "max_output_tokens_per_call": 232, + "model": "minimax/minimax-m3", + "output_tokens": 1302, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 170031, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 160 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 90.116, + "illegal_actions": 0, + "memo_writes": 6, + "protocol_penalty": 0.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 90.116, + "usage": { + "api_calls": 26, + "api_latency_ms": 65901.1, + "cached_input_tokens": 10158, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.056062, + "decisions_with_usage": 20, + "harness_latency_ms": 68073.1, + "input_tokens": 187189, + "max_output_tokens_per_call": 321, + "model": "minimax/minimax-m3", + "output_tokens": 1952, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 189141, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 157 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 96.486, + "illegal_actions": 2, + "memo_writes": 9, + "protocol_penalty": 5.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 101.486, + "usage": { + "api_calls": 24, + "api_latency_ms": 61395.8, + "cached_input_tokens": 9930, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.052185, + "decisions_with_usage": 20, + "harness_latency_ms": 63376.7, + "input_tokens": 172266, + "max_output_tokens_per_call": 286, + "model": "minimax/minimax-m3", + "output_tokens": 2406, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 174672, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 189 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 140.631, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 140.631, + "usage": { + "api_calls": 23, + "api_latency_ms": 61933.1, + "cached_input_tokens": 4414, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.05165, + "decisions_with_usage": 20, + "harness_latency_ms": 63870.2, + "input_tokens": 167107, + "max_output_tokens_per_call": 330, + "model": "minimax/minimax-m3", + "output_tokens": 2146, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 169253, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 146.01, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 146.01, + "usage": { + "api_calls": 27, + "api_latency_ms": 65616.3, + "cached_input_tokens": 12062, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.059668, + "decisions_with_usage": 20, + "harness_latency_ms": 67852.7, + "input_tokens": 198775, + "max_output_tokens_per_call": 353, + "model": "minimax/minimax-m3", + "output_tokens": 2442, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 201217, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 276 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 147.196, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 147.196, + "usage": { + "api_calls": 22, + "api_latency_ms": 57468.2, + "cached_input_tokens": 22194, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.043821, + "decisions_with_usage": 20, + "harness_latency_ms": 59296.2, + "input_tokens": 155140, + "max_output_tokens_per_call": 267, + "model": "minimax/minimax-m3", + "output_tokens": 2172, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157312, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 280 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 130.005, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 130.005, + "usage": { + "api_calls": 24, + "api_latency_ms": 69464.3, + "cached_input_tokens": 5168, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.056693, + "decisions_with_usage": 20, + "harness_latency_ms": 71440.9, + "input_tokens": 181768, + "max_output_tokens_per_call": 665, + "model": "minimax/minimax-m3", + "output_tokens": 2838, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 184606, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 180 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 74.368, + "illegal_actions": 0, + "memo_writes": 8, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 74.368, + "usage": { + "api_calls": 23, + "api_latency_ms": 60922.0, + "cached_input_tokens": 12751, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.048982, + "decisions_with_usage": 20, + "harness_latency_ms": 62849.8, + "input_tokens": 165357, + "max_output_tokens_per_call": 305, + "model": "minimax/minimax-m3", + "output_tokens": 2027, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167384, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 170 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 118.459, + "illegal_actions": 0, + "memo_writes": 8, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 118.459, + "usage": { + "api_calls": 21, + "api_latency_ms": 55278.1, + "cached_input_tokens": 17131, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.044489, + "decisions_with_usage": 20, + "harness_latency_ms": 57003.4, + "input_tokens": 152475, + "max_output_tokens_per_call": 361, + "model": "minimax/minimax-m3", + "output_tokens": 2382, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 154857, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 172 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 124.769, + "illegal_actions": 0, + "memo_writes": 9, + "protocol_penalty": 0.0, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 124.769, + "usage": { + "api_calls": 25, + "api_latency_ms": 73012.0, + "cached_input_tokens": 5410, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.057521, + "decisions_with_usage": 20, + "harness_latency_ms": 75087.0, + "input_tokens": 182586, + "max_output_tokens_per_call": 587, + "model": "minimax/minimax-m3", + "output_tokens": 3371, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 185957, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 258 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 124.383, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 7, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 124.383, + "usage": { + "api_calls": 25, + "api_latency_ms": 53012.6, + "cached_input_tokens": 23606, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.048489, + "decisions_with_usage": 20, + "harness_latency_ms": 55092.8, + "input_tokens": 172753, + "max_output_tokens_per_call": 340, + "model": "minimax/minimax-m3", + "output_tokens": 1943, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 174696, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 259 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 112.1, + "illegal_actions": 1, + "memo_writes": 8, + "protocol_penalty": 2.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 114.6, + "usage": { + "api_calls": 23, + "api_latency_ms": 63939.9, + "cached_input_tokens": 42627, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.042187, + "decisions_with_usage": 20, + "harness_latency_ms": 65883.7, + "input_tokens": 162775, + "max_output_tokens_per_call": 419, + "model": "minimax/minimax-m3", + "output_tokens": 2988, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 165763, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 258 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 115.172, + "illegal_actions": 0, + "memo_writes": 4, + "protocol_penalty": 0.0, + "query_declines": 10, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 115.172, + "usage": { + "api_calls": 23, + "api_latency_ms": 57621.4, + "cached_input_tokens": 6078, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.050388, + "decisions_with_usage": 20, + "harness_latency_ms": 59565.7, + "input_tokens": 166218, + "max_output_tokens_per_call": 277, + "model": "minimax/minimax-m3", + "output_tokens": 1651, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167869, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 195 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 294.617, + "illegal_actions": 0, + "memo_writes": 8, + "protocol_penalty": 0.0, + "query_declines": 4, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 294.617, + "usage": { + "api_calls": 24, + "api_latency_ms": 63575.7, + "cached_input_tokens": 12488, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.05437, + "decisions_with_usage": 20, + "harness_latency_ms": 65582.5, + "input_tokens": 180481, + "max_output_tokens_per_call": 338, + "model": "minimax/minimax-m3", + "output_tokens": 2686, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 183167, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 210 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 123.482, + "illegal_actions": 0, + "memo_writes": 8, + "protocol_penalty": 0.0, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 123.482, + "usage": { + "api_calls": 25, + "api_latency_ms": 63094.8, + "cached_input_tokens": 15418, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.0559, + "decisions_with_usage": 20, + "harness_latency_ms": 65203.0, + "input_tokens": 187055, + "max_output_tokens_per_call": 489, + "model": "minimax/minimax-m3", + "output_tokens": 2902, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 189957, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 197 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 132.083, + "illegal_actions": 0, + "memo_writes": 8, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 132.083, + "usage": { + "api_calls": 23, + "api_latency_ms": 60885.5, + "cached_input_tokens": 4542, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.051678, + "decisions_with_usage": 20, + "harness_latency_ms": 62806.0, + "input_tokens": 165332, + "max_output_tokens_per_call": 325, + "model": "minimax/minimax-m3", + "output_tokens": 2641, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167973, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 185 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 90.441, + "illegal_actions": 9, + "memo_writes": 12, + "protocol_penalty": 22.5, + "query_declines": 3, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 112.941, + "usage": { + "api_calls": 25, + "api_latency_ms": 71295.2, + "cached_input_tokens": 10046, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.056214, + "decisions_with_usage": 20, + "harness_latency_ms": 73392.8, + "input_tokens": 182210, + "max_output_tokens_per_call": 377, + "model": "minimax/minimax-m3", + "output_tokens": 3301, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 185511, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 76.888, + "illegal_actions": 0, + "memo_writes": 9, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 76.888, + "usage": { + "api_calls": 25, + "api_latency_ms": 59136.4, + "cached_input_tokens": 15550, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.05237, + "decisions_with_usage": 20, + "harness_latency_ms": 61261.4, + "input_tokens": 178394, + "max_output_tokens_per_call": 302, + "model": "minimax/minimax-m3", + "output_tokens": 2154, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 180548, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 170 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 163.673, + "illegal_actions": 0, + "memo_writes": 10, + "protocol_penalty": 0.0, + "query_declines": 5, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 163.673, + "usage": { + "api_calls": 28, + "api_latency_ms": 81804.6, + "cached_input_tokens": 3192, + "calls_with_finish_reason": 28, + "cost_decisions": 20, + "cost_usd": 0.063483, + "decisions_with_usage": 20, + "harness_latency_ms": 84217.9, + "input_tokens": 203700, + "max_output_tokens_per_call": 367, + "model": "minimax/minimax-m3", + "output_tokens": 2616, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 206316, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 223 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 162.382, + "illegal_actions": 1, + "memo_writes": 5, + "protocol_penalty": 2.5, + "query_declines": 3, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 164.882, + "usage": { + "api_calls": 21, + "api_latency_ms": 44543.4, + "cached_input_tokens": 12147, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.04464, + "decisions_with_usage": 20, + "harness_latency_ms": 46310.8, + "input_tokens": 151404, + "max_output_tokens_per_call": 331, + "model": "minimax/minimax-m3", + "output_tokens": 1776, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153180, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 230 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 151.537, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 9, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 151.537, + "usage": { + "api_calls": 22, + "api_latency_ms": 56974.9, + "cached_input_tokens": 13285, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.045626, + "decisions_with_usage": 20, + "harness_latency_ms": 58789.9, + "input_tokens": 155394, + "max_output_tokens_per_call": 293, + "model": "minimax/minimax-m3", + "output_tokens": 1829, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157223, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 226 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 1, + "failed_queries": 2, + "final_score": 125.568, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 125.568, + "usage": { + "api_calls": 24, + "api_latency_ms": 53397.9, + "cached_input_tokens": 3632, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.051007, + "decisions_with_usage": 20, + "harness_latency_ms": 55426.7, + "input_tokens": 166083, + "max_output_tokens_per_call": 251, + "model": "minimax/minimax-m3", + "output_tokens": 1711, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167794, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 250 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 126.905, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 126.905, + "usage": { + "api_calls": 24, + "api_latency_ms": 54637.2, + "cached_input_tokens": 13258, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.051083, + "decisions_with_usage": 20, + "harness_latency_ms": 56584.8, + "input_tokens": 171531, + "max_output_tokens_per_call": 301, + "model": "minimax/minimax-m3", + "output_tokens": 2338, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 173869, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 264 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 141.356, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 6, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 141.356, + "usage": { + "api_calls": 23, + "api_latency_ms": 62223.0, + "cached_input_tokens": 10584, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.047333, + "decisions_with_usage": 20, + "harness_latency_ms": 64177.5, + "input_tokens": 159566, + "max_output_tokens_per_call": 378, + "model": "minimax/minimax-m3", + "output_tokens": 1671, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 161237, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "wins": 256 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 3, + "decision_failure_rate": 0.002, + "decisions": 480, + "failed_decisions": 1, + "failed_queries": 65, + "illegal_actions": 14, + "mean_score": 129.88, + "mean_strategy_score": 131.338, + "mean_total_wins": 217.083, + "memo_writes": 174, + "query_declines": 107, + "rejected_offers": 1, + "score_stddev": 27.051, + "total_protocol_penalty": 35.0, + "usage": { + "api_calls": 574, + "api_latency_ms": 1476272.4, + "cached_input_tokens": 289303, + "calls_with_finish_reason": 574, + "cost_decisions": 480, + "cost_usd": 1.237149, + "decisions_with_usage": 480, + "harness_latency_ms": 1524186.0, + "input_tokens": 4134288, + "max_output_tokens_per_call": 665, + "mean_input_tokens_per_decision": 8613.1, + "mean_output_tokens_per_decision": 115.1, + "mean_tokens_per_decision": 8728.2, + "model": "minimax/minimax-m3", + "output_tokens": 55245, + "protocol_repair_attempts": 7, + "protocol_repairs_succeeded": 6, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4189533, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "within_seed_score_stddev": 19.889 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.002, + "candidate_decisions": 480, + "candidate_failed_decisions": 1, + "candidate_failed_queries": 65, + "candidate_illegal_actions": 14, + "candidate_mean_score": 129.88, + "candidate_mean_strategy_score": 131.338, + "candidate_mean_tokens_per_decision": 8728.2, + "candidate_memo_writes": 174, + "candidate_protocol_penalty": 35.0, + "candidate_query_declines": 107, + "candidate_rejected_offers": 1, + "candidate_usage": { + "api_calls": 574, + "api_latency_ms": 1476272.4, + "cached_input_tokens": 289303, + "calls_with_finish_reason": 574, + "cost_decisions": 480, + "cost_usd": 1.237149, + "decisions_with_usage": 480, + "harness_latency_ms": 1524186.0, + "input_tokens": 4134288, + "max_output_tokens_per_call": 665, + "mean_input_tokens_per_decision": 8613.1, + "mean_output_tokens_per_decision": 115.1, + "mean_tokens_per_decision": 8728.2, + "model": "minimax/minimax-m3", + "output_tokens": 55245, + "protocol_repair_attempts": 7, + "protocol_repairs_succeeded": 6, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 4189533, + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_providers": [ + "Minimax" + ] + }, + "score_lift": -143.914, + "score_lift_pct": -52.56 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -281.739, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -167.324, + -116.177 + ], + "paired_lift_mean": -143.914, + "paired_lift_stddev": 35.958, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 98.365, + "lift": -168.404, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 144.612, + "lift": -140.817, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 107.611, + "lift": -173.832, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 120.417, + "lift": -183.733, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 177.757, + "lift": -93.161, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 99.804, + "lift": -163.518, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 159.197, + "lift": -78.607, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 131.276, + "lift": -149.242, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 54, + "rejected": 9 + }, + "draft": { + "accepted": 1718, + "rejected": 107 + }, + "information_memory": { + "accepted": 224, + "rejected": 65 + }, + "lineup": { + "accepted": 86, + "rejected": 6 + }, + "trades": { + "accepted": 551, + "rejected": 0 + } + }, + "raw_artifact_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:minimax/minimax-m3", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "minimax/minimax-m3", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Minimax | minimax/minimax-m3-20260531", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Minimax", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "minimax/fp8", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T06:19:33+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-muse-spark-1.1-meta.json b/results/leaderboard/openrouter-muse-spark-1.1-meta.json new file mode 100644 index 0000000..1db566b --- /dev/null +++ b/results/leaderboard/openrouter-muse-spark-1.1-meta.json @@ -0,0 +1,4150 @@ +{ + "agent": "openrouter:meta/muse-spark-1.1", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:meta/muse-spark-1.1", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 39, + "final_score": 249.831, + "illegal_actions": 3, + "memo_writes": 7, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 257.331, + "usage": { + "api_calls": 23, + "api_latency_ms": 268866.5, + "cached_input_tokens": 64, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.297091, + "decisions_with_usage": 20, + "harness_latency_ms": 270754.9, + "input_tokens": 164609, + "max_output_tokens_per_call": 2374, + "model": "meta/muse-spark-1.1", + "output_tokens": 21506, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 18510, + "total_tokens": 186115, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 202 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 18, + "final_score": 252.667, + "illegal_actions": 3, + "memo_writes": 6, + "protocol_penalty": 7.5, + "query_declines": 1, + "rejected_offers": 5, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 260.167, + "usage": { + "api_calls": 24, + "api_latency_ms": 283062.1, + "cached_input_tokens": 512, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.316482, + "decisions_with_usage": 20, + "harness_latency_ms": 285122.1, + "input_tokens": 173321, + "max_output_tokens_per_call": 2545, + "model": "meta/muse-spark-1.1", + "output_tokens": 23622, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 20894, + "total_tokens": 196943, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 165 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 17, + "final_score": 242.984, + "illegal_actions": 0, + "memo_writes": 11, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 242.984, + "usage": { + "api_calls": 26, + "api_latency_ms": 330422.5, + "cached_input_tokens": 1728, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.354429, + "decisions_with_usage": 20, + "harness_latency_ms": 332807.0, + "input_tokens": 194348, + "max_output_tokens_per_call": 2095, + "model": "meta/muse-spark-1.1", + "output_tokens": 26681, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 23074, + "total_tokens": 221029, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 11, + "final_score": 256.831, + "illegal_actions": 1, + "memo_writes": 13, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 259.331, + "usage": { + "api_calls": 26, + "api_latency_ms": 309902.8, + "cached_input_tokens": 2560, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.355848, + "decisions_with_usage": 20, + "harness_latency_ms": 312113.8, + "input_tokens": 195614, + "max_output_tokens_per_call": 1893, + "model": "meta/muse-spark-1.1", + "output_tokens": 26858, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 22639, + "total_tokens": 222472, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 287 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 324.359, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 324.359, + "usage": { + "api_calls": 23, + "api_latency_ms": 256647.1, + "cached_input_tokens": 3456, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.308758, + "decisions_with_usage": 20, + "harness_latency_ms": 258569.5, + "input_tokens": 173224, + "max_output_tokens_per_call": 1752, + "model": "meta/muse-spark-1.1", + "output_tokens": 22595, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 19398, + "total_tokens": 195819, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 322 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 336.022, + "illegal_actions": 1, + "memo_writes": 10, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 338.522, + "usage": { + "api_calls": 21, + "api_latency_ms": 213557.6, + "cached_input_tokens": 7936, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.265104, + "decisions_with_usage": 20, + "harness_latency_ms": 215415.1, + "input_tokens": 155128, + "max_output_tokens_per_call": 1718, + "model": "meta/muse-spark-1.1", + "output_tokens": 18806, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 15603, + "total_tokens": 173934, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 300 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 29, + "final_score": 367.786, + "illegal_actions": 5, + "memo_writes": 9, + "protocol_penalty": 12.5, + "query_declines": 0, + "rejected_offers": 3, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 380.286, + "usage": { + "api_calls": 32, + "api_latency_ms": 322059.2, + "cached_input_tokens": 5632, + "calls_with_finish_reason": 32, + "cost_decisions": 20, + "cost_usd": 0.407756, + "decisions_with_usage": 20, + "harness_latency_ms": 324714.4, + "input_tokens": 238520, + "max_output_tokens_per_call": 1953, + "model": "meta/muse-spark-1.1", + "output_tokens": 27247, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 22676, + "total_tokens": 265767, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 245 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 288.813, + "illegal_actions": 4, + "memo_writes": 12, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 298.813, + "usage": { + "api_calls": 24, + "api_latency_ms": 257251.9, + "cached_input_tokens": 3072, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.317872, + "decisions_with_usage": 20, + "harness_latency_ms": 259305.3, + "input_tokens": 181709, + "max_output_tokens_per_call": 2152, + "model": "meta/muse-spark-1.1", + "output_tokens": 22145, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 18440, + "total_tokens": 203854, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 193 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 17, + "final_score": 167.221, + "illegal_actions": 8, + "memo_writes": 10, + "protocol_penalty": 20.0, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 187.221, + "usage": { + "api_calls": 22, + "api_latency_ms": 259534.4, + "cached_input_tokens": 3648, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.291877, + "decisions_with_usage": 20, + "harness_latency_ms": 261473.2, + "input_tokens": 159096, + "max_output_tokens_per_call": 2108, + "model": "meta/muse-spark-1.1", + "output_tokens": 22828, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 19429, + "total_tokens": 181924, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 174 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 22, + "final_score": 133.534, + "illegal_actions": 2, + "memo_writes": 7, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 138.534, + "usage": { + "api_calls": 25, + "api_latency_ms": 266219.5, + "cached_input_tokens": 2816, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.308935, + "decisions_with_usage": 20, + "harness_latency_ms": 268792.6, + "input_tokens": 183679, + "max_output_tokens_per_call": 1587, + "model": "meta/muse-spark-1.1", + "output_tokens": 19396, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 16124, + "total_tokens": 203075, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 246 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 32, + "final_score": 166.611, + "illegal_actions": 1, + "memo_writes": 7, + "protocol_penalty": 2.5, + "query_declines": 2, + "rejected_offers": 4, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 169.111, + "usage": { + "api_calls": 22, + "api_latency_ms": 244450.1, + "cached_input_tokens": 8320, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.282951, + "decisions_with_usage": 20, + "harness_latency_ms": 246308.7, + "input_tokens": 160138, + "max_output_tokens_per_call": 2596, + "model": "meta/muse-spark-1.1", + "output_tokens": 21630, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 18344, + "total_tokens": 181768, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 257 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 9, + "final_score": 366.616, + "illegal_actions": 8, + "memo_writes": 12, + "protocol_penalty": 20.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 386.616, + "usage": { + "api_calls": 24, + "api_latency_ms": 289194.3, + "cached_input_tokens": 3008, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.337695, + "decisions_with_usage": 20, + "harness_latency_ms": 291286.2, + "input_tokens": 183547, + "max_output_tokens_per_call": 1805, + "model": "meta/muse-spark-1.1", + "output_tokens": 26252, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 21784, + "total_tokens": 209799, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 316 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 13, + "final_score": 191.55, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 4, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 191.55, + "usage": { + "api_calls": 25, + "api_latency_ms": 330642.5, + "cached_input_tokens": 2368, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.34788, + "decisions_with_usage": 20, + "harness_latency_ms": 332910.0, + "input_tokens": 190261, + "max_output_tokens_per_call": 2437, + "model": "meta/muse-spark-1.1", + "output_tokens": 26508, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 22747, + "total_tokens": 216769, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 247 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 238.071, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 238.071, + "usage": { + "api_calls": 22, + "api_latency_ms": 263865.9, + "cached_input_tokens": 10816, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.295623, + "decisions_with_usage": 20, + "harness_latency_ms": 265908.4, + "input_tokens": 164893, + "max_output_tokens_per_call": 2244, + "model": "meta/muse-spark-1.1", + "output_tokens": 23860, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 20557, + "total_tokens": 188753, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 245 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 6, + "final_score": 136.238, + "illegal_actions": 0, + "memo_writes": 7, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 136.238, + "usage": { + "api_calls": 22, + "api_latency_ms": 227762.5, + "cached_input_tokens": 4480, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.287502, + "decisions_with_usage": 20, + "harness_latency_ms": 229744.8, + "input_tokens": 166927, + "max_output_tokens_per_call": 1956, + "model": "meta/muse-spark-1.1", + "output_tokens": 19711, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 16979, + "total_tokens": 186638, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 200 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 15, + "final_score": 141.514, + "illegal_actions": 3, + "memo_writes": 8, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 149.014, + "usage": { + "api_calls": 21, + "api_latency_ms": 212285.4, + "cached_input_tokens": 3584, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.258752, + "decisions_with_usage": 20, + "harness_latency_ms": 214197.8, + "input_tokens": 149533, + "max_output_tokens_per_call": 2692, + "model": "meta/muse-spark-1.1", + "output_tokens": 17830, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 15383, + "total_tokens": 167363, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 177 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 9, + "final_score": 230.345, + "illegal_actions": 1, + "memo_writes": 9, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 4, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 232.845, + "usage": { + "api_calls": 23, + "api_latency_ms": 223601.6, + "cached_input_tokens": 4480, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.292256, + "decisions_with_usage": 20, + "harness_latency_ms": 225639.9, + "input_tokens": 170554, + "max_output_tokens_per_call": 1484, + "model": "meta/muse-spark-1.1", + "output_tokens": 19762, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 16682, + "total_tokens": 190316, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 261 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 31, + "final_score": 173.387, + "illegal_actions": 4, + "memo_writes": 9, + "protocol_penalty": 10.0, + "query_declines": 3, + "rejected_offers": 4, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 183.387, + "usage": { + "api_calls": 27, + "api_latency_ms": 254906.0, + "cached_input_tokens": 7168, + "calls_with_finish_reason": 27, + "cost_decisions": 20, + "cost_usd": 0.33038, + "decisions_with_usage": 20, + "harness_latency_ms": 257322.4, + "input_tokens": 196560, + "max_output_tokens_per_call": 1860, + "model": "meta/muse-spark-1.1", + "output_tokens": 21780, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 17783, + "total_tokens": 218340, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 261 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 36, + "final_score": 150.519, + "illegal_actions": 0, + "memo_writes": 6, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 150.519, + "usage": { + "api_calls": 24, + "api_latency_ms": 239497.2, + "cached_input_tokens": 3072, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.30453, + "decisions_with_usage": 20, + "harness_latency_ms": 241672.2, + "input_tokens": 175765, + "max_output_tokens_per_call": 1382, + "model": "meta/muse-spark-1.1", + "output_tokens": 20753, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 17035, + "total_tokens": 196518, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 258 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 18, + "final_score": 253.442, + "illegal_actions": 1, + "memo_writes": 11, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 255.942, + "usage": { + "api_calls": 24, + "api_latency_ms": 300783.5, + "cached_input_tokens": 4672, + "calls_with_finish_reason": 24, + "cost_decisions": 20, + "cost_usd": 0.324755, + "decisions_with_usage": 20, + "harness_latency_ms": 302983.0, + "input_tokens": 176307, + "max_output_tokens_per_call": 2737, + "model": "meta/muse-spark-1.1", + "output_tokens": 25767, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 21637, + "total_tokens": 202074, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 284 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 11, + "final_score": 219.828, + "illegal_actions": 1, + "memo_writes": 13, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 222.328, + "usage": { + "api_calls": 23, + "api_latency_ms": 275952.5, + "cached_input_tokens": 3712, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.307618, + "decisions_with_usage": 20, + "harness_latency_ms": 278031.1, + "input_tokens": 167950, + "max_output_tokens_per_call": 1701, + "model": "meta/muse-spark-1.1", + "output_tokens": 23944, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 20292, + "total_tokens": 191894, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 253 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 24, + "final_score": 377.894, + "illegal_actions": 4, + "memo_writes": 9, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 387.894, + "usage": { + "api_calls": 25, + "api_latency_ms": 311050.2, + "cached_input_tokens": 5888, + "calls_with_finish_reason": 25, + "cost_decisions": 20, + "cost_usd": 0.34325, + "decisions_with_usage": 20, + "harness_latency_ms": 313314.4, + "input_tokens": 188641, + "max_output_tokens_per_call": 2235, + "model": "meta/muse-spark-1.1", + "output_tokens": 26806, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 22665, + "total_tokens": 215447, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 241 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 52, + "final_score": 161.332, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 161.332, + "usage": { + "api_calls": 26, + "api_latency_ms": 254023.0, + "cached_input_tokens": 7168, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.316457, + "decisions_with_usage": 20, + "harness_latency_ms": 256416.6, + "input_tokens": 188586, + "max_output_tokens_per_call": 1584, + "model": "meta/muse-spark-1.1", + "output_tokens": 20849, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 17450, + "total_tokens": 209435, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 246 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 24, + "final_score": 137.026, + "illegal_actions": 1, + "memo_writes": 12, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 139.526, + "usage": { + "api_calls": 26, + "api_latency_ms": 286093.4, + "cached_input_tokens": 7424, + "calls_with_finish_reason": 26, + "cost_decisions": 20, + "cost_usd": 0.332478, + "decisions_with_usage": 20, + "harness_latency_ms": 288468.9, + "input_tokens": 190166, + "max_output_tokens_per_call": 2378, + "model": "meta/muse-spark-1.1", + "output_tokens": 24220, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 20479, + "total_tokens": 214386, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "wins": 254 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 8, + "decision_failure_rate": 0.0, + "decisions": 480, + "failed_decisions": 0, + "failed_queries": 453, + "illegal_actions": 51, + "mean_score": 231.851, + "mean_strategy_score": 237.163, + "mean_total_wins": 242.792, + "memo_writes": 229, + "query_declines": 7, + "rejected_offers": 38, + "score_stddev": 39.764, + "total_protocol_penalty": 127.5, + "usage": { + "api_calls": 580, + "api_latency_ms": 6481631.7, + "cached_input_tokens": 107584, + "calls_with_finish_reason": 580, + "cost_decisions": 480, + "cost_usd": 7.586279, + "decisions_with_usage": 480, + "harness_latency_ms": 6533272.3, + "input_tokens": 4289076, + "max_output_tokens_per_call": 2737, + "mean_input_tokens_per_decision": 8935.6, + "mean_output_tokens_per_decision": 1148.7, + "mean_tokens_per_decision": 10084.2, + "model": "meta/muse-spark-1.1", + "output_tokens": 551356, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 466604, + "total_tokens": 4840432, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "within_seed_score_stddev": 56.741 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.0, + "candidate_decisions": 480, + "candidate_failed_decisions": 0, + "candidate_failed_queries": 453, + "candidate_illegal_actions": 51, + "candidate_mean_score": 231.851, + "candidate_mean_strategy_score": 237.163, + "candidate_mean_tokens_per_decision": 10084.2, + "candidate_memo_writes": 229, + "candidate_protocol_penalty": 127.5, + "candidate_query_declines": 7, + "candidate_rejected_offers": 38, + "candidate_usage": { + "api_calls": 580, + "api_latency_ms": 6481631.7, + "cached_input_tokens": 107584, + "calls_with_finish_reason": 580, + "cost_decisions": 480, + "cost_usd": 7.586279, + "decisions_with_usage": 480, + "harness_latency_ms": 6533272.3, + "input_tokens": 4289076, + "max_output_tokens_per_call": 2737, + "mean_input_tokens_per_decision": 8935.6, + "mean_output_tokens_per_decision": 1148.7, + "mean_tokens_per_decision": 10084.2, + "model": "meta/muse-spark-1.1", + "output_tokens": 551356, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 466604, + "total_tokens": 4840432, + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_providers": [ + "Meta" + ] + }, + "score_lift": -41.944, + "score_lift_pct": -15.32 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -179.768, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.125, + "num_seeds": 8, + "paired_lift_ci95": [ + -66.061, + -16.342 + ], + "paired_lift_mean": -41.944, + "paired_lift_stddev": 36.712, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 248.494, + "lift": -18.275, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 305.737, + "lift": 20.308, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 274.607, + "lift": -6.836, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 222.254, + "lift": -81.897, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 188.62, + "lift": -82.298, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 181.749, + "lift": -81.574, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 207.93, + "lift": -29.875, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 225.417, + "lift": -55.101, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0312, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 279, + "rejected": 30 + }, + "draft": { + "accepted": 1795, + "rejected": 12 + }, + "information_memory": { + "accepted": 543, + "rejected": 453 + }, + "lineup": { + "accepted": 219, + "rejected": 12 + }, + "trades": { + "accepted": 653, + "rejected": 42 + } + }, + "raw_artifact_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:meta/muse-spark-1.1", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "meta/muse-spark-1.1", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Meta | meta/muse-spark-1.1-20260709", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Meta", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "meta", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_EFFORT": "minimal", + "OPENROUTER_REASONING_ENABLED": "true", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T04:08:17+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-qwen3.7-plus-alibaba.json b/results/leaderboard/openrouter-qwen3.7-plus-alibaba.json new file mode 100644 index 0000000..a01720f --- /dev/null +++ b/results/leaderboard/openrouter-qwen3.7-plus-alibaba.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:qwen/qwen3.7-plus", + "baseline_cache": { + "enabled": true, + "hits": 64, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:qwen/qwen3.7-plus", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 171.418, + "illegal_actions": 12, + "memo_writes": 14, + "protocol_penalty": 30.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 201.418, + "usage": { + "api_calls": 22, + "api_latency_ms": 93386.4, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.058446, + "decisions_with_usage": 20, + "harness_latency_ms": 95190.5, + "input_tokens": 164865, + "max_output_tokens_per_call": 433, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4446, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 169311, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 138.24, + "illegal_actions": 11, + "memo_writes": 9, + "protocol_penalty": 27.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 165.74, + "usage": { + "api_calls": 21, + "api_latency_ms": 83303.2, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.053143, + "decisions_with_usage": 20, + "harness_latency_ms": 85065.5, + "input_tokens": 155750, + "max_output_tokens_per_call": 482, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3834, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 159584, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 195 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 131.811, + "illegal_actions": 14, + "memo_writes": 9, + "protocol_penalty": 35.0, + "query_declines": 1, + "rejected_offers": 1, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 166.811, + "usage": { + "api_calls": 20, + "api_latency_ms": 81857.9, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.05157, + "decisions_with_usage": 20, + "harness_latency_ms": 83565.0, + "input_tokens": 149775, + "max_output_tokens_per_call": 529, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4100, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153875, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 210 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 180.046, + "illegal_actions": 5, + "memo_writes": 7, + "protocol_penalty": 12.5, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 192.546, + "usage": { + "api_calls": 20, + "api_latency_ms": 73016.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.05279, + "decisions_with_usage": 20, + "harness_latency_ms": 74682.2, + "input_tokens": 150839, + "max_output_tokens_per_call": 444, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3534, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 154373, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 293 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 170.609, + "illegal_actions": 3, + "memo_writes": 10, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 178.109, + "usage": { + "api_calls": 22, + "api_latency_ms": 74922.8, + "cached_input_tokens": 10496, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.054558, + "decisions_with_usage": 20, + "harness_latency_ms": 76747.5, + "input_tokens": 164458, + "max_output_tokens_per_call": 354, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3606, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 168064, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 136.768, + "illegal_actions": 4, + "memo_writes": 8, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 3, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 146.768, + "usage": { + "api_calls": 22, + "api_latency_ms": 82736.3, + "cached_input_tokens": 25088, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.051147, + "decisions_with_usage": 20, + "harness_latency_ms": 84592.4, + "input_tokens": 163333, + "max_output_tokens_per_call": 418, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4142, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167475, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 280 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 166.813, + "illegal_actions": 4, + "memo_writes": 11, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 176.813, + "usage": { + "api_calls": 21, + "api_latency_ms": 92226.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.058146, + "decisions_with_usage": 20, + "harness_latency_ms": 93973.8, + "input_tokens": 163035, + "max_output_tokens_per_call": 458, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4668, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 167703, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 192 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 158.304, + "illegal_actions": 2, + "memo_writes": 11, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 163.304, + "usage": { + "api_calls": 20, + "api_latency_ms": 79846.8, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.052381, + "decisions_with_usage": 20, + "harness_latency_ms": 81567.9, + "input_tokens": 153575, + "max_output_tokens_per_call": 346, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3782, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157357, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 192 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 170.294, + "illegal_actions": 1, + "memo_writes": 7, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 172.794, + "usage": { + "api_calls": 20, + "api_latency_ms": 71825.1, + "cached_input_tokens": 8320, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.051947, + "decisions_with_usage": 20, + "harness_latency_ms": 73437.9, + "input_tokens": 154436, + "max_output_tokens_per_call": 379, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3640, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 158076, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 187 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 7, + "final_score": 272.411, + "illegal_actions": 1, + "memo_writes": 9, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 274.911, + "usage": { + "api_calls": 21, + "api_latency_ms": 75965.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.055788, + "decisions_with_usage": 20, + "harness_latency_ms": 77762.8, + "input_tokens": 158738, + "max_output_tokens_per_call": 444, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3899, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 162637, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 277 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 214.533, + "illegal_actions": 3, + "memo_writes": 10, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 222.033, + "usage": { + "api_calls": 21, + "api_latency_ms": 85010.4, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.054938, + "decisions_with_usage": 20, + "harness_latency_ms": 86735.6, + "input_tokens": 159330, + "max_output_tokens_per_call": 406, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4344, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 163674, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 154.817, + "illegal_actions": 8, + "memo_writes": 12, + "protocol_penalty": 20.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 174.817, + "usage": { + "api_calls": 21, + "api_latency_ms": 87189.8, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.053694, + "decisions_with_usage": 20, + "harness_latency_ms": 88954.1, + "input_tokens": 154800, + "max_output_tokens_per_call": 358, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4502, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 159302, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 263 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 223.649, + "illegal_actions": 3, + "memo_writes": 8, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 231.149, + "usage": { + "api_calls": 21, + "api_latency_ms": 81410.6, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.057361, + "decisions_with_usage": 20, + "harness_latency_ms": 83162.5, + "input_tokens": 162704, + "max_output_tokens_per_call": 457, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4137, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 166841, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 231 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 165.229, + "illegal_actions": 6, + "memo_writes": 8, + "protocol_penalty": 15.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 180.229, + "usage": { + "api_calls": 20, + "api_latency_ms": 75510.2, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.052448, + "decisions_with_usage": 20, + "harness_latency_ms": 77159.2, + "input_tokens": 153391, + "max_output_tokens_per_call": 354, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3882, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157273, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 229 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 137.946, + "illegal_actions": 0, + "memo_writes": 10, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 3, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 137.946, + "usage": { + "api_calls": 20, + "api_latency_ms": 75569.7, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.051161, + "decisions_with_usage": 20, + "harness_latency_ms": 77285.3, + "input_tokens": 152363, + "max_output_tokens_per_call": 356, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3134, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 155497, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 220 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 186.272, + "illegal_actions": 22, + "memo_writes": 8, + "protocol_penalty": 55.0, + "query_declines": 0, + "rejected_offers": 1, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 241.272, + "usage": { + "api_calls": 22, + "api_latency_ms": 95510.5, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.059401, + "decisions_with_usage": 20, + "harness_latency_ms": 97315.7, + "input_tokens": 167614, + "max_output_tokens_per_call": 680, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4505, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 172119, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 224 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 105.32, + "illegal_actions": 9, + "memo_writes": 8, + "protocol_penalty": 22.5, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 127.82, + "usage": { + "api_calls": 20, + "api_latency_ms": 77623.5, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.050241, + "decisions_with_usage": 20, + "harness_latency_ms": 79268.0, + "input_tokens": 148070, + "max_output_tokens_per_call": 568, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3487, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 151557, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 198 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 130.442, + "illegal_actions": 1, + "memo_writes": 6, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 132.942, + "usage": { + "api_calls": 20, + "api_latency_ms": 67989.4, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.048991, + "decisions_with_usage": 20, + "harness_latency_ms": 69618.5, + "input_tokens": 146775, + "max_output_tokens_per_call": 343, + "model": "qwen/qwen3.7-plus", + "output_tokens": 2834, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 149609, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 184 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 244.779, + "illegal_actions": 15, + "memo_writes": 10, + "protocol_penalty": 37.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 282.279, + "usage": { + "api_calls": 21, + "api_latency_ms": 88730.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.055998, + "decisions_with_usage": 20, + "harness_latency_ms": 90474.7, + "input_tokens": 157138, + "max_output_tokens_per_call": 583, + "model": "qwen/qwen3.7-plus", + "output_tokens": 4466, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 161604, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 246 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 238.587, + "illegal_actions": 1, + "memo_writes": 8, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 241.087, + "usage": { + "api_calls": 22, + "api_latency_ms": 67552.5, + "cached_input_tokens": 0, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.057297, + "decisions_with_usage": 20, + "harness_latency_ms": 69393.1, + "input_tokens": 164602, + "max_output_tokens_per_call": 403, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3612, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 168214, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 247 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 168.519, + "illegal_actions": 2, + "memo_writes": 8, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 173.519, + "usage": { + "api_calls": 21, + "api_latency_ms": 70727.4, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.054482, + "decisions_with_usage": 20, + "harness_latency_ms": 72458.4, + "input_tokens": 160176, + "max_output_tokens_per_call": 348, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3774, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 163950, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 231 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 211.885, + "illegal_actions": 3, + "memo_writes": 11, + "protocol_penalty": 7.5, + "query_declines": 0, + "rejected_offers": 2, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 219.385, + "usage": { + "api_calls": 23, + "api_latency_ms": 75254.3, + "cached_input_tokens": 0, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.05919, + "decisions_with_usage": 20, + "harness_latency_ms": 77226.8, + "input_tokens": 169168, + "max_output_tokens_per_call": 372, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3951, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 173119, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 268 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 213.703, + "illegal_actions": 2, + "memo_writes": 9, + "protocol_penalty": 5.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 218.703, + "usage": { + "api_calls": 20, + "api_latency_ms": 68996.3, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.049708, + "decisions_with_usage": 20, + "harness_latency_ms": 70651.5, + "input_tokens": 146415, + "max_output_tokens_per_call": 436, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3487, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 149902, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 249 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 120.078, + "illegal_actions": 4, + "memo_writes": 8, + "protocol_penalty": 10.0, + "query_declines": 0, + "rejected_offers": 6, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 130.078, + "usage": { + "api_calls": 20, + "api_latency_ms": 76981.6, + "cached_input_tokens": 6272, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.048967, + "decisions_with_usage": 20, + "harness_latency_ms": 78650.5, + "input_tokens": 144515, + "max_output_tokens_per_call": 342, + "model": "qwen/qwen3.7-plus", + "output_tokens": 3382, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147897, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "wins": 247 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 5, + "decision_failure_rate": 0.0, + "decisions": 480, + "failed_decisions": 0, + "failed_queries": 44, + "illegal_actions": 136, + "mean_score": 175.52, + "mean_strategy_score": 189.686, + "mean_total_wins": 234.125, + "memo_writes": 219, + "query_declines": 1, + "rejected_offers": 22, + "score_stddev": 26.355, + "total_protocol_penalty": 340.0, + "usage": { + "api_calls": 501, + "api_latency_ms": 1903142.9, + "cached_input_tokens": 119168, + "calls_with_finish_reason": 501, + "cost_decisions": 480, + "cost_usd": 1.293793, + "decisions_with_usage": 480, + "harness_latency_ms": 1944939.4, + "input_tokens": 3765865, + "max_output_tokens_per_call": 680, + "mean_input_tokens_per_decision": 7845.6, + "mean_output_tokens_per_decision": 194.1, + "mean_tokens_per_decision": 8039.6, + "model": "qwen/qwen3.7-plus", + "output_tokens": 93148, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3859013, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "within_seed_score_stddev": 29.607 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.0, + "candidate_decisions": 480, + "candidate_failed_decisions": 0, + "candidate_failed_queries": 44, + "candidate_illegal_actions": 136, + "candidate_mean_score": 175.52, + "candidate_mean_strategy_score": 189.686, + "candidate_mean_tokens_per_decision": 8039.6, + "candidate_memo_writes": 219, + "candidate_protocol_penalty": 340.0, + "candidate_query_declines": 1, + "candidate_rejected_offers": 22, + "candidate_usage": { + "api_calls": 501, + "api_latency_ms": 1903142.9, + "cached_input_tokens": 119168, + "calls_with_finish_reason": 501, + "cost_decisions": 480, + "cost_usd": 1.293793, + "decisions_with_usage": 480, + "harness_latency_ms": 1944939.4, + "input_tokens": 3765865, + "max_output_tokens_per_call": 680, + "mean_input_tokens_per_decision": 7845.6, + "mean_output_tokens_per_decision": 194.1, + "mean_tokens_per_decision": 8039.6, + "model": "qwen/qwen3.7-plus", + "output_tokens": 93148, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3859013, + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_providers": [ + "Alibaba" + ] + }, + "score_lift": -98.275, + "score_lift_pct": -35.89 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -236.099, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -116.78, + -73.081 + ], + "paired_lift_mean": -98.275, + "paired_lift_stddev": 31.829, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 147.156, + "lift": -119.613, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 162.474, + "lift": -122.955, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 165.137, + "lift": -116.306, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 213.92, + "lift": -90.23, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 175.608, + "lift": -95.31, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 140.678, + "lift": -122.644, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 217.295, + "lift": -20.51, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 181.889, + "lift": -98.629, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 141, + "rejected": 82 + }, + "draft": { + "accepted": 1539, + "rejected": 3 + }, + "information_memory": { + "accepted": 321, + "rejected": 44 + }, + "lineup": { + "accepted": 184, + "rejected": 69 + }, + "trades": { + "accepted": 610, + "rejected": 4 + } + }, + "raw_artifact_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:qwen/qwen3.7-plus", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "qwen/qwen3.7-plus", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Alibaba | qwen/qwen3.7-plus-20260602", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Alibaba", + "OPENROUTER_JSON_MODE": "true", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "alibaba", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-19T21:00:23+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/results/leaderboard/openrouter-tencent-hy3-free-novita.json b/results/leaderboard/openrouter-tencent-hy3-free-novita.json new file mode 100644 index 0000000..8b06999 --- /dev/null +++ b/results/leaderboard/openrouter-tencent-hy3-free-novita.json @@ -0,0 +1,4149 @@ +{ + "agent": "openrouter:tencent/hy3:free", + "baseline_cache": { + "enabled": true, + "hits": 0, + "total": 64 + }, + "baselines": [ + { + "agent": "random", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 88.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 88.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 128 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 97.564, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 97.564, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 206 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 72.323, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 72.323, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 127 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 103.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 103.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 199 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.327, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 108.327, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 184 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 68.959, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 68.959, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 120 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 108.958, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 108.958, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 209 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 125.939, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 125.939, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 228 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 0, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 96.715, + "mean_strategy_score": 96.715, + "mean_total_wins": 175.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.063, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "conservative", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.422, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.422, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 169 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 146.071, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 146.071, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 117.624, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 117.624, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 178 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.102, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 157.102, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 230 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 130.007, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 130.007, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 212 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.025, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.025, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 188 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 175.666, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 175.666, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 236 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 129.322, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 129.322, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 248 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 2, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 139.03, + "mean_strategy_score": 139.03, + "mean_total_wins": 217.5, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 18.297, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "win-now", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 229.158, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 229.158, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 269 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 289.76, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 289.76, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 347 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 232.259, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 232.259, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 271 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 313.554, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 313.554, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 338 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 293.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 248.33, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 248.33, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 323 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 235.576, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 235.576, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 313 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.45, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 364.45, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 322 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 275.834, + "mean_strategy_score": 275.834, + "mean_total_wins": 308.125, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 45.055, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "rebuild", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 121.469, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 121.469, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 183 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 153.946, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 153.946, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 114.67, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 114.67, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 173 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 133.156, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 133.156, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 232 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 139.851, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 139.851, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 194 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 135.457, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 135.457, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 193 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 167.778, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 167.778, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 143.633, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 143.633, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 255 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 1, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 138.745, + "mean_strategy_score": 138.745, + "mean_total_wins": 216.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 15.887, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "value", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.297, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 348.297, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 272 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 326.996, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 326.996, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 331 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 380.056, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 380.056, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 276 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 348.622, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 348.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 295 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 401.811, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 401.811, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 281 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 365.58, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 365.58, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 293.121, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 293.121, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 287 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 372.466, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 372.466, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 306 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 6, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 354.619, + "mean_strategy_score": 354.619, + "mean_total_wins": 294.25, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 31.492, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "shrewd", + "episodes": [ + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 347.841, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 347.841, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 270 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 400.707, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 400.707, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 335 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 412.444, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 412.444, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 282 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.691, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.691, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 315 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 367.261, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 367.261, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.211, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.211, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 314.409, + "illegal_actions": 0, + "memo_writes": 0, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 314.409, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 297 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 371.769, + "mean_strategy_score": 371.769, + "mean_total_wins": 296.375, + "memo_writes": 0, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 47.86, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "strategic", + "episodes": [ + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 421.622, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 421.622, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 371.661, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 371.661, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 308 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 12, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 402.025, + "mean_strategy_score": 402.025, + "mean_total_wins": 298.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 49.4, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + }, + { + "agent": "pick-trader", + "episodes": [ + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 455.725, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 11, + "strategy_score": 455.725, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 258 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 434.196, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 12, + "strategy_score": 434.196, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 327 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 461.084, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 13, + "strategy_score": 461.084, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 277 + }, + { + "championships": 3, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 459.026, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 14, + "strategy_score": 459.026, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 318 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 364.587, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 15, + "strategy_score": 364.587, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 279 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 414.307, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 16, + "strategy_score": 414.307, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 311 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 307.063, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 17, + "strategy_score": 307.063, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 285 + }, + { + "championships": 2, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 396.963, + "illegal_actions": 0, + "memo_writes": 5, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "seasons": 5, + "seed": 18, + "strategy_score": 396.963, + "usage": { + "api_calls": 0, + "api_latency_ms": 0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "wins": 310 + } + ], + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 14, + "decision_failure_rate": 0.0, + "decisions": 160, + "failed_decisions": 0, + "failed_queries": 0, + "illegal_actions": 0, + "mean_score": 411.619, + "mean_strategy_score": 411.619, + "mean_total_wins": 295.625, + "memo_writes": 40, + "query_declines": 0, + "rejected_offers": 0, + "score_stddev": 50.64, + "total_protocol_penalty": 0.0, + "usage": { + "api_calls": 0, + "api_latency_ms": 0.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 0, + "cost_decisions": 0, + "cost_usd": null, + "decisions_with_usage": 0, + "harness_latency_ms": 0.0, + "input_tokens": 0, + "max_output_tokens_per_call": 0, + "mean_input_tokens_per_decision": 0.0, + "mean_output_tokens_per_decision": 0.0, + "mean_tokens_per_decision": 0.0, + "model": null, + "output_tokens": 0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": null, + "reasoning_tokens": 0, + "total_tokens": 0, + "truncated_calls": 0, + "upstream_provider": null, + "upstream_providers": [] + }, + "within_seed_score_stddev": 0.0 + } + } + ], + "candidate": { + "agent": "openrouter:tencent/hy3:free", + "episodes": [ + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 136.314, + "illegal_actions": 2, + "memo_writes": 13, + "protocol_penalty": 5.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 11, + "strategy_score": 141.314, + "usage": { + "api_calls": 21, + "api_latency_ms": 101117.0, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 102937.6, + "input_tokens": 150235, + "max_output_tokens_per_call": 207, + "model": "tencent/hy3:free", + "output_tokens": 1998, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 152233, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 204 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 291.366, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 11, + "strategy_score": 291.366, + "usage": { + "api_calls": 20, + "api_latency_ms": 71027.5, + "cached_input_tokens": 7168, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 72711.6, + "input_tokens": 144519, + "max_output_tokens_per_call": 264, + "model": "tencent/hy3:free", + "output_tokens": 2095, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 146614, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 222 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 99.79, + "illegal_actions": 3, + "memo_writes": 13, + "protocol_penalty": 7.5, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 11, + "strategy_score": 107.29, + "usage": { + "api_calls": 20, + "api_latency_ms": 67114.3, + "cached_input_tokens": 7168, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 68759.8, + "input_tokens": 141845, + "max_output_tokens_per_call": 196, + "model": "tencent/hy3:free", + "output_tokens": 1869, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 143714, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 202 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 179.854, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 12, + "strategy_score": 179.854, + "usage": { + "api_calls": 21, + "api_latency_ms": 75923.3, + "cached_input_tokens": 768, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 77688.2, + "input_tokens": 151671, + "max_output_tokens_per_call": 210, + "model": "tencent/hy3:free", + "output_tokens": 2063, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153734, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 268 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 258.217, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 12, + "strategy_score": 258.217, + "usage": { + "api_calls": 22, + "api_latency_ms": 79421.9, + "cached_input_tokens": 7936, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 81297.7, + "input_tokens": 161830, + "max_output_tokens_per_call": 186, + "model": "tencent/hy3:free", + "output_tokens": 2084, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 163914, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 282 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 180.078, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 12, + "strategy_score": 180.078, + "usage": { + "api_calls": 20, + "api_latency_ms": 75549.5, + "cached_input_tokens": 9984, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 77230.4, + "input_tokens": 145696, + "max_output_tokens_per_call": 238, + "model": "tencent/hy3:free", + "output_tokens": 2227, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147923, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 294 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 181.372, + "illegal_actions": 1, + "memo_writes": 13, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 13, + "strategy_score": 183.872, + "usage": { + "api_calls": 21, + "api_latency_ms": 74938.8, + "cached_input_tokens": 832, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 76630.6, + "input_tokens": 158651, + "max_output_tokens_per_call": 193, + "model": "tencent/hy3:free", + "output_tokens": 2179, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 160830, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 201 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 157.075, + "illegal_actions": 0, + "memo_writes": 11, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 13, + "strategy_score": 157.075, + "usage": { + "api_calls": 21, + "api_latency_ms": 72786.5, + "cached_input_tokens": 7936, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 74462.7, + "input_tokens": 157170, + "max_output_tokens_per_call": 206, + "model": "tencent/hy3:free", + "output_tokens": 2022, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 159192, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 198 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 170.859, + "illegal_actions": 1, + "memo_writes": 13, + "protocol_penalty": 2.5, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 13, + "strategy_score": 173.359, + "usage": { + "api_calls": 20, + "api_latency_ms": 82551.6, + "cached_input_tokens": 8576, + "calls_with_finish_reason": 19, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 84192.0, + "input_tokens": 145839, + "max_output_tokens_per_call": 200, + "model": "tencent/hy3:free", + "output_tokens": 2018, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 147857, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 191 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 245.321, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 14, + "strategy_score": 245.321, + "usage": { + "api_calls": 21, + "api_latency_ms": 77878.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 79625.8, + "input_tokens": 147034, + "max_output_tokens_per_call": 203, + "model": "tencent/hy3:free", + "output_tokens": 2047, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 149081, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 268 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 176.666, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 14, + "strategy_score": 176.666, + "usage": { + "api_calls": 21, + "api_latency_ms": 74283.1, + "cached_input_tokens": 7232, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 75997.6, + "input_tokens": 151112, + "max_output_tokens_per_call": 203, + "model": "tencent/hy3:free", + "output_tokens": 2028, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 153140, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 263 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 217.753, + "illegal_actions": 0, + "memo_writes": 15, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 14, + "strategy_score": 217.753, + "usage": { + "api_calls": 22, + "api_latency_ms": 81102.1, + "cached_input_tokens": 8832, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 82902.6, + "input_tokens": 157066, + "max_output_tokens_per_call": 209, + "model": "tencent/hy3:free", + "output_tokens": 2375, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 159441, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 275 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 195.765, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 15, + "strategy_score": 195.765, + "usage": { + "api_calls": 22, + "api_latency_ms": 93040.6, + "cached_input_tokens": 1600, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 94850.4, + "input_tokens": 159911, + "max_output_tokens_per_call": 203, + "model": "tencent/hy3:free", + "output_tokens": 2137, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 162048, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 220 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 211.227, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 15, + "strategy_score": 211.227, + "usage": { + "api_calls": 22, + "api_latency_ms": 80399.5, + "cached_input_tokens": 9280, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 82211.6, + "input_tokens": 163709, + "max_output_tokens_per_call": 193, + "model": "tencent/hy3:free", + "output_tokens": 2188, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 165897, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 228 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.054, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 15, + "strategy_score": 157.054, + "usage": { + "api_calls": 20, + "api_latency_ms": 71471.3, + "cached_input_tokens": 7232, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 73069.2, + "input_tokens": 149144, + "max_output_tokens_per_call": 195, + "model": "tencent/hy3:free", + "output_tokens": 2320, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 151464, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 225 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 237.151, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 16, + "strategy_score": 237.151, + "usage": { + "api_calls": 21, + "api_latency_ms": 74724.2, + "cached_input_tokens": 0, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 76456.5, + "input_tokens": 150018, + "max_output_tokens_per_call": 204, + "model": "tencent/hy3:free", + "output_tokens": 1978, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 151996, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 218 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 5, + "final_score": 261.005, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 16, + "strategy_score": 261.005, + "usage": { + "api_calls": 20, + "api_latency_ms": 69188.4, + "cached_input_tokens": 7232, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 70823.4, + "input_tokens": 143452, + "max_output_tokens_per_call": 190, + "model": "tencent/hy3:free", + "output_tokens": 1881, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 145333, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 214 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 0, + "final_score": 157.014, + "illegal_actions": 0, + "memo_writes": 14, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 16, + "strategy_score": 157.014, + "usage": { + "api_calls": 21, + "api_latency_ms": 72106.0, + "cached_input_tokens": 7232, + "calls_with_finish_reason": 21, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 73867.5, + "input_tokens": 153121, + "max_output_tokens_per_call": 188, + "model": "tencent/hy3:free", + "output_tokens": 2101, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 155222, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 245 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 198.968, + "illegal_actions": 0, + "memo_writes": 11, + "protocol_penalty": 0.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 17, + "strategy_score": 198.968, + "usage": { + "api_calls": 22, + "api_latency_ms": 72326.3, + "cached_input_tokens": 2496, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 74085.5, + "input_tokens": 157506, + "max_output_tokens_per_call": 189, + "model": "tencent/hy3:free", + "output_tokens": 1721, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 159227, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 223 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 2, + "final_score": 157.77, + "illegal_actions": 1, + "memo_writes": 17, + "protocol_penalty": 2.5, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 17, + "strategy_score": 160.27, + "usage": { + "api_calls": 22, + "api_latency_ms": 78439.7, + "cached_input_tokens": 8896, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 80177.0, + "input_tokens": 158770, + "max_output_tokens_per_call": 186, + "model": "tencent/hy3:free", + "output_tokens": 2070, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 160840, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 244 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 197.434, + "illegal_actions": 2, + "memo_writes": 13, + "protocol_penalty": 5.0, + "query_declines": 2, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 17, + "strategy_score": 202.434, + "usage": { + "api_calls": 22, + "api_latency_ms": 119824.0, + "cached_input_tokens": 8064, + "calls_with_finish_reason": 22, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 121726.0, + "input_tokens": 155978, + "max_output_tokens_per_call": 196, + "model": "tencent/hy3:free", + "output_tokens": 2016, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 157994, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 245 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 4, + "final_score": 187.912, + "illegal_actions": 0, + "memo_writes": 13, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 1, + "seasons": 5, + "seed": 18, + "strategy_score": 187.912, + "usage": { + "api_calls": 21, + "api_latency_ms": 100663.7, + "cached_input_tokens": 1600, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 102382.0, + "input_tokens": 146336, + "max_output_tokens_per_call": 202, + "model": "tencent/hy3:free", + "output_tokens": 1891, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 148227, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 268 + }, + { + "championships": 0, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 3, + "final_score": 214.684, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 0, + "rejected_offers": 0, + "repeat": 2, + "seasons": 5, + "seed": 18, + "strategy_score": 214.684, + "usage": { + "api_calls": 22, + "api_latency_ms": 162625.5, + "cached_input_tokens": 8704, + "calls_with_finish_reason": 20, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 164742.2, + "input_tokens": 152224, + "max_output_tokens_per_call": 521, + "model": "tencent/hy3:free", + "output_tokens": 2677, + "protocol_repair_attempts": 1, + "protocol_repairs_succeeded": 1, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 154901, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 275 + }, + { + "championships": 1, + "decisions": 20, + "failed_decisions": 0, + "failed_queries": 1, + "final_score": 229.533, + "illegal_actions": 0, + "memo_writes": 12, + "protocol_penalty": 0.0, + "query_declines": 1, + "rejected_offers": 0, + "repeat": 3, + "seasons": 5, + "seed": 18, + "strategy_score": 229.533, + "usage": { + "api_calls": 23, + "api_latency_ms": 87006.2, + "cached_input_tokens": 10880, + "calls_with_finish_reason": 23, + "cost_decisions": 20, + "cost_usd": 0.0, + "decisions_with_usage": 20, + "harness_latency_ms": 89068.9, + "input_tokens": 163754, + "max_output_tokens_per_call": 234, + "model": "tencent/hy3:free", + "output_tokens": 2217, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 165971, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "wins": 273 + } + ], + "repeats": 3, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "summary": { + "championships": 7, + "decision_failure_rate": 0.0, + "decisions": 480, + "failed_decisions": 0, + "failed_queries": 36, + "illegal_actions": 10, + "mean_score": 195.841, + "mean_strategy_score": 196.883, + "mean_total_wins": 239.417, + "memo_writes": 318, + "query_declines": 17, + "rejected_offers": 0, + "score_stddev": 17.328, + "total_protocol_penalty": 25.0, + "usage": { + "api_calls": 508, + "api_latency_ms": 2015509.2, + "cached_input_tokens": 139648, + "calls_with_finish_reason": 503, + "cost_decisions": 480, + "cost_usd": 0.0, + "decisions_with_usage": 480, + "harness_latency_ms": 2057896.8, + "input_tokens": 3666591, + "max_output_tokens_per_call": 521, + "mean_input_tokens_per_decision": 7638.7, + "mean_output_tokens_per_decision": 104.6, + "mean_tokens_per_decision": 7743.3, + "model": "tencent/hy3:free", + "output_tokens": 50202, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3716793, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "within_seed_score_stddev": 32.707 + } + }, + "normalized": { + "baseline_illegal_actions": 0, + "baseline_panel_mean_score": 273.794, + "baseline_panel_mean_strategy_score": 273.795, + "baseline_panel_total_protocol_penalty": 0.0, + "candidate_decision_failure_rate": 0.0, + "candidate_decisions": 480, + "candidate_failed_decisions": 0, + "candidate_failed_queries": 36, + "candidate_illegal_actions": 10, + "candidate_mean_score": 195.841, + "candidate_mean_strategy_score": 196.883, + "candidate_mean_tokens_per_decision": 7743.3, + "candidate_memo_writes": 318, + "candidate_protocol_penalty": 25.0, + "candidate_query_declines": 17, + "candidate_rejected_offers": 0, + "candidate_usage": { + "api_calls": 508, + "api_latency_ms": 2015509.2, + "cached_input_tokens": 139648, + "calls_with_finish_reason": 503, + "cost_decisions": 480, + "cost_usd": 0.0, + "decisions_with_usage": 480, + "harness_latency_ms": 2057896.8, + "input_tokens": 3666591, + "max_output_tokens_per_call": 521, + "mean_input_tokens_per_decision": 7638.7, + "mean_output_tokens_per_decision": 104.6, + "mean_tokens_per_decision": 7743.3, + "model": "tencent/hy3:free", + "output_tokens": 50202, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "reasoning_tokens": 0, + "total_tokens": 3716793, + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_providers": [ + "Novita" + ] + }, + "score_lift": -77.954, + "score_lift_pct": -28.47 + }, + "paired": { + "best_baseline": { + "agent": "pick-trader", + "mean_score": 411.619, + "paired_lift_mean": -215.778, + "seed_win_rate": 0.0 + }, + "candidate_seed_win_rate": 0.0, + "num_seeds": 8, + "paired_lift_ci95": [ + -91.535, + -63.891 + ], + "paired_lift_mean": -77.954, + "paired_lift_stddev": 20.235, + "per_seed": [ + { + "baseline_panel_score": 266.769, + "candidate_score": 175.823, + "lift": -90.946, + "seed": 11 + }, + { + "baseline_panel_score": 285.43, + "candidate_score": 206.05, + "lift": -79.38, + "seed": 12 + }, + { + "baseline_panel_score": 281.443, + "candidate_score": 169.769, + "lift": -111.674, + "seed": 13 + }, + { + "baseline_panel_score": 304.15, + "candidate_score": 213.247, + "lift": -90.904, + "seed": 14 + }, + { + "baseline_panel_score": 270.918, + "candidate_score": 188.015, + "lift": -82.903, + "seed": 15 + }, + { + "baseline_panel_score": 263.322, + "candidate_score": 218.39, + "lift": -44.932, + "seed": 16 + }, + { + "baseline_panel_score": 237.804, + "candidate_score": 184.724, + "lift": -53.081, + "seed": 17 + }, + { + "baseline_panel_score": 280.518, + "candidate_score": 210.71, + "lift": -69.808, + "seed": 18 + } + ], + "sign_flip_p_value": 0.0078, + "significant_at_95": true + }, + "publication": { + "format": "gm-bench-result-summary-v1", + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 203, + "rejected": 1 + }, + "draft": { + "accepted": 1684, + "rejected": 18 + }, + "information_memory": { + "accepted": 411, + "rejected": 35 + }, + "lineup": { + "accepted": 165, + "rejected": 9 + }, + "trades": { + "accepted": 594, + "rejected": 0 + } + }, + "raw_artifact_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "traces_included": false + }, + "run_info": { + "agent": "openrouter:tencent/hy3:free", + "agent_timeout": 180.0, + "benchmark_contract": { + "action_protocol_version": "actions-v2", + "benchmark_version": "sota-v2", + "contract_fingerprint": "558e8f35ea1d66b9", + "observation_version": "observation-v1", + "scoring_scale_fingerprint": "05a60ff4f691e734", + "scoring_version": "score-v1", + "simulator_version": "sim-v2" + }, + "command": "model", + "gm_bench_version": "0.1.0", + "model": "tencent/hy3:free", + "preset": "leaderboard", + "profile": "compact", + "protocol_repair_attempts": 1, + "provider": "openrouter", + "provider_options": { + "GM_BENCH_OUTPUT_BUDGET_CELL": "4096", + "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", + "OPENROUTER_ALLOW_FALLBACKS": "false", + "OPENROUTER_DATA_COLLECTION": "deny", + "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Novita | tencent/hy3-20260706:free", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Novita", + "OPENROUTER_JSON_MODE": "false", + "OPENROUTER_MAX_TOKENS": "4096", + "OPENROUTER_PROVIDER_ONLY": "novita", + "OPENROUTER_PROVIDER_SORT": "price", + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_REQUIRE_PARAMETERS": "true" + }, + "python_version": "3.12.7", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "seed_panel": { + "count": 8, + "name": "public-leaderboard", + "preset": "leaderboard", + "sha256": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143" + }, + "session": false, + "timestamp_utc": "2026-07-18T22:55:14+00:00", + "transport": "gateway-api" + }, + "seasons": 5, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ] +} diff --git a/tests/test_publication.py b/tests/test_publication.py index b942c17..80f66b1 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -22,7 +22,16 @@ def _payload() -> dict: "season_summaries": [{"season": 1}], "usage": {"total_tokens": 100, "per_decision": [{"input_tokens": 4}]}, } - return {"candidate": {"episodes": [episode]}, "baselines": [{"episodes": [episode]}]} + return { + "baseline_cache": { + "enabled": True, + "hits": 1, + "path": "/Users/example/project/data/baseline_cache.json", + "total": 1, + }, + "candidate": {"episodes": [episode]}, + "baselines": [{"episodes": [episode]}], + } def test_compact_result_removes_traces_and_hashes_raw_payload() -> None: @@ -46,6 +55,7 @@ def test_compact_result_removes_traces_and_hashes_raw_payload() -> None: assert "transactions" not in episode assert "season_summaries" not in episode assert episode["usage"] == {"total_tokens": 100} + assert compact["baseline_cache"] == {"enabled": True, "hits": 1, "total": 1} def test_compact_result_rejects_already_compacted_artifact() -> None: diff --git a/web/src/data/leaderboard.json b/web/src/data/leaderboard.json index 1c84f76..0a4a46e 100644 --- a/web/src/data/leaderboard.json +++ b/web/src/data/leaderboard.json @@ -53,12 +53,699 @@ }, "excluded_models": [], "headroom": { - "best_model": null, + "best_model": 231.851, "oracle": 431.153, "pick_trader": 411.619, "random": 96.715 }, - "models": [], + "models": [ + { + "api_latency_s_per_decision": 13.5, + "artifact_sha256": "aa8f4bdf5228664b7d95e2bb749b82e2954920551b7cca47a5e9a0c5f85aa4c6", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -66.061, + -16.342 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.3161, + "cost_usd": 7.586279, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 453, + "fallback_rate": 0.0, + "harness_latency_s_per_decision": 13.61, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:meta/muse-spark-1.1", + "illegal_actions": 51, + "input_tokens_per_decision": 8935.6, + "lane": "api", + "lift_vs_best_baseline": -179.768, + "mean_score": 231.851, + "mean_strategy_score": 237.163, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 279, + "rejected": 30 + }, + "draft": { + "accepted": 1795, + "rejected": 12 + }, + "information_memory": { + "accepted": 543, + "rejected": 453 + }, + "lineup": { + "accepted": 219, + "rejected": 12 + }, + "trades": { + "accepted": 653, + "rejected": 42 + } + }, + "model": "meta/muse-spark-1.1", + "output_token_cap": 4096, + "output_tokens_per_decision": 1148.7, + "paired_lift": -41.944, + "protocol_penalty": 127.5, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "7b664cbd57aab8975905a08ff057cf528211fd67efc52aac0ffde5b6d829e030", + "score_stddev": 39.764, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.125, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate has 453 failed queries across 480 decisions (0.94 per decision); misfired scout/inspect lookups may indicate the model is not reading query errors", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 10084.2, + "total_tokens": 4840432 + }, + { + "api_latency_s_per_decision": 13.03, + "artifact_sha256": "cf7912a55ec91468a339bfa660ef3b55edf4d11c286202bab6574d4f642fd00f", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -81.974, + -30.464 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.0534, + "cost_usd": 1.281193, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 209, + "fallback_rate": 0.017, + "harness_latency_s_per_decision": 13.2, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:z-ai/glm-5.2", + "illegal_actions": 168, + "input_tokens_per_decision": 10354.4, + "lane": "api", + "lift_vs_best_baseline": -194.08, + "mean_score": 217.539, + "mean_strategy_score": 235.039, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 229, + "rejected": 76 + }, + "draft": { + "accepted": 1741, + "rejected": 92 + }, + "information_memory": { + "accepted": 1042, + "rejected": 199 + }, + "lineup": { + "accepted": 195, + "rejected": 76 + }, + "trades": { + "accepted": 595, + "rejected": 55 + } + }, + "model": "z-ai/glm-5.2", + "output_token_cap": 4096, + "output_tokens_per_decision": 410.0, + "paired_lift": -56.255, + "protocol_penalty": 420.0, + "protocol_repair_attempts": 29, + "protocol_repairs_succeeded": 21, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "36d1ed42d44f97041253f162d8173d82d58b1ba12c58131ffd510bd180df0727", + "score_stddev": 37.736, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.25, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate used adapter fallback/error output on at least one decision", + "candidate has 209 failed queries across 480 decisions (0.44 per decision); misfired scout/inspect lookups may indicate the model is not reading query errors", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 10764.4, + "total_tokens": 5166907 + }, + { + "api_latency_s_per_decision": 1.49, + "artifact_sha256": "082b1964fcccecf7e5ba37ff31ca76a3cf7256052dc1237fe198004fa9eeebfb", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -89.586, + -27.162 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.2634, + "cost_usd": 6.320513, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 5, + "fallback_rate": 0.013, + "harness_latency_s_per_decision": 1.59, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:google/gemini-3.5-flash", + "illegal_actions": 18, + "input_tokens_per_decision": 8708.5, + "lane": "api", + "lift_vs_best_baseline": -195.995, + "mean_score": 215.624, + "mean_strategy_score": 217.499, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 195, + "rejected": 3 + }, + "draft": { + "accepted": 1507, + "rejected": 2 + }, + "information_memory": { + "accepted": 7, + "rejected": 5 + }, + "lineup": { + "accepted": 183, + "rejected": 16 + }, + "trades": { + "accepted": 571, + "rejected": 0 + } + }, + "model": "google/gemini-3.5-flash", + "output_token_cap": 4096, + "output_tokens_per_decision": 62.2, + "paired_lift": -58.17, + "protocol_penalty": 45.0, + "protocol_repair_attempts": 52, + "protocol_repairs_succeeded": 46, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "2fd28b3ab507e76dc512ead32564b0328635b5b66f0a330d71a470f3b1b8a786", + "score_stddev": 33.263, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.125, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate used adapter fallback/error output on at least one decision", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 8770.7, + "total_tokens": 4209929 + }, + { + "api_latency_s_per_decision": 4.2, + "artifact_sha256": "cd0d7d7cd0a3cdf9e59a739ce097cbbde90d249e39c5e7d8b11e7a69c171cf3b", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -91.535, + -63.891 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.0, + "cost_usd": 0.0, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 36, + "fallback_rate": 0.0, + "harness_latency_s_per_decision": 4.29, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:tencent/hy3:free", + "illegal_actions": 10, + "input_tokens_per_decision": 7638.7, + "lane": "api", + "lift_vs_best_baseline": -215.778, + "mean_score": 195.841, + "mean_strategy_score": 196.883, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 203, + "rejected": 1 + }, + "draft": { + "accepted": 1684, + "rejected": 18 + }, + "information_memory": { + "accepted": 411, + "rejected": 35 + }, + "lineup": { + "accepted": 165, + "rejected": 9 + }, + "trades": { + "accepted": 594, + "rejected": 0 + } + }, + "model": "tencent/hy3:free", + "output_token_cap": 4096, + "output_tokens_per_decision": 104.6, + "paired_lift": -77.954, + "protocol_penalty": 25.0, + "protocol_repair_attempts": 3, + "protocol_repairs_succeeded": 3, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "e4fdd8876e631be94466352fe23e795d122b3a85e70dbb30b0c2c84951935eec", + "score_stddev": 17.328, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.0, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 7743.3, + "total_tokens": 3716793 + }, + { + "api_latency_s_per_decision": 3.96, + "artifact_sha256": "de34dcb82a8d5c7823d5ca4670cb01d2f4ed5f0e32c718545c89bbfd785f1da5", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -116.78, + -73.081 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.0539, + "cost_usd": 1.293793, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 44, + "fallback_rate": 0.0, + "harness_latency_s_per_decision": 4.05, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:qwen/qwen3.7-plus", + "illegal_actions": 136, + "input_tokens_per_decision": 7845.6, + "lane": "api", + "lift_vs_best_baseline": -236.099, + "mean_score": 175.52, + "mean_strategy_score": 189.686, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 141, + "rejected": 82 + }, + "draft": { + "accepted": 1539, + "rejected": 3 + }, + "information_memory": { + "accepted": 321, + "rejected": 44 + }, + "lineup": { + "accepted": 184, + "rejected": 69 + }, + "trades": { + "accepted": 610, + "rejected": 4 + } + }, + "model": "qwen/qwen3.7-plus", + "output_token_cap": 4096, + "output_tokens_per_decision": 194.1, + "paired_lift": -98.275, + "protocol_penalty": 340.0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "e544be5ae70d7bbb5b228fab3742d07e89504a286a3ca244a01eb68924acf6c3", + "score_stddev": 26.355, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.0, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 8039.6, + "total_tokens": 3859013 + }, + { + "api_latency_s_per_decision": 1.45, + "artifact_sha256": "16da22113c2447cf21fc3f6fc3ec94eb21531380534df3599e43e7fb5cae89e1", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -114.217, + -83.091 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.195, + "cost_usd": 4.680175, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 5, + "fallback_rate": 0.0, + "harness_latency_s_per_decision": 1.54, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:openai/gpt-5.6-luna", + "illegal_actions": 126, + "input_tokens_per_decision": 7511.5, + "lane": "api", + "lift_vs_best_baseline": -237.693, + "mean_score": 173.926, + "mean_strategy_score": 187.051, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 94, + "rejected": 12 + }, + "draft": { + "accepted": 1764, + "rejected": 133 + }, + "information_memory": { + "accepted": 339, + "rejected": 5 + }, + "lineup": { + "accepted": 197, + "rejected": 123 + }, + "trades": { + "accepted": 583, + "rejected": 0 + } + }, + "model": "openai/gpt-5.6-luna", + "output_token_cap": 4096, + "output_tokens_per_decision": 99.8, + "paired_lift": -99.869, + "protocol_penalty": 315.0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "1af66b6e0e0f6f492e10d29b5e27635e0a0187781a512262cbfdc1e6b847ac81", + "score_stddev": 24.839, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.0, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 7611.2, + "total_tokens": 3653397 + }, + { + "api_latency_s_per_decision": 6.4, + "artifact_sha256": "9e2a74fbc55eb51ed1bcbd25064f178fcf73d962d650cbe04cec1ff5ffa75455", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -154.031, + -104.212 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.4606, + "cost_usd": 11.05436, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 137, + "fallback_rate": 0.0, + "harness_latency_s_per_decision": 6.49, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:anthropic/claude-sonnet-5", + "illegal_actions": 44, + "input_tokens_per_decision": 9551.8, + "lane": "api", + "lift_vs_best_baseline": -269.475, + "mean_score": 142.143, + "mean_strategy_score": 146.727, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 85, + "rejected": 9 + }, + "draft": { + "accepted": 1699, + "rejected": 27 + }, + "information_memory": { + "accepted": 743, + "rejected": 137 + }, + "lineup": { + "accepted": 138, + "rejected": 39 + }, + "trades": { + "accepted": 588, + "rejected": 0 + } + }, + "model": "anthropic/claude-sonnet-5", + "output_token_cap": 4096, + "output_tokens_per_decision": 392.6, + "paired_lift": -131.651, + "protocol_penalty": 110.0, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "f2e1057f7e72a99bb556b30c058ccc827eb7316bff287d075f2061cee244d799", + "score_stddev": 25.552, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.0, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate has 137 failed queries across 480 decisions (0.29 per decision); misfired scout/inspect lookups may indicate the model is not reading query errors", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 9944.4, + "total_tokens": 4773328 + }, + { + "api_latency_s_per_decision": 3.08, + "artifact_sha256": "53e93b066a1ade0c72f99ec0bbb4fc56b4c07ee48f18e4f53eca5150cf8b6ddf", + "baseline_panel_mean_score": 273.794, + "benchmark_version": "sota-v2", + "ci95": [ + -167.324, + -116.177 + ], + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_per_episode_usd": 0.0515, + "cost_usd": 1.237149, + "decision_points": 480, + "decisions_with_usage": 480, + "failed_queries": 65, + "fallback_rate": 0.002, + "harness_latency_s_per_decision": 3.18, + "holm_adjusted_p_value": 0.078125, + "holm_reject_at_0_05": false, + "id": "openrouter:minimax/minimax-m3", + "illegal_actions": 14, + "input_tokens_per_decision": 8613.1, + "lane": "api", + "lift_vs_best_baseline": -281.739, + "mean_score": 129.88, + "mean_strategy_score": 131.338, + "mechanic_breakdown": { + "cap_free_agency": { + "accepted": 54, + "rejected": 9 + }, + "draft": { + "accepted": 1718, + "rejected": 107 + }, + "information_memory": { + "accepted": 224, + "rejected": 65 + }, + "lineup": { + "accepted": 86, + "rejected": 6 + }, + "trades": { + "accepted": 551, + "rejected": 0 + } + }, + "model": "minimax/minimax-m3", + "output_token_cap": 4096, + "output_tokens_per_decision": 115.1, + "paired_lift": -143.914, + "protocol_penalty": 35.0, + "protocol_repair_attempts": 7, + "protocol_repairs_succeeded": 6, + "provider": "openrouter", + "publication_eligible": true, + "publication_issues": [], + "raw_artifact_sha256": "29a94c23afe74508bdf3e82e4d79697d508de4a715b865c99a6317805f75c32f", + "score_stddev": 27.051, + "seasons": 5, + "seed_panel": "public-leaderboard", + "seed_panel_hash": "c68416e4d00b7198765b5b20d89c04c030dbc8c2516fe98c280ae8ae0afb0143", + "seed_win_rate": 0.0, + "seeds": [ + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18 + ], + "session": false, + "significant": true, + "sota_v2_eligible": true, + "sota_v2_issues": [ + "candidate has illegal actions; score includes protocol penalties", + "candidate used adapter fallback/error output on at least one decision", + "candidate does not beat the strongest scripted baseline" + ], + "tier": 1, + "tokens_per_decision": 8728.2, + "total_tokens": 4189533 + } + ], "preset": { "decision_points_per_episode": 20, "name": "leaderboard", @@ -79,7 +766,7 @@ "decision_recommendation": null, "duplicate_cells": [], "duplicate_headline_rows": 0, - "eligible_headline_models": 0, + "eligible_headline_models": 8, "frozen_output_token_cap": 4096, "minimum_headline_models": 8, "missing": [], @@ -137,10 +824,8 @@ } ], "output_policy_basis": "common-safety-ceiling-with-native-minimum-reasoning", - "panel_analysis_issues": [ - "publication panel analysis artifact is missing" - ], - "panel_analysis_ready": false, + "panel_analysis_issues": [], + "panel_analysis_ready": true, "planned_caps": [ 256, 1024, @@ -148,12 +833,12 @@ 16384 ], "points": [], - "publishable_ranking": false, - "reason": "frozen lane has 0 eligible headline models; at least 8 are required", + "publishable_ranking": true, + "reason": "four-cap sweep retired before replacement official cells; fixed safety ceiling governs publication", "rejected_artifacts": [], "schema_version": 1, "smoke_gate_issues": [], "status": "retired" }, - "updated": "" + "updated": "2026-07-19" }