diff --git a/README.md b/README.md index 5e4d148..3ece48e 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,11 @@ cd web && bun install && bun run dev # local site The site in `web/` deploys to GitHub Pages automatically on pushes to `main` (`.github/workflows/pages.yml`). -The headline API lane uses the common 1,024-token safety ceiling frozen in -[`config/sota_v2_lane.json`](config/sota_v2_lane.json), with reasoning disabled -and actual token/cost efficiency reported beside score. The earlier four-cap +The provisional headline API lane uses the common 4,096-token safety ceiling in +[`config/sota_v2_lane.json`](config/sota_v2_lane.json), with native-minimum +reasoning and actual token/cost efficiency reported beside score. Reasoning is +disabled where optional and set to the lowest OpenRouter-supported effort where +mandatory. The earlier four-cap study in [`config/output_budget_sweep.json`](config/output_budget_sweep.json) is retained for auditability but retired as a publication prerequisite. Until all registered routes pass the common smoke, the registry is frozen, and at least diff --git a/config/openrouter_pricing_snapshot.json b/config/openrouter_pricing_snapshot.json index 7414ba5..4cd38b1 100644 --- a/config/openrouter_pricing_snapshot.json +++ b/config/openrouter_pricing_snapshot.json @@ -1,38 +1,44 @@ { "schema_version": 1, - "checked_at_utc": "2026-07-15T20:41:27Z", + "checked_at_utc": "2026-07-17T19:37:57Z", "source": "https://openrouter.ai/api/v1/models and per-model /endpoints responses", "currency": "USD", "rates_are_per_token": true, "planning_assumptions": { "input_tokens_per_decision": 8000, - "expected_output_tokens_per_decision": 1024, + "expected_output_tokens_per_decision": 4096, "cost_contingency_multiplier": 1.2, "runtime_contingency_multiplier": 1.5 }, "runtime_observations": { - "source": "legacy standardized 1,024-token Luna, MiniMax, and Qwen observations; use only for conservative reservations and refresh every registered route through the fixed-cap smoke gate before the full panel", - "observed_at_utc": "2026-07-15T01:22:44Z", + "source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md", + "observed_at_utc": "2026-07-17T19:42:44Z", "api_seconds_per_decision": { - "minimax/minimax-m3": 2.546, - "openai/gpt-5.6-luna": 1.981, - "qwen/qwen3.5-9b": 11.622 + "anthropic/claude-sonnet-5": 8.657, + "google/gemini-3.5-flash": 1.497, + "meta/muse-spark-1.1": 13.827, + "minimax/minimax-m3": 2.955, + "mistralai/mistral-medium-3-5": 2.464, + "openai/gpt-5.6-luna": 4.759, + "qwen/qwen3.7-plus": 3.586, + "tencent/hy3:free": 4.087, + "x-ai/grok-4.5": 7.683, + "z-ai/glm-5.2": 6.656 } }, "models": { "anthropic/claude-sonnet-5": {"prompt": 0.000002, "completion": 0.00001}, - "anthropic/claude-haiku-4.5": {"prompt": 0.000001, "completion": 0.000005}, - "deepseek/deepseek-v4-pro": {"prompt": 0.0000013, "completion": 0.0000026}, + "deepseek/deepseek-v4-pro": {"prompt": 0.000000435, "completion": 0.00000087}, + "google/gemini-3.5-flash": {"prompt": 0.0000015, "completion": 0.000009}, + "meta/muse-spark-1.1": {"prompt": 0.00000125, "completion": 0.00000425}, "minimax/minimax-m3": {"prompt": 0.0000003, "completion": 0.0000012}, - "moonshotai/kimi-k2.6": {"prompt": 0.00000075, "completion": 0.0000035}, + "moonshotai/kimi-k3": {"prompt": 0.000003, "completion": 0.000015}, "mistralai/mistral-medium-3-5": {"prompt": 0.0000015, "completion": 0.0000075}, - "mistralai/mistral-small-2603": {"prompt": 0.00000015, "completion": 0.0000006}, - "nvidia/nemotron-3-nano-30b-a3b": {"prompt": 0.00000005, "completion": 0.0000002}, - "nvidia/nemotron-3-ultra-550b-a55b": {"prompt": 0.0000006, "completion": 0.0000036}, - "openai/gpt-5.4-mini": {"prompt": 0.00000075, "completion": 0.0000045}, + "nvidia/nemotron-3-ultra-550b-a55b:free": {"prompt": 0, "completion": 0}, "openai/gpt-5.6-luna": {"prompt": 0.000001, "completion": 0.000006}, - "qwen/qwen3.5-9b": {"prompt": 0.0000001, "completion": 0.00000015}, "qwen/qwen3.7-plus": {"prompt": 0.00000032, "completion": 0.00000128}, - "z-ai/glm-5.2": {"prompt": 0.00000091, "completion": 0.00000286} + "tencent/hy3:free": {"prompt": 0, "completion": 0}, + "x-ai/grok-4.5": {"prompt": 0.000002, "completion": 0.000006}, + "z-ai/glm-5.2": {"prompt": 0.0000014, "completion": 0.0000044} } } diff --git a/config/output_budget_sweep.json b/config/output_budget_sweep.json index 3dda253..460f5ef 100644 --- a/config/output_budget_sweep.json +++ b/config/output_budget_sweep.json @@ -95,13 +95,13 @@ "require_complete_cost": true, "selection_frozen_at_utc": null, "selection_rationale": "Retained for auditability: small open Qwen, capable low-cost MiniMax, and frontier GPT-5.6 Luna spanned model scale and expected capability in the superseded four-cap design.", - "protocol_amendment": "The first Luna cell exposed phase-ambiguous scaffold examples and is superseded as diagnostic evidence. On 2026-07-15 the project retired the four-cap experiment before any replacement official cell and froze a common 1,024-token safety ceiling instead. Observed token efficiency is a secondary metric, not an experimental axis.", + "protocol_amendment": "The first Luna cell exposed phase-ambiguous scaffold examples and is superseded as diagnostic evidence. On 2026-07-15 the project retired the four-cap experiment before any replacement official cell and froze a common 1,024-token safety ceiling. On 2026-07-16 the user deliberately revised the model panel before any full-panel score existed; four requested models require reasoning, so the replacement lane now uses a provisional common 4,096-token cap with native-minimum reasoning. Observed token efficiency remains a secondary metric, not an experimental axis.", "replacement_policy": { "config": "config/sota_v2_lane.json", - "output_token_cap": 1024, - "reasoning_enabled": false, - "pre_full_panel_cap_pressure_threshold_tokens": 768, - "fallback_output_token_cap": 2048 + "output_token_cap": 4096, + "reasoning_policy": "native-minimum", + "pre_full_panel_cap_pressure_threshold_tokens": 3072, + "fallback_output_token_cap": 8192 }, "status": "retired-fixed-safety-cap" } diff --git a/config/publication_protocol.json b/config/publication_protocol.json index eff4551..e9c9218 100644 --- a/config/publication_protocol.json +++ b/config/publication_protocol.json @@ -29,7 +29,7 @@ "reason": "The first Luna smoke and 1,024-token cell exposed phase-ambiguous global action examples in the shared prompt scaffold. The prompt advertised draft actions outside the draft phase even though available_actions did not.", "scope": "Supersede every result with the prior scaffold fingerprint, including the completed Luna cell; preserve those raw artifacts as diagnostic evidence; restart all accepted publication evidence under one hardened scaffold fingerprint.", "selection_guard": "The revised model registry remains provisional until every changed route passes the common parameter, privacy, JSON, and reasoning-disabled smoke checks.", - "compute_guard": "The retired four-cap sweep may not run. All ten registered models must pass the common 1,024-token smoke and the predeclared cap-pressure check before any full-panel result." + "compute_guard": "The retired four-cap sweep may not run. All ten phase-one registered models must pass the common 4,096-token smoke and the predeclared cap-pressure check before any full-panel result. Kimi K3 and the unavailable Nemotron 3 Ultra and DeepSeek V4 Pro routes are retained as exclusion evidence and may not be rerun in phase one." }, "rerun_policy": { "maximum_infrastructure_attempts_per_cell": 2, @@ -72,7 +72,7 @@ "unit_of_inference": "Eight public seeds are the sample; three repeats per seed are nested within seeds and only reduce model-sampling noise. All per-seed lifts are reported.", "primary_contrast": "One per model: the paired lift versus the named pick-trader baseline, using the per-seed candidate mean over three repeats minus the same seed's pick-trader score.", "headline_claim_contrast": "paired lift versus pick-trader", - "multiplicity": "Holm-Bonferroni correction across the 10 registered models for the primary contrast family; per-model uncorrected p-values are descriptive only.", + "multiplicity": "Holm-Bonferroni correction across the 10 phase-one registered models for the primary contrast family; per-model uncorrected p-values are descriptive only.", "inference_labels": "Sign-flip permutation p-values are exact only under a symmetry assumption and are labeled descriptive; the n=8 percentile-bootstrap CI is descriptive and its significant_at_95 flag must not be used as a headline claim.", "ranking_rule": "Publish tiers, not ordinal ranks. Models whose Holm-adjusted pick-trader primary contrasts and paired-lift intervals overlap are reported as one tier; no model-vs-model superiority claim without a predeclared contrast.", "power_disclosure": "The predeclared minimum detectable difference (~62 points from docs/scoring_calibration.md) exceeds the oracle-to-pick-trader headroom (~19.5), so a null result between adjacent models is uninformative and must be reported as such.", diff --git a/config/sota_v2_lane.json b/config/sota_v2_lane.json index 736544c..01f938a 100644 --- a/config/sota_v2_lane.json +++ b/config/sota_v2_lane.json @@ -8,13 +8,15 @@ "model_registry": "config/sota_v2_models.json", "smoke_manifest": "config/sota_v2_smoke_manifest.json", "minimum_headline_models": 8, - "output_token_cap": 1024, - "output_budget_status": "frozen-fixed-budget", - "output_policy_basis": "fixed-safety-ceiling", + "reasoning_policy": "native-minimum", + "reasoning_policy_detail": "Disable reasoning where optional; where OpenRouter marks reasoning mandatory, use the lowest supported effort. Kimi K3 is excluded from phase one after its mandatory-max smoke failed and triggered cap pressure.", + "output_token_cap": 4096, + "output_budget_status": "frozen-native-reasoning-cap", + "output_policy_basis": "common-safety-ceiling-with-native-minimum-reasoning", "cap_pressure_threshold_fraction": 0.75, - "cap_pressure_threshold_tokens": 768, - "fallback_output_token_cap": 2048, - "cap_pressure_rule": "Before any full-panel result is run, raise the common lane cap to 2048 if any registered-model smoke emits at least 768 output tokens in one API call or shows provider or adapter evidence of cap-induced truncation. Do not change the cap after full-panel scores are visible.", + "cap_pressure_threshold_tokens": 3072, + "fallback_output_token_cap": 8192, + "cap_pressure_rule": "Before any full-panel result is run, raise the common lane cap to 8192 if any registered-model smoke emits at least 3072 total output tokens in one API call or shows provider or adapter evidence of cap-induced truncation. Total output includes reported reasoning tokens. Do not change the cap after full-panel scores are visible.", "token_efficiency_policy": "Report observed input tokens, output tokens, cost, and latency per decision as secondary efficiency metrics; do not include them in the benchmark score.", "cli_harness_policy": "separate-table", "cost_policy": "required-for-api-null-for-subscription-cli" diff --git a/config/sota_v2_models.json b/config/sota_v2_models.json index c27ee46..2f1957a 100644 --- a/config/sota_v2_models.json +++ b/config/sota_v2_models.json @@ -1,5 +1,5 @@ { - "schema_version": 1, + "schema_version": 2, "contract": "sota-v2", "lane": "api", "provider": "openrouter", @@ -7,42 +7,209 @@ "session": false, "preset": "leaderboard", "repeats": 3, - "selection_status": "provisional-awaiting-route-smokes", - "selection_frozen_at_utc": null, - "catalog_checked_at_utc": "2026-07-15T20:41:27Z", - "selection_policy": "One current model per major compatible family, plus one small open efficiency floor; exact model IDs, dated endpoint names, and one pinned OpenRouter upstream per model; no provider fallback, JSON response mode, reasoning explicitly disabled with reasoning.enabled=false, a common 1,024-token safety ceiling, one bounded protocol repair.", + "selection_status": "frozen", + "selection_frozen_at_utc": "2026-07-17T19:43:55Z", + "selection_revision": "2026-07-17-phase-one-after-smoke-exclusions", + "catalog_checked_at_utc": "2026-07-17T19:37:57Z", + "selection_policy": "User-curated current frontier panel: five large American-lab proprietary models and five leading open-weight models. Every OpenRouter route pins an exact provider slug, endpoint tag, and dated endpoint name with fallbacks disabled. Models run at native-minimum reasoning: disabled when optional and the lowest catalog-supported effort when mandatory. Kimi K3 is parked after its clean smoke failed and demonstrated cap pressure; the exact Nemotron 3 Ultra and DeepSeek V4 Pro routes are parked after bounded HTTP 404 infrastructure failures.", "models": [ - {"id": "openrouter-gpt-5.6-luna-openai", "provider": "openrouter", "model": "openai/gpt-5.6-luna", "transport": "gateway-api", "upstream_provider": "OpenAI", "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", "role": "frontier OpenAI anchor"}, - {"id": "openrouter-claude-sonnet-5-anthropic", "provider": "openrouter", "model": "anthropic/claude-sonnet-5", "transport": "gateway-api", "upstream_provider": "Anthropic", "endpoint_name": "Anthropic | anthropic/claude-sonnet-5-20260630", "role": "frontier Anthropic anchor"}, - {"id": "openrouter-deepseek-v4-pro-deepinfra", "provider": "openrouter", "model": "deepseek/deepseek-v4-pro", "transport": "gateway-api", "upstream_provider": "DeepInfra", "endpoint_name": "DeepInfra | deepseek/deepseek-v4-pro-20260423", "role": "DeepSeek family anchor"}, - {"id": "openrouter-kimi-k2.6-deepinfra", "provider": "openrouter", "model": "moonshotai/kimi-k2.6", "transport": "gateway-api", "upstream_provider": "DeepInfra", "endpoint_name": "DeepInfra | moonshotai/kimi-k2.6-20260420", "role": "Moonshot family anchor"}, - {"id": "openrouter-glm-5.2-streamlake", "provider": "openrouter", "model": "z-ai/glm-5.2", "transport": "gateway-api", "upstream_provider": "StreamLake", "endpoint_name": "StreamLake | z-ai/glm-5.2-20260616", "role": "Z.ai family anchor"}, - {"id": "openrouter-qwen3.7-plus-alibaba", "provider": "openrouter", "model": "qwen/qwen3.7-plus", "transport": "gateway-api", "upstream_provider": "Alibaba", "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", "role": "large Qwen family anchor"}, - {"id": "openrouter-minimax-m3-minimax", "provider": "openrouter", "model": "minimax/minimax-m3", "transport": "gateway-api", "upstream_provider": "Minimax", "endpoint_name": "Minimax | minimax/minimax-m3-20260531", "role": "capable low-cost anchor"}, - {"id": "openrouter-mistral-medium-3.5-mistral", "provider": "openrouter", "model": "mistralai/mistral-medium-3-5", "transport": "gateway-api", "upstream_provider": "Mistral", "endpoint_name": "Mistral | mistralai/mistral-medium-3.5-20260430", "role": "European model-family anchor"}, - {"id": "openrouter-nemotron-3-nano-deepinfra", "provider": "openrouter", "model": "nvidia/nemotron-3-nano-30b-a3b", "transport": "gateway-api", "upstream_provider": "DeepInfra", "endpoint_name": "DeepInfra | nvidia/nemotron-3-nano-30b-a3b", "role": "small NVIDIA open-weight anchor"}, - {"id": "openrouter-qwen3.5-9b-deepinfra", "provider": "openrouter", "model": "qwen/qwen3.5-9b", "transport": "gateway-api", "upstream_provider": "DeepInfra", "endpoint_name": "DeepInfra | qwen/qwen3.5-9b-20260310", "role": "small open efficiency floor"} + { + "id": "openrouter-gpt-5.6-luna-openai", + "provider": "openrouter", + "model": "openai/gpt-5.6-luna", + "transport": "gateway-api", + "cohort": "big-american-lab-proprietary", + "upstream_provider": "OpenAI", + "upstream_provider_slug": "openai", + "endpoint_tag": "openai", + "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "OpenAI frontier anchor" + }, + { + "id": "openrouter-claude-sonnet-5-bedrock", + "provider": "openrouter", + "model": "anthropic/claude-sonnet-5", + "transport": "gateway-api", + "cohort": "big-american-lab-proprietary", + "upstream_provider": "Amazon Bedrock", + "upstream_provider_slug": "amazon-bedrock/global", + "endpoint_tag": "amazon-bedrock/global", + "endpoint_name": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "Anthropic frontier anchor; healthy global Bedrock route replaces unhealthy direct Anthropic endpoint" + }, + { + "id": "openrouter-gemini-3.5-flash-google-ai-studio", + "provider": "openrouter", + "model": "google/gemini-3.5-flash", + "transport": "gateway-api", + "cohort": "big-american-lab-proprietary", + "upstream_provider": "Google AI Studio", + "upstream_provider_slug": "google-ai-studio", + "endpoint_tag": "google-ai-studio", + "endpoint_name": "Google AI Studio | google/gemini-3.5-flash-20260519", + "reasoning_policy": "mandatory-minimum", + "reasoning_effort": "minimal", + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "true", "OPENROUTER_REASONING_EFFORT": "minimal"}, + "absent_options": [], + "role": "Google fast frontier anchor" + }, + { + "id": "openrouter-grok-4.5-xai", + "provider": "openrouter", + "model": "x-ai/grok-4.5", + "transport": "gateway-api", + "cohort": "big-american-lab-proprietary", + "upstream_provider": "xAI", + "upstream_provider_slug": "xai", + "endpoint_tag": "xai", + "endpoint_name": "xAI | x-ai/grok-4.5-20260708", + "reasoning_policy": "mandatory-minimum", + "reasoning_effort": "low", + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "true", "OPENROUTER_REASONING_EFFORT": "low"}, + "absent_options": [], + "role": "xAI frontier anchor" + }, + { + "id": "openrouter-muse-spark-1.1-meta", + "provider": "openrouter", + "model": "meta/muse-spark-1.1", + "transport": "gateway-api", + "cohort": "big-american-lab-proprietary", + "upstream_provider": "Meta", + "upstream_provider_slug": "meta", + "endpoint_tag": "meta", + "endpoint_name": "Meta | meta/muse-spark-1.1-20260709", + "reasoning_policy": "mandatory-minimum", + "reasoning_effort": "minimal", + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "true", "OPENROUTER_REASONING_EFFORT": "minimal"}, + "absent_options": [], + "role": "Meta proprietary frontier anchor" + }, + { + "id": "openrouter-glm-5.2-zai", + "provider": "openrouter", + "model": "z-ai/glm-5.2", + "transport": "gateway-api", + "cohort": "open-weight", + "upstream_provider": "Z.AI", + "upstream_provider_slug": "z-ai/fp8", + "endpoint_tag": "z-ai/fp8", + "endpoint_name": "Z.AI | z-ai/glm-5.2-20260616", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "Z.ai open-weight anchor on first-party FP8 route" + }, + { + "id": "openrouter-minimax-m3-minimax", + "provider": "openrouter", + "model": "minimax/minimax-m3", + "transport": "gateway-api", + "cohort": "open-weight", + "upstream_provider": "Minimax", + "upstream_provider_slug": "minimax/fp8", + "endpoint_tag": "minimax/fp8", + "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "MiniMax open-weight anchor on first-party FP8 route" + }, + { + "id": "openrouter-qwen3.7-plus-alibaba", + "provider": "openrouter", + "model": "qwen/qwen3.7-plus", + "transport": "gateway-api", + "cohort": "open-weight", + "upstream_provider": "Alibaba", + "upstream_provider_slug": "alibaba", + "endpoint_tag": "alibaba", + "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "Qwen open-weight frontier anchor" + }, + { + "id": "openrouter-mistral-medium-3.5-mistral", + "provider": "openrouter", + "model": "mistralai/mistral-medium-3-5", + "transport": "gateway-api", + "cohort": "open-weight", + "upstream_provider": "Mistral", + "upstream_provider_slug": "mistral", + "endpoint_tag": "mistral", + "endpoint_name": "Mistral | mistralai/mistral-medium-3.5-20260430", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "Mistral open-weight European anchor" + }, + { + "id": "openrouter-tencent-hy3-free-novita", + "provider": "openrouter", + "model": "tencent/hy3:free", + "transport": "gateway-api", + "cohort": "open-weight", + "upstream_provider": "Novita", + "upstream_provider_slug": "novita", + "endpoint_tag": "novita", + "endpoint_name": "Novita | tencent/hy3-20260706:free", + "catalog_expiration_date": "2026-07-21", + "reasoning_policy": "disabled", + "reasoning_effort": null, + "fixed_options": { + "OPENROUTER_REASONING_ENABLED": "false", + "OPENROUTER_JSON_MODE": "false" + }, + "absent_options": ["OPENROUTER_REASONING_EFFORT"], + "role": "Tencent open-weight frontier anchor on OpenRouter's temporary free Novita route; prompt-only JSON because the endpoint does not advertise response_format" + } ], - "changed_routes_pending_smoke": [ - "openrouter-deepseek-v4-pro-deepinfra", - "openrouter-kimi-k2.6-deepinfra", - "openrouter-nemotron-3-nano-deepinfra" + "future_models": [ + {"model": "openai/gpt-5.6-terra", "status": "later date"}, + {"model": "openai/gpt-5.6-sol", "status": "later date"}, + {"model": "anthropic/claude-opus-4.8", "status": "later date"}, + {"model": "anthropic/fable-5", "status": "later date"} ], + "changed_routes_pending_smoke": [], "required_smokes": [ "openrouter-gpt-5.6-luna-openai", - "openrouter-claude-sonnet-5-anthropic", - "openrouter-deepseek-v4-pro-deepinfra", - "openrouter-kimi-k2.6-deepinfra", - "openrouter-glm-5.2-streamlake", - "openrouter-qwen3.7-plus-alibaba", + "openrouter-claude-sonnet-5-bedrock", + "openrouter-gemini-3.5-flash-google-ai-studio", + "openrouter-grok-4.5-xai", + "openrouter-muse-spark-1.1-meta", + "openrouter-glm-5.2-zai", "openrouter-minimax-m3-minimax", + "openrouter-qwen3.7-plus-alibaba", "openrouter-mistral-medium-3.5-mistral", - "openrouter-nemotron-3-nano-deepinfra", - "openrouter-qwen3.5-9b-deepinfra" + "openrouter-tencent-hy3-free-novita" ], "explicit_exclusions": [ - {"model": "google/gemini-3.1-pro-preview", "reason": "OpenRouter catalog marks reasoning mandatory; incompatible with the reasoning-disabled headline lane"}, - {"model": "x-ai/grok-4.5", "reason": "OpenRouter catalog marks reasoning mandatory; incompatible with the reasoning-disabled headline lane"} + { + "model": "moonshotai/kimi-k3", + "reason": "Phase-one smoke at mandatory max reasoning hit the 4,096-token ceiling in two calls, produced two truncated responses and two failed decisions, and cost $0.301296. Preserve the raw artifact as diagnostic evidence; do not rerun under the under-$100 phase-one budget." + }, + { + "model": "nvidia/nemotron-3-ultra-550b-a55b:free", + "reason": "The exact healthy Nvidia free route passed catalog and authentication preflight but returned HTTP 404 twice on real chat-completion calls. Preserve the checkpoint as infrastructure evidence; do not retry or substitute the paid route in phase one." + }, + { + "model": "deepseek/deepseek-v4-pro", + "reason": "The exact healthy first-party route passed catalog and authentication preflight but returned HTTP 404 twice on real chat-completion calls. Preserve the checkpoint as infrastructure evidence; do not retry or substitute another route in phase one." + } ], "shared_fixed_options": { "OPENROUTER_PROVIDER_SORT": "price", @@ -50,17 +217,15 @@ "OPENROUTER_REQUIRE_PARAMETERS": "true", "OPENROUTER_DATA_COLLECTION": "deny", "OPENROUTER_JSON_MODE": "true", - "OPENROUTER_REASONING_ENABLED": "false", "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1" }, "shared_absent_options": [ - "OPENROUTER_REASONING_EFFORT", "OPENROUTER_REASONING_MAX_TOKENS", "OPENROUTER_TEMPERATURE", "OPENROUTER_QUANTIZATIONS", "OPENROUTER_ZDR" ], - "output_token_cap": 1024, - "output_budget_status": "frozen-fixed-budget", - "output_policy_basis": "fixed-safety-ceiling" + "output_token_cap": 4096, + "output_budget_status": "frozen-native-reasoning-cap", + "output_policy_basis": "common-safety-ceiling-with-native-minimum-reasoning" } diff --git a/config/sota_v2_smoke_manifest.json b/config/sota_v2_smoke_manifest.json new file mode 100644 index 0000000..17d7b60 --- /dev/null +++ b/config/sota_v2_smoke_manifest.json @@ -0,0 +1,276 @@ +{ + "entries": { + "openrouter-claude-sonnet-5-bedrock": { + "accepted": true, + "api_calls": 5, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-claude-sonnet-5-bedrock--4096.json", + "artifact_sha256": "fad63a07bf2ab43fb2a651a320856e24f533b56f8964eadb0d7b8fa02b456d09", + "calls_with_finish_reason": 5, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Amazon Bedrock | anthropic/claude-sonnet-5-20260630", + "endpoint_tag": "amazon-bedrock/global", + "max_output_tokens_per_call": 604, + "model": "anthropic/claude-sonnet-5", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:24+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Amazon Bedrock", + "upstream_provider_slug": "amazon-bedrock/global" + }, + "openrouter-gemini-3.5-flash-google-ai-studio": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-gemini-3.5-flash-google-ai-studio--4096.json", + "artifact_sha256": "275c6ad7fc0bb4da812f9d72b5ebdf110285d6a842dfc7261427292c6ddad4a3", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Google AI Studio | google/gemini-3.5-flash-20260519", + "endpoint_tag": "google-ai-studio", + "max_output_tokens_per_call": 107, + "model": "google/gemini-3.5-flash", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": "minimal", + "reasoning_policy": "mandatory-minimum", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:24+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Google AI Studio", + "upstream_provider_slug": "google-ai-studio" + }, + "openrouter-glm-5.2-zai": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-glm-5.2-zai--4096.json", + "artifact_sha256": "37da29291723f868a10668cbbe9c667108693800227d56f058db38a1ec61cb18", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Z.AI | z-ai/glm-5.2-20260616", + "endpoint_tag": "z-ai/fp8", + "max_output_tokens_per_call": 311, + "model": "z-ai/glm-5.2", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Z.AI", + "upstream_provider_slug": "z-ai/fp8" + }, + "openrouter-gpt-5.6-luna-openai": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-gpt-5.6-luna-openai--4096.json", + "artifact_sha256": "9a72e0d69debdd70b7494e1d5c4d401638d136cf03293ad44e33dd2fe38e7776", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", + "endpoint_tag": "openai", + "max_output_tokens_per_call": 141, + "model": "openai/gpt-5.6-luna", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:24+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "OpenAI", + "upstream_provider_slug": "openai" + }, + "openrouter-grok-4.5-xai": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-grok-4.5-xai--4096.json", + "artifact_sha256": "6939a0db513033a3b4a324f09fe9cfba171458f71f4c548ff18a02124210308e", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "xAI | x-ai/grok-4.5-20260708", + "endpoint_tag": "xai", + "max_output_tokens_per_call": 937, + "model": "x-ai/grok-4.5", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": "low", + "reasoning_policy": "mandatory-minimum", + "reasoning_tokens": 2064, + "recorded_at_utc": "2026-07-17T19:43:24+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "xAI", + "upstream_provider_slug": "xai" + }, + "openrouter-minimax-m3-minimax": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-minimax-m3-minimax--4096.json", + "artifact_sha256": "7148060c76960df4f82b425dfcc64a4760e2cc7471a02d631c3dc4463e4fc402", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Minimax | minimax/minimax-m3-20260531", + "endpoint_tag": "minimax/fp8", + "max_output_tokens_per_call": 260, + "model": "minimax/minimax-m3", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Minimax", + "upstream_provider_slug": "minimax/fp8" + }, + "openrouter-mistral-medium-3.5-mistral": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-mistral-medium-3.5-mistral--4096.json", + "artifact_sha256": "900f45ad4bd7953cb72eac7fd66a5b704f241bcf6d6798b2bf4b483c6d91c6f8", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Mistral | mistralai/mistral-medium-3.5-20260430", + "endpoint_tag": "mistral", + "max_output_tokens_per_call": 201, + "model": "mistralai/mistral-medium-3-5", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Mistral", + "upstream_provider_slug": "mistral" + }, + "openrouter-muse-spark-1.1-meta": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-muse-spark-1.1-meta--4096.json", + "artifact_sha256": "b1e037350024c5a4b464624ce9a7f822a49fd22676e24db2b385152fb17e0350", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Meta | meta/muse-spark-1.1-20260709", + "endpoint_tag": "meta", + "max_output_tokens_per_call": 1432, + "model": "meta/muse-spark-1.1", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": "minimal", + "reasoning_policy": "mandatory-minimum", + "reasoning_tokens": 4444, + "recorded_at_utc": "2026-07-17T19:43:24+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Meta", + "upstream_provider_slug": "meta" + }, + "openrouter-qwen3.7-plus-alibaba": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-qwen3.7-plus-alibaba--4096.json", + "artifact_sha256": "910d5614f0f34251346a420377a1bb6575e569241bc582cc7008a99c89d707b9", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Alibaba | qwen/qwen3.7-plus-20260602", + "endpoint_tag": "alibaba", + "max_output_tokens_per_call": 266, + "model": "qwen/qwen3.7-plus", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Alibaba", + "upstream_provider_slug": "alibaba" + }, + "openrouter-tencent-hy3-free-novita": { + "accepted": true, + "api_calls": 4, + "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-tencent-hy3-free-novita--4096.json", + "artifact_sha256": "0201c570fa47915bf61133027b00b04410da7cbb8f638f3a05a75d38ae7875c2", + "calls_with_finish_reason": 4, + "contract_fingerprint": "558e8f35ea1d66b9", + "cost_decisions": 4, + "decision_failure_rate": 0.0, + "decisions_with_usage": 4, + "endpoint_name": "Novita | tencent/hy3-20260706:free", + "endpoint_tag": "novita", + "max_output_tokens_per_call": 150, + "model": "tencent/hy3:free", + "output_token_cap": 4096, + "protocol_repair_attempts": 0, + "protocol_repairs_succeeded": 0, + "provider": "openrouter", + "reasoning_effort": null, + "reasoning_policy": "disabled", + "reasoning_tokens": 0, + "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "scaffold_fingerprint": "d7321ad9d0a739b4", + "truncated_calls": 0, + "upstream_provider": "Novita", + "upstream_provider_slug": "novita" + } + }, + "format": "gm-bench-smoke-manifest-v1", + "schema_version": 1 +} diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 4827880..2663203 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -7,13 +7,15 @@ > to preserve this first draft; the goal is to make it more accurate as the > project develops. -**Last reviewed:** 2026-07-16 +**Last reviewed:** 2026-07-18 **Current target:** Publish a validated `sota-v2` leaderboard, accompanying blog post, GitHub release, and public site. -**Current state:** Scaffold hardening and the fixed output-safety policy merged -in #66 (`c500e26`); #70 and #71 subsequently landed the machine-enforced smoke -gate. The ten-model registry remains provisional, and no accepted smoke manifest -or eligible panel row exists. +**Current state:** The phase-one ten-model registry is frozen as of 2026-07-17, +uses exact OpenRouter endpoint tags, and completed its fresh 4,096-token +native-minimum-reasoning smoke gate; all ten registered routes are accepted with +zero failures, zero truncations, and no cap-pressure trigger. Earlier 1,024-token +smokes are retained only as superseded operational evidence. The full panel run +is the remaining blocker before any eligible model row exists. **Current weekly focus:** [#63 — Publication sprint: freeze and ship GM-Bench `sota-v2`](https://github.com/nedcut/gm-bench/issues/63) **Broader roadmap:** [#60 — Roadmap to a publishable leaderboard + blog @@ -26,8 +28,9 @@ scaffolds on synthetic, long-horizon resource allocation in a frozen sports management environment. It is already strong enough to be a flagship AI/ML portfolio project. It is not yet ready for a headline model-ranking claim because the current `sota-v2` leaderboard has no eligible model rows. The -compute policy is now frozen, but every registered route must pass the common -smoke gate before any full-panel result is generated. +compute policy is frozen at a 4,096-token native-minimum-reasoning ceiling, and +every registered route has passed the common smoke gate; the remaining blocker +is running the paid full panel itself. The strongest story is not merely that GM-Bench runs LLMs through a simulator. It is that the project: @@ -69,7 +72,7 @@ The project is publish-ready only when all four gates pass. | Core engineering | Strong | Deterministic simulator, adapters, CLI, GUI, site, tests, and CI are substantial. | | Reproducibility | Strong | Contract fingerprints, seed provenance, compact artifacts, and validators are in place. | | Benchmark validity | Strong but scoped | Scripted references, exploit canaries, oracle headroom, calibration, and mechanic coverage exist. | -| Compute comparability | Policy frozen; smoke validation pending | The API lane has a common 1,024-token safety ceiling, reasoning disabled, exact routes, a pre-full-panel 75% cap-pressure rule, and actual token-efficiency reporting. All ten current routes still require standardized smokes. | +| Compute comparability | Frozen for phase one | The API lane has a common 4,096-token total-output ceiling, native-minimum reasoning, exact provider slugs and endpoint tags, a pre-full-panel 75% cap-pressure rule, and actual reasoning/token-efficiency reporting. All ten phase-one routes passed and were accepted; Kimi K3 and the unavailable Nemotron and DeepSeek routes are retained as exclusion evidence. | | Current model evidence | Blocked | The active `sota-v2` leaderboard has no eligible model rows. | | Statistical evidence | Partially ready | Paired analysis and power tooling exist; final model results do not. | | External validation | Missing | No independent reproduction or third-party result has been recorded. | @@ -127,31 +130,31 @@ before any full-panel score is visible. explicitly disposition every substantive finding before expensive runs begin. - [x] Retain the earlier three-model/four-cap design and analyzer as auditable history without treating it as an active publication prerequisite. -- [x] Freeze common lane and scaffold conditions: reasoning disabled, - temperature omitted, hardened scaffold, compact profile, one repair, and - fresh-spawn execution. -- [ ] Freeze exact model routes and the registry only after all ten smokes are +- [x] Freeze common scaffold conditions: temperature omitted, hardened + scaffold, compact profile, one repair, and fresh-spawn execution. +- [x] Freeze exact model routes and the registry only after all ten phase-one smokes are accepted. -- [x] Freeze a common 1,024-token safety ceiling with reasoning disabled. -- [x] Predeclare the primary endpoint and the 75% cap-pressure rule: raise the - entire lane to 2,048 before full results if any smoke call reaches 768 output - tokens or shows cap-induced truncation. +- [ ] Validate the provisional common 4,096-token safety ceiling with + native-minimum reasoning. +- [x] Predeclare exact provider slugs/tags and the 75% cap-pressure rule: raise + the entire lane to 8,192 before full results if any smoke call reaches 3,072 + total output tokens or shows cap-induced truncation. - [x] Freeze the permitted retry conditions, exclusion rules, and stopping rule in `config/publication_protocol.json` before seeing sweep outcomes. - [x] Distinguish infrastructure/provider failures that permit a resumed run from poor model behavior that must remain part of the measured result. -- [ ] Smoke all ten provisional models serially at 1,024 using the hardened +- [x] Smoke all ten provisional phase-one models serially at 4,096 using the hardened scaffold. Start with dry-run and endpoint preflight; then run one paid model, inspect it, and approve the next rather than launching the set blindly. -- [ ] Verify each smoke's exact route, JSON behavior, reasoning-off provenance, +- [ ] Verify each smoke's exact route, JSON behavior, registered reasoning provenance, repair/failure telemetry, cost coverage, per-call output distribution, and absence of truncation. Then record it with `run_publication_matrix.py record-smoke` so the machine-enforced manifest accepts it; the panel phase refuses to run until every registered model has an accepted entry. -- [ ] If the cap-pressure rule fires, amend the whole lane to 2,048 and repeat +- [ ] If the cap-pressure rule fires, amend the whole lane to 8,192 and repeat affected smokes before any full-panel result. Do not make this decision after seeing full scores. -- [ ] Freeze the provisional model registry only after all ten routes pass. +- [x] Freeze the provisional model registry only after all ten phase-one routes pass. - [ ] Re-estimate and record expected full-panel cost, runtime, serial concurrency, and quota after the smokes. The existing cost artifact describes the retired 12-cell plan and is not current spend guidance. @@ -159,20 +162,20 @@ before any full-panel score is visible. - [x] Confirm in automated tests that the retired analyzer still rejects missing, duplicate, mixed-provenance, wrong-route, wrong-lane, wrong-repeat, incomplete-telemetry, and invalid-contract cells. -- [x] Update `config/sota_v2_lane.json` to `frozen-fixed-budget` with the explicit - `fixed-safety-ceiling` basis. +- [ ] Update `config/sota_v2_lane.json` to `frozen-native-reasoning-cap` only + after the ten phase-one smokes clear the cap-pressure audit. - [x] Record the rationale, decision date, pre-full-panel trigger, and known limitations in the decision log below. - [ ] Regenerate the site and confirm it still refuses to publish a ranking if any publication prerequisite is missing. **Exit condition:** the official API lane has a documented fixed-cap policy, -all ten registered-model smokes pass without a cap-pressure trigger, the model +all ten phase-one registered-model smokes pass without a cap-pressure trigger, the model registry is frozen, and the full-panel cost plan is refreshed. #### Safe execution workflow -All model calls are serial. Inspect all ten smoke commands and their non-secret +All model calls are serial. Inspect all ten phase-one smoke commands and their non-secret provider options first. ```bash @@ -195,24 +198,38 @@ usage and cap pressure after every cell: ```bash python3 scripts/run_publication_matrix.py smoke \ - --model-id openrouter-qwen3.5-9b-deepinfra \ - --run-dir data/publication-runs/smoke-fixed-1024-2026-07-15 \ + --model-id openrouter-qwen3.7-plus-alibaba \ + --run-dir data/publication-runs/smoke-native-4096-2026-07-16 \ --max-spend-usd 5 ``` +In a second terminal, use the read-only monitor for live episode progress, +active/interrupted state, accepted-smoke state, artifact-reported cost, and +reserved spend. The runner checkpoints after every seed/repeat episode, so the +display updates at that durable boundary: + +```bash +python3 scripts/run_publication_matrix.py status \ + --run-dir data/publication-runs/smoke-native-4096-2026-07-16 \ + --watch +``` + +Omit `--watch` for a one-shot status table, or add `--json` for machine-readable +status suitable for logs and other wrappers. + After each smoke passes inspection, record it so the machine-enforced manifest accepts the route. The command validates route, options, fingerprints, zero failures, complete finish-reason coverage, absence of truncation, and peak -output tokens below 768 before writing an accepted entry to +total output tokens below 3,072 before writing an accepted entry to `config/sota_v2_smoke_manifest.json`: ```bash python3 scripts/run_publication_matrix.py record-smoke \ - --model-id openrouter-qwen3.5-9b-deepinfra \ - --artifact data/publication-runs/smoke-fixed-1024-2026-07-15/raw/openrouter-qwen3.5-9b-deepinfra--1024.json + --model-id openrouter-qwen3.7-plus-alibaba \ + --artifact data/publication-runs/smoke-native-4096-2026-07-16/raw/openrouter-qwen3.7-plus-alibaba--4096.json ``` -Repeat one registered model at a time. Only after all ten standardized smokes, +Repeat one registered model at a time. Only after all ten phase-one standardized smokes, the cap-pressure audit, and the refreshed cost estimate are acceptable should the model registry be frozen and the full panel begin. The `panel` phase now refuses to run unless every registered model has an accepted manifest entry; @@ -222,7 +239,7 @@ checkpoint already exists, and refuses to fan out workers: ```bash python3 scripts/run_publication_matrix.py panel \ - --run-dir data/publication-runs/sota-v2-fixed-1024 \ + --run-dir data/publication-runs/sota-v2-native-4096 \ --max-spend-usd ``` @@ -231,7 +248,7 @@ The driver enforces that lock. ### Phase 2 — run the publishable model panel -- [ ] Freeze the revised 10-model provider/model/route registry in +- [x] Freeze the revised 10-model phase-one provider/model/route registry in `config/sota_v2_models.json` after changed-route smokes pass. - [x] Pre-register the full-panel rerun and exclusion policy. A disappointing valid result is not a reason to rerun a model. @@ -243,22 +260,21 @@ The provisional restarted headline panel is: | Model | Pinned upstream | Panel role | | --- | --- | --- | | GPT-5.6 Luna | OpenAI | Frontier OpenAI anchor | -| Claude Sonnet 5 | Anthropic | Frontier Anthropic anchor | -| DeepSeek V4 Pro | DeepInfra | DeepSeek-family anchor; changed route requires smoke | -| Kimi K2.6 | DeepInfra | Moonshot-family anchor; changed route requires smoke | -| GLM 5.2 | StreamLake | Z.ai-family anchor | -| Qwen 3.7 Plus | Alibaba | Large Qwen-family comparison | -| Mistral Medium 3.5 | Mistral | Mid-tier European-family comparison | -| MiniMax M3 | MiniMax | Capable low-cost model | -| Nemotron 3 Nano 30B A3B | DeepInfra | Small NVIDIA open-weight anchor; changed route requires smoke | -| Qwen 3.5 9B | DeepInfra | Small open efficiency floor | - -Gemini 3.1 Pro Preview and Grok 4.5 remain explicit exclusions because the live -OpenRouter catalog marks reasoning as mandatory for both; mixing them into a -reasoning-disabled headline lane would change the measured system. Kimi and -DeepSeek now expose non-mandatory reasoning and compatible DeepInfra routes in -the catalog. They are provisional until a paid smoke proves the common JSON, -privacy, pinned-provider, and `reasoning.enabled=false` request end to end. +| Claude Sonnet 5 | Amazon Bedrock global | Anthropic frontier anchor; direct Anthropic route was unhealthy at revision time | +| Gemini 3.5 Flash | Google AI Studio | Google fast frontier anchor; mandatory `minimal` reasoning | +| Grok 4.5 | xAI | xAI frontier anchor; mandatory `low` reasoning | +| Muse Spark 1.1 | Meta | Meta frontier anchor; mandatory `minimal` reasoning | +| GLM 5.2 | Z.AI FP8 | First-party open-weight anchor | +| MiniMax M3 | MiniMax FP8 | First-party open-weight anchor | +| Qwen 3.7 Plus | Alibaba | Qwen frontier open-weight anchor | +| Mistral Medium 3.5 | Mistral | European open-weight anchor | +| Tencent HY3 | Novita free | Temporary free route expiring July 21; prompt-only JSON because the endpoint does not advertise `response_format` | + +The mixed reasoning policy is explicit rather than hidden: reasoning is off for +the seven optional-reasoning models and set to the lowest catalog-supported +effort for the three mandatory-reasoning models. Scores remain comparable as +model-plus-native-inference systems, while reasoning tokens, cost, and latency +must be reported beside score. - [x] Record exact model identifiers, endpoint snapshot names, and upstream routes; never collapse distinct snapshots @@ -267,8 +283,8 @@ privacy, pinned-provider, and `reasoning.enabled=false` request end to end. frozen output policy. - [x] Keep coding-agent CLI harnesses in a separate diagnostic table. - [ ] Never parallelize Claude or another subscription/rate-limited CLI. -- [ ] Verify all 10 provisional provider/model routes can accept the common - privacy, parameter, JSON, reasoning-off, and bounded-output policy. +- [x] Verify all 10 provisional phase-one provider/model routes can accept the common + privacy, parameter, JSON, registered reasoning, and bounded-output policy. - [ ] Run a benchmark-level smoke for every provider/model combination at the shared frozen cap immediately before the full panel. - [ ] Use serial execution, fail-fast behavior, atomic checkpoints, and validated @@ -392,7 +408,7 @@ Claims to avoid: - [ ] Lead the blog with the research question and the measurement problem, not with implementation history. - [ ] Explain the simulator and decision loop with one compact diagram. -- [ ] Explain the fixed 1,024-token safety policy and cap-pressure audit before +- [ ] Explain the provisional 4,096-token native-reasoning policy and cap-pressure audit before the final model ranking. - [ ] Show Oracle → `pick-trader` → best eligible model → `random` headroom. - [ ] Include cost and compute beside score in every model table. @@ -537,6 +553,14 @@ decision and why. | 2026-07-15 | Machine-enforce the pre-panel smoke gate and unique-row counting. | Review found the panel and ranking were unlockable by editing status strings and by row aliasing: `selection_status` "frozen" was accepted as smoke completion, and duplicate aliases for one model could satisfy the eight-row floor. | Panel and `publishable_ranking` now require recorded, accepted smoke-manifest entries per registered model, count by unique registered model identity, and can never require fewer rows than the protocol's pre-registered minimum floor. | | 2026-07-15 | Re-fingerprint the v2 contract and OpenRouter scaffold before any accepted evidence. | `failed_queries` narrowed to unresolved lookups plus ambiguous-scout rejection changed the contract (`a65a4359ca3c6e64` → `558e8f35ea1d66b9`), and per-call `finish_reason`/`native_finish_reason` recording made cap-induced truncation auditable (scaffold `317371cf66b436fe` → `d7321ad9d0a739b4`). No accepted smoke or eligible row existed, so nothing was invalidated. | All ten route smokes must run under the new fingerprints, and the statistical analysis plan is frozen pre-data in `config/publication_protocol.json`. | | 2026-07-16 | Amend the headline contrast to paired lift versus pick-trader. | The full baseline-panel mean includes random and other weak references, so clearing it would not show that a model-plus-scaffold system beats the transparent competent heuristic bar. No accepted `sota-v2` smoke manifest, eligible panel row, or observed full-panel score existed when this was amended. | Pick-trader is the Holm-adjusted primary contrast; full-panel lift remains a secondary descriptive endpoint, and publication still uses tiers rather than ordinal ranks. | +| 2026-07-16 | Replace the stale ten-model panel with the user-curated twelve-model frontier panel. | No full-panel score existed. Live OpenRouter catalog and endpoint checks confirmed all requested models, but Gemini 3.5 Flash, Grok 4.5, Muse Spark 1.1, and newly released Kimi K3 require reasoning. | Reset the smoke manifest; pin provider slugs plus endpoint tags; use native-minimum reasoning and a provisional common 4,096-token cap; require all twelve fresh smokes before freezing. | +| 2026-07-17 | Add Tencent HY3 on OpenRouter's free Novita route before smoke execution. | The live catalog exposed `tencent/hy3:free` at zero input and output cost with one exact healthy Novita endpoint, optional reasoning, and a July 21 catalog expiration. The route advertises structured outputs but not the `response_format` parameter used by the other routes. No revised-panel smoke or full-panel score existed. | Expand the provisional registry and multiplicity family to thirteen, keep reasoning and JSON response mode disabled for HY3, rely on the same explicit JSON-only prompt plus clean-smoke gate, and retain the pinned dated free endpoint rather than silently falling back after it expires. | +| 2026-07-17 | Park Kimi K3 for the under-$100 phase-one panel after its first clean-gate smoke. | At mandatory `max` reasoning, two of four calls hit the 4,096-token ceiling and were truncated; the episode had two failed decisions, 13,275 reasoning tokens, 100.264 seconds per decision, and $0.301296 cost. Raising the common lane to 8,192 would invalidate the six clean 4,096-token smokes and push the conservative one-repeat plan beyond the phase-one budget. | Preserve the raw Kimi artifact as diagnostic evidence, prohibit a phase-one rerun, return the registry and Holm family to twelve models, and continue the untouched routes at 4,096. | +| 2026-07-17 | Switch Nemotron 3 Ultra from paid Together to the exact free Nvidia route before its first smoke. | OpenRouter's live catalog exposed `nvidia/nemotron-3-ultra-550b-a55b:free` at zero input/output cost on one healthy first-party Nvidia endpoint with no listed expiration. It supports optional reasoning but does not advertise `response_format`. | Pin the dated Nvidia free endpoint with fallbacks and reasoning disabled, use prompt-only JSON plus the clean-smoke gate, and regenerate the cost plan before continuing. | +| 2026-07-17 | Park the listed-free Nemotron 3 Ultra route after bounded infrastructure failure. | The exact Nvidia endpoint passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint as infrastructure evidence, do not retry or silently substitute the paid Together route, reduce the phase-one registry and Holm family to eleven, and continue the untouched models. | +| 2026-07-17 | Park DeepSeek V4 Pro after the same bounded infrastructure failure. | The exact first-party DeepSeek route passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint, do not retry or substitute a different route, reduce the phase-one registry and Holm family to ten, and continue Mistral and HY3. | +| 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Artifact-reported spend was $0.728909 against a $2 ceiling. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. | +| 2026-07-18 | Settle successful serial-cell reservations against measured spend. | The runner retained every historical worst-case reservation, so a healthy panel could stop against cumulative hypothetical spend even after completed artifacts and the OpenRouter account established a much lower real cost. | Mark successful-cell reservations settled after post-cell spend measurement, keep failed/interrupted reservations active, and evaluate each next cell against measured spend plus only unresolved liabilities. | ## Experiment and release log @@ -555,6 +579,7 @@ than pasting large outputs. | 2026-07-15 | GPT-5.6 Luna standardized smoke | Superseded diagnostic | raw SHA-256 `e8f83c6516cb3cc8105b173c826c1b5d91314a487b729f9b06b8fc6beda2bc8f` | Exact OpenAI route and complete telemetry remain useful, but all four illegal actions were preseason draft attempts primed by the old global action catalog. | | 2026-07-15 | GPT-5.6 Luna 1,024-token sweep cell | Superseded diagnostic | canonical SHA-256 `b681bdc56f3d176d194c9c1e20cc688be4ef4b58f7669862fb2268af99a0e37a`; byte SHA-256 `74d342c5d4c799524dadd6f668350eede67b8b74e5522833723e25ab4f50480b` | The run remains operationally valid and auditable, but its scaffold fingerprint is intentionally invalidated. It cannot count toward the restarted sweep or headline panel. | | 2026-07-15 | Sweep cost/runtime plan | Superseded by fixed-cap policy | `results/analysis/output-budget-cost-estimate.json` | The prior figures describe the retired 12-cell matrix. Replace with a full-panel estimate after all ten route smokes. | +| 2026-07-16 | First fixed-1,024 smoke series | Superseded by deliberate panel revision | `docs/run_logs/sota-v2-smokes-2026-07-16.md` | Six routes were accepted, two completed with protocol failures, Nemotron Nano exhausted infrastructure retries, and Claude direct remained unhealthy. The evidence remains auditable but cannot unlock the revised 4,096-token native-reasoning panel. | | 2026-07-15 | Statistical analysis plan | Frozen | `config/publication_protocol.json` | Pre-registered pre-data: unit of inference, primary paired contrast, Holm-Bonferroni multiplicity, descriptive inference labels, tiered ranking, power disclosure, temperature policy, and registry exclusion criteria. | ## Living-document maintenance checklist diff --git a/docs/output_budget_sweep.md b/docs/output_budget_sweep.md index b4bb138..a416d34 100644 --- a/docs/output_budget_sweep.md +++ b/docs/output_budget_sweep.md @@ -1,8 +1,10 @@ # Output policy and retired budget sweep -**Current policy (frozen 2026-07-15):** the `sota-v2` API headline lane uses a -common 1,024-output-token safety ceiling with reasoning disabled. Actual input -tokens, output tokens, cost, and latency are reported as secondary efficiency +**Current policy (provisional revision 2026-07-16):** the `sota-v2` API headline +lane uses a common 4,096-total-output-token safety ceiling and native-minimum +reasoning. Reasoning is disabled where optional; models for which OpenRouter +marks reasoning mandatory use their lowest supported effort. Actual input, +output, and reasoning tokens, cost, and latency are secondary efficiency metrics; they do not change the benchmark score. The earlier 256 / 1,024 / 4,096 / 16,384 experiment was retired before any @@ -21,19 +23,22 @@ treatment. This evidence does **not** promote the old Luna score: that run used the superseded prompt scaffold. It only supports the operational conclusion that -1,024 left substantial output headroom. +1,024 left substantial output headroom for reasoning-disabled models. It does +not establish a safe cap for Kimi K3 or the other mandatory-reasoning models in +the revised panel. ## Pre-full-panel cap-pressure rule Before any full-panel result is run: -1. Smoke all ten models in `config/sota_v2_models.json` serially at 1,024. -2. Verify exact route, required parameters, JSON behavior, reasoning disabled, - complete token/cost telemetry, and clean completion. +1. Smoke all ten phase-one models in `config/sota_v2_models.json` serially at 4,096. +2. Verify exact provider slug, endpoint tag and snapshot, required parameters, + JSON behavior, the registered per-model reasoning policy, complete + token/cost telemetry, and clean completion. 3. Inspect per-call output usage and truncation evidence. -4. If any call emits at least 768 output tokens (75% of the cap), or provider or +4. If any call emits at least 3,072 total output tokens (75% of the cap), or provider or adapter telemetry indicates cap-induced truncation, raise the **entire** API - lane to 2,048 and repeat affected smokes before any full result. + lane to 8,192 and repeat affected smokes before any full result. 5. Do not change the common cap after any full-panel score is visible. The 75% trigger is deliberately conservative: it detects meaningful cap @@ -54,9 +59,10 @@ model. ## Frozen publication lane - Headline rows are API-only, fresh-spawn, `compact`, with one bounded protocol - repair and exactly 1,024 maximum output tokens per call. -- Reasoning is explicitly disabled with `reasoning.enabled=false`; reasoning - effort and reasoning-token caps are absent. + repair and exactly 4,096 maximum total output tokens per call. +- Reasoning is disabled with `reasoning.enabled=false` where optional. Gemini + 3.5 Flash and Muse Spark 1.1 use `minimal`, Grok 4.5 uses `low`, and Kimi K3 + uses its only supported effort, `max`. - Every row reports score, failures, illegal actions, input/output tokens, latency, and cost. Token efficiency is interpretive evidence, not a hidden score adjustment. @@ -73,8 +79,7 @@ only after strict validation; it retains per-seed/repeat outcomes and aggregate usage plus a canonical raw-artifact SHA-256. CI requires the compact format and caps committed current rows at 1 MB. -`config/output_budget_sweep.json` and -`results/analysis/output-budget-cost-estimate.json` describe the retired design. -Do not use the old cost artifact as current spend guidance. Replace it with a -fixed-panel estimate after the ten route smokes provide current latency and -usage evidence. +`config/output_budget_sweep.json` describes the retired four-cap design. +`results/analysis/output-budget-cost-estimate.json` must be regenerated from the +current ten-model registry and refreshed again after the route smokes provide +current latency and usage evidence. diff --git a/docs/run_logs/sota-v2-panel-revision-2026-07-16.md b/docs/run_logs/sota-v2-panel-revision-2026-07-16.md new file mode 100644 index 0000000..5b8daa9 --- /dev/null +++ b/docs/run_logs/sota-v2-panel-revision-2026-07-16.md @@ -0,0 +1,54 @@ +# sota-v2 model-panel revision — 2026-07-16 + +This is a pre-full-panel protocol amendment. No full-panel model score existed +when the user replaced the stale ten-model registry with the following current +frontier panel. + +## Registered panel + +| Cohort | Model | OpenRouter model ID | Exact route | Reasoning | +| --- | --- | --- | --- | --- | +| Big American lab proprietary | GPT-5.6 Luna | `openai/gpt-5.6-luna` | `openai` / OpenAI dated endpoint | disabled | +| Big American lab proprietary | Claude Sonnet 5 | `anthropic/claude-sonnet-5` | `amazon-bedrock/global` / Bedrock global | disabled | +| Big American lab proprietary | Gemini 3.5 Flash | `google/gemini-3.5-flash` | `google-ai-studio` | mandatory `minimal` | +| Big American lab proprietary | Grok 4.5 | `x-ai/grok-4.5` | `xai` | mandatory `low` | +| Big American lab proprietary | Muse Spark 1.1 | `meta/muse-spark-1.1` | `meta` | mandatory `minimal` | +| Open-weight | GLM 5.2 | `z-ai/glm-5.2` | `z-ai/fp8` | disabled | +| Open-weight | Kimi K3 | `moonshotai/kimi-k3` | `moonshotai/int4` | mandatory `max` (only exposed effort) | +| Open-weight | Nemotron 3 Ultra | `nvidia/nemotron-3-ultra-550b-a55b` | `together` | disabled | +| Open-weight | MiniMax M3 | `minimax/minimax-m3` | `minimax/fp8` | disabled | +| Open-weight | Qwen 3.7 Plus | `qwen/qwen3.7-plus` | `alibaba` | disabled | +| Open-weight | DeepSeek V4 Pro | `deepseek/deepseek-v4-pro` | `deepseek` | disabled | +| Open-weight | Mistral Medium 3.5 | `mistralai/mistral-medium-3-5` | `mistral` | disabled | + +Terra, Sol, Claude Opus 4.8, and Fable 5 are recorded as future candidates, +not aliases or fallbacks for this panel. + +## Route policy + +- Pin the exact provider slug, endpoint tag, and dated endpoint name. +- Disable provider fallback and require every requested parameter. +- Deny data-collection routes and request JSON mode. +- Prefer a healthy first-party route. Claude is the declared exception because + the direct Anthropic endpoint was unhealthy; the healthy global Bedrock tag + is pinned instead. +- Nemotron uses Together rather than the cheaper DeepInfra FP4 endpoint to + avoid the prior Nano/DeepInfra failure mode and retain the 100% five-minute + uptime route observed during selection. +- Reasoning is disabled where optional. Mandatory-reasoning models use the + lowest effort exposed by OpenRouter; Kimi K3 exposes only `max`. + +## Compute and evidence policy + +The common provisional cap is 4,096 total output tokens. If any smoke reaches +3,072 tokens or shows truncation, raise the entire panel to 8,192 before any +full-panel score is visible. Reasoning tokens, output tokens, latency, and cost +remain reported secondary efficiency metrics. + +The earlier 1,024-token manifest was reset. Every route requires a new serial +smoke and exact-route preflight before the registry can become frozen. + +At 2026-07-16T19:13Z, the publication runner's no-spend preflight passed all +twelve exact endpoint tags, required-parameter checks, 4,096-token limits, and +local OpenRouter credential checks. This proves route availability only; it is +not a substitute for the paid benchmark-level smokes. diff --git a/docs/run_logs/sota-v2-smokes-2026-07-16.md b/docs/run_logs/sota-v2-smokes-2026-07-16.md new file mode 100644 index 0000000..b617286 --- /dev/null +++ b/docs/run_logs/sota-v2-smokes-2026-07-16.md @@ -0,0 +1,28 @@ +# sota-v2 fixed-cap smoke ledger — 2026-07-16 + +> **Superseded:** On 2026-07-16 the user deliberately replaced this ten-model, +> reasoning-disabled, 1,024-token panel before any full-panel score existed. +> These outcomes remain operational evidence but cannot satisfy the revised +> ten-model, native-minimum-reasoning phase-one smoke gate. + +This ledger summarizes the serial pre-panel smokes under the frozen 1,024-token, +reasoning-disabled API lane. Raw artifacts, checkpoints, and provider-attempt +evidence remain under +`data/publication-runs/smoke-fixed-1024-2026-07-16/` outside Git. A valid but +poor model response is never rerun merely to obtain a cleaner result. + +| Model | Route | Outcome | Cost | Evidence | +| --- | --- | --- | ---: | --- | +| Qwen 3.5 9B | DeepInfra | Accepted | $0.004227 | 4/4 decisions, zero failures or repairs, 5/5 finish reasons, zero truncations or reasoning tokens, peak 428 output tokens. Recorded in `config/sota_v2_smoke_manifest.json`. | +| Nemotron 3 Nano 30B A3B | DeepInfra | Infrastructure retries exhausted | $0 artifact-reported | Both permitted serial attempts aborted after two consecutive `HTTP 405 Method Not Allowed` responses before a complete episode. Immediately before attempt 2, the exact endpoint reported healthy status `0`, 99.46% five-minute uptime, and support for the required parameters in OpenRouter's live catalog. The checkpoint and conservative reservations are retained; the frozen route will not be substituted or retried. | +| MiniMax M3 | MiniMax | Completed, ineligible | $0.011866 | 4/4 metered decisions but one failed decision: the draft response was not a JSON action array and the one bounded repair also failed, causing one fallback. Zero truncations or reasoning tokens; peak 261 output tokens. Frozen policy forbids a rerun for this model behavior. | +| Qwen 3.7 Plus | Alibaba | Accepted | $0.013661 | 4/4 decisions, zero failures or repairs, 5/5 finish reasons, zero truncations or reasoning tokens, peak 271 output tokens. Recorded in `config/sota_v2_smoke_manifest.json`. | +| Kimi K2.6 | DeepInfra | Accepted | $0.024844 | 4/4 decisions, zero failures or repairs, 4/4 finish reasons, zero truncations or reasoning tokens, peak 289 output tokens. The changed route passed and was recorded in `config/sota_v2_smoke_manifest.json`. | +| GLM 5.2 | StreamLake | Completed, ineligible | $0.021550 | 4/4 metered decisions but one failed decision: the model JSON did not contain typed actions and the one bounded repair also failed, causing one fallback. Zero truncations or reasoning tokens; peak 414 output tokens. Frozen policy forbids a rerun for this model behavior. | +| DeepSeek V4 Pro | DeepInfra | Accepted on infrastructure attempt 2 | $0.051511 | Attempt 1 received two consecutive `HTTP 429 Too Many Requests` responses with no incremental spend and retained its checkpoint. The resumed final infrastructure attempt completed with 4/4 decisions, zero failures or repairs, 5/5 finish reasons, zero truncations or reasoning tokens, and a peak of 305 output tokens. Recorded in `config/sota_v2_smoke_manifest.json`. | +| GPT-5.6 Luna | OpenAI | Accepted | $0.039452 | 4/4 decisions, zero failures or repairs, 4/4 finish reasons, zero truncations or reasoning tokens, peak 144 output tokens. Recorded in `config/sota_v2_smoke_manifest.json`. | +| Mistral Medium 3.5 | Mistral | Accepted | $0.056330 | 4/4 decisions, zero failures or repairs, 4/4 finish reasons, zero truncations or reasoning tokens, peak 299 output tokens. One illegal action remains measured model behavior and does not invalidate smoke infrastructure. Recorded in `config/sota_v2_smoke_manifest.json`. | +| Claude Sonnet 5 | Anthropic | Blocked at preflight | $0 | The exact registered Anthropic endpoint reported unhealthy status `-2` on both checks; the latest check showed 82.32% five-minute uptime. Healthy Bedrock and Google routes are not permitted substitutes. No reservation or model request was made; run only if the pinned Anthropic endpoint returns healthy. | + +Current artifact-reported run-directory spend: **$0.223441**. The conservative +reservation ledger additionally retains both failed Nemotron attempts. diff --git a/docs/run_logs/sota-v2-smokes-2026-07-17.md b/docs/run_logs/sota-v2-smokes-2026-07-17.md new file mode 100644 index 0000000..a28bf98 --- /dev/null +++ b/docs/run_logs/sota-v2-smokes-2026-07-17.md @@ -0,0 +1,40 @@ +# sota-v2 phase-one smoke ledger — 2026-07-17 + +The frozen phase-one registry completed its serial 4,096-token OpenRouter smoke +gate under a $2 operator ceiling. Raw artifacts and checkpoints remain outside +Git under `data/publication-runs/smoke-frontier-4096-2026-07-17/`. + +## Accepted registry + +All ten registered models completed four decisions with zero failed decisions, +zero truncations, complete cost/finish-reason telemetry, and the exact pinned +upstream route. Artifact-reported spend was **$0.728909**. + +| Model | Cost | Peak output tokens per call | +| --- | ---: | ---: | +| GPT-5.6 Luna | $0.039654 | 141 | +| Claude Sonnet 5 | $0.113126 | 604 | +| Gemini 3.5 Flash | $0.050141 | 107 | +| Grok 4.5 | $0.075268 | 937 | +| Muse Spark 1.1 | $0.026332 | 1,432 | +| GLM 5.2 | $0.047053 | 311 | +| MiniMax M3 | $0.009713 | 260 | +| Qwen 3.7 Plus | $0.011145 | 266 | +| Mistral Medium 3.5 | $0.055181 | 201 | +| Tencent HY3 free | $0.000000 | 150 | + +Muse had the largest response at 1,432 tokens, below the predeclared 3,072-token +cap-pressure trigger. Mistral recorded one illegal action; this is measured +model behavior and did not invalidate its clean execution smoke. + +## Retained exclusions + +- Kimi K3 at mandatory `max` reasoning hit the 4,096-token ceiling in two calls, + produced two truncated responses and two failed decisions, and cost $0.301296. + Its raw artifact is retained and the route may not be rerun in phase one. +- Nemotron 3 Ultra's exact free Nvidia route passed catalog/authentication + preflight, then returned HTTP 404 on both permitted real calls. The checkpoint + is retained; no paid-route substitution is allowed in phase one. +- DeepSeek V4 Pro's exact first-party route likewise returned HTTP 404 on both + permitted real calls after passing preflight. The checkpoint is retained and + no route substitution is allowed in phase one. diff --git a/gm_bench/cli.py b/gm_bench/cli.py index 059f20e..4aa979d 100644 --- a/gm_bench/cli.py +++ b/gm_bench/cli.py @@ -868,7 +868,10 @@ def _model_clean_errors(result: dict[str, Any]) -> list[str]: if run_info.get("provider") == "openrouter" and len(usage.get("upstream_providers") or []) != 1: errors.append("OpenRouter requires exactly one observed upstream provider") if run_info.get("provider") == "openrouter": - requested = str((run_info.get("provider_options") or {}).get("OPENROUTER_PROVIDER_ONLY", "")).strip() + options = run_info.get("provider_options") or {} + requested = str( + options.get("OPENROUTER_EXPECTED_UPSTREAM_PROVIDER") or options.get("OPENROUTER_PROVIDER_ONLY", "") + ).strip() observed = [str(value) for value in usage.get("upstream_providers") or []] if requested and len(observed) == 1 and observed[0].casefold() != requested.casefold(): errors.append(f"OpenRouter upstream mismatch: requested {requested!r}, observed {observed[0]!r}") diff --git a/gm_bench/official.py b/gm_bench/official.py index dae456e..e4ffbb6 100644 --- a/gm_bench/official.py +++ b/gm_bench/official.py @@ -373,6 +373,7 @@ def _validate_openrouter_route( """Keep flexible gateway rows visible without treating them as canonical.""" options = _dict(run_info.get("provider_options")) only = str(options.get("OPENROUTER_PROVIDER_ONLY", "")).strip() + expected_upstream = str(options.get("OPENROUTER_EXPECTED_UPSTREAM_PROVIDER") or only).strip() expected_endpoint = str(options.get("OPENROUTER_EXPECTED_ENDPOINT_NAME", "")).strip() fallbacks = str(options.get("OPENROUTER_ALLOW_FALLBACKS", "")).lower() upstreams = [str(value) for value in _list(usage.get("upstream_providers")) if value] @@ -385,10 +386,10 @@ def _validate_openrouter_route( issues.append("run_info.provider_options.OPENROUTER_ALLOW_FALLBACKS must be false") if len(set(upstreams)) != 1: issues.append("candidate usage must report exactly one OpenRouter upstream provider") - elif only and upstreams[0].casefold() != only.casefold(): + elif expected_upstream and upstreams[0].casefold() != expected_upstream.casefold(): issues.append( "candidate usage upstream provider does not match " - f"OPENROUTER_PROVIDER_ONLY: requested {only!r}, observed {upstreams[0]!r}" + f"the registered route: requested {expected_upstream!r}, observed {upstreams[0]!r}" ) if strict: errors.extend(issues) diff --git a/gm_bench/providers.py b/gm_bench/providers.py index 9e557ef..a2255e8 100644 --- a/gm_bench/providers.py +++ b/gm_bench/providers.py @@ -206,6 +206,7 @@ def max_output_tokens_per_call(usage: dict[str, Any]) -> int: }, provenance_env=( "OPENROUTER_PROVIDER_ONLY", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER", "OPENROUTER_EXPECTED_ENDPOINT_NAME", "OPENROUTER_PROVIDER_SORT", "OPENROUTER_ALLOW_FALLBACKS", diff --git a/gm_bench/publication.py b/gm_bench/publication.py index eb15b3d..ee2abc7 100644 --- a/gm_bench/publication.py +++ b/gm_bench/publication.py @@ -119,7 +119,16 @@ def smoke_manifest_issues( issues.append(f"{prefix} is not accepted") if entry.get("decision_failure_rate") != 0: issues.append(f"{prefix} decision_failure_rate must be zero") - for key in ("provider", "model", "upstream_provider", "endpoint_name"): + for key in ( + "provider", + "model", + "upstream_provider", + "upstream_provider_slug", + "endpoint_tag", + "endpoint_name", + "reasoning_policy", + "reasoning_effort", + ): if entry.get(key) != model.get(key): issues.append(f"{prefix} {key} does not match the registered route") if entry.get("output_token_cap") != frozen_cap: @@ -151,8 +160,13 @@ def smoke_manifest_issues( f"{prefix} peaked at {max_output} output tokens, at or above the " f"{threshold}-token cap-pressure threshold; apply the cap-pressure rule before the panel" ) - if int(entry.get("reasoning_tokens") or 0): - issues.append(f"{prefix} recorded reasoning tokens in the reasoning-disabled lane") + reasoning_tokens = entry.get("reasoning_tokens") + if model.get("reasoning_policy") == "disabled" and int(reasoning_tokens or 0): + issues.append(f"{prefix} recorded reasoning tokens for a reasoning-disabled model") + if model.get("reasoning_policy") == "mandatory-minimum" and ( + not isinstance(reasoning_tokens, int) or isinstance(reasoning_tokens, bool) or reasoning_tokens < 0 + ): + issues.append(f"{prefix} is missing reasoning-token telemetry for a mandatory-reasoning model") if entry.get("contract_fingerprint") != contract_fingerprint(): issues.append(f"{prefix} was recorded under a different benchmark contract") expected_scaffold = scaffold_fingerprint(str(model.get("provider") or "")) diff --git a/results/analysis/output-budget-cost-estimate.json b/results/analysis/output-budget-cost-estimate.json index 5eda73a..e058b53 100644 --- a/results/analysis/output-budget-cost-estimate.json +++ b/results/analysis/output-budget-cost-estimate.json @@ -3,7 +3,7 @@ "caveat": "Provider prices and actual token usage can change. Recheck before paid runs; the operator spend guard remains mandatory.", "cost_contingency_multiplier": 1.2, "input_tokens_per_decision": 8000, - "output_tokens_per_decision": 1024, + "output_tokens_per_decision": 4096, "panel_preset": "leaderboard", "panel_repeats": 3, "panel_seasons": 5, @@ -25,121 +25,135 @@ "total_calls": 4840 }, "costs_usd": { - "panel": 48.9489408, - "smoke": 0.40790784, - "total_unrounded": 49.35684864, - "total_with_1_2x_contingency": 59.228218368 + "panel": 140.8533504, + "smoke": 1.17377792, + "total_unrounded": 142.02712832, + "total_with_1_2x_contingency": 170.432553984 }, "models": [ { - "cost_per_decision_usd": 0.014144, + "cost_per_decision_usd": 0.032576, "experiment_id": "openrouter-gpt-5.6-luna-openai", "model": "openai/gpt-5.6-luna", - "observed_api_seconds_per_decision": 1.981, + "observed_api_seconds_per_decision": 4.759, "panel_calls": 480, - "panel_cost_usd": 6.78912, + "panel_cost_usd": 15.63648, "smoke_calls": 4, - "smoke_cost_usd": 0.056576 + "smoke_cost_usd": 0.130304 }, { - "cost_per_decision_usd": 0.02624, - "experiment_id": "openrouter-claude-sonnet-5-anthropic", + "cost_per_decision_usd": 0.05696, + "experiment_id": "openrouter-claude-sonnet-5-bedrock", "model": "anthropic/claude-sonnet-5", + "observed_api_seconds_per_decision": 8.657, "panel_calls": 480, - "panel_cost_usd": 12.5952, + "panel_cost_usd": 27.3408, "smoke_calls": 4, - "smoke_cost_usd": 0.10496 + "smoke_cost_usd": 0.22784 }, { - "cost_per_decision_usd": 0.0130624, - "experiment_id": "openrouter-deepseek-v4-pro-deepinfra", - "model": "deepseek/deepseek-v4-pro", + "cost_per_decision_usd": 0.048864, + "experiment_id": "openrouter-gemini-3.5-flash-google-ai-studio", + "model": "google/gemini-3.5-flash", + "observed_api_seconds_per_decision": 1.497, "panel_calls": 480, - "panel_cost_usd": 6.269952, + "panel_cost_usd": 23.45472, "smoke_calls": 4, - "smoke_cost_usd": 0.0522496 + "smoke_cost_usd": 0.195456 }, { - "cost_per_decision_usd": 0.009584, - "experiment_id": "openrouter-kimi-k2.6-deepinfra", - "model": "moonshotai/kimi-k2.6", + "cost_per_decision_usd": 0.040576, + "experiment_id": "openrouter-grok-4.5-xai", + "model": "x-ai/grok-4.5", + "observed_api_seconds_per_decision": 7.683, "panel_calls": 480, - "panel_cost_usd": 4.60032, + "panel_cost_usd": 19.47648, "smoke_calls": 4, - "smoke_cost_usd": 0.038336 + "smoke_cost_usd": 0.162304 }, { - "cost_per_decision_usd": 0.01020864, - "experiment_id": "openrouter-glm-5.2-streamlake", - "model": "z-ai/glm-5.2", + "cost_per_decision_usd": 0.027408, + "experiment_id": "openrouter-muse-spark-1.1-meta", + "model": "meta/muse-spark-1.1", + "observed_api_seconds_per_decision": 13.827, "panel_calls": 480, - "panel_cost_usd": 4.9001472, + "panel_cost_usd": 13.15584, "smoke_calls": 4, - "smoke_cost_usd": 0.04083456 + "smoke_cost_usd": 0.109632 }, { - "cost_per_decision_usd": 0.00387072, - "experiment_id": "openrouter-qwen3.7-plus-alibaba", - "model": "qwen/qwen3.7-plus", + "cost_per_decision_usd": 0.0292224, + "experiment_id": "openrouter-glm-5.2-zai", + "model": "z-ai/glm-5.2", + "observed_api_seconds_per_decision": 6.656, "panel_calls": 480, - "panel_cost_usd": 1.8579456, + "panel_cost_usd": 14.026752, "smoke_calls": 4, - "smoke_cost_usd": 0.01548288 + "smoke_cost_usd": 0.1168896 }, { - "cost_per_decision_usd": 0.0036288, + "cost_per_decision_usd": 0.0073152, "experiment_id": "openrouter-minimax-m3-minimax", "model": "minimax/minimax-m3", - "observed_api_seconds_per_decision": 2.546, + "observed_api_seconds_per_decision": 2.955, "panel_calls": 480, - "panel_cost_usd": 1.741824, + "panel_cost_usd": 3.511296, "smoke_calls": 4, - "smoke_cost_usd": 0.0145152 + "smoke_cost_usd": 0.0292608 }, { - "cost_per_decision_usd": 0.01968, - "experiment_id": "openrouter-mistral-medium-3.5-mistral", - "model": "mistralai/mistral-medium-3-5", + "cost_per_decision_usd": 0.00780288, + "experiment_id": "openrouter-qwen3.7-plus-alibaba", + "model": "qwen/qwen3.7-plus", + "observed_api_seconds_per_decision": 3.586, "panel_calls": 480, - "panel_cost_usd": 9.4464, + "panel_cost_usd": 3.7453824, "smoke_calls": 4, - "smoke_cost_usd": 0.07872 + "smoke_cost_usd": 0.03121152 }, { - "cost_per_decision_usd": 0.0006048, - "experiment_id": "openrouter-nemotron-3-nano-deepinfra", - "model": "nvidia/nemotron-3-nano-30b-a3b", + "cost_per_decision_usd": 0.04272, + "experiment_id": "openrouter-mistral-medium-3.5-mistral", + "model": "mistralai/mistral-medium-3-5", + "observed_api_seconds_per_decision": 2.464, "panel_calls": 480, - "panel_cost_usd": 0.290304, + "panel_cost_usd": 20.5056, "smoke_calls": 4, - "smoke_cost_usd": 0.0024192 + "smoke_cost_usd": 0.17088 }, { - "cost_per_decision_usd": 0.0009536, - "experiment_id": "openrouter-qwen3.5-9b-deepinfra", - "model": "qwen/qwen3.5-9b", - "observed_api_seconds_per_decision": 11.622, + "cost_per_decision_usd": 0.0, + "experiment_id": "openrouter-tencent-hy3-free-novita", + "model": "tencent/hy3:free", + "observed_api_seconds_per_decision": 4.087, "panel_calls": 480, - "panel_cost_usd": 0.457728, + "panel_cost_usd": 0.0, "smoke_calls": 4, - "smoke_cost_usd": 0.0038144 + "smoke_cost_usd": 0.0 } ], - "pricing_checked_at_utc": "2026-07-15T20:41:27Z", + "pricing_checked_at_utc": "2026-07-17T19:37:57Z", "runtime": { - "note": "Regenerate this artifact from accepted smoke telemetry before approving the full panel; latency is reported only for models with committed observations.", - "observation_source": "legacy standardized 1,024-token Luna, MiniMax, and Qwen observations; use only for conservative reservations and refresh every registered route through the fixed-cap smoke gate before the full panel", + "note": "Runtime observations are sourced from every currently registered model's accepted smoke; recheck before paid full-panel runs if pricing, routes, or prompts change.", + "observation_source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md", "observed_api_seconds_per_decision_by_model": { - "minimax/minimax-m3": 2.546, - "openai/gpt-5.6-luna": 1.981, - "qwen/qwen3.5-9b": 11.622 + "anthropic/claude-sonnet-5": 8.657, + "google/gemini-3.5-flash": 1.497, + "meta/muse-spark-1.1": 13.827, + "minimax/minimax-m3": 2.955, + "mistralai/mistral-medium-3-5": 2.464, + "openai/gpt-5.6-luna": 4.759, + "qwen/qwen3.7-plus": 3.586, + "tencent/hy3:free": 4.087, + "x-ai/grok-4.5": 7.683, + "z-ai/glm-5.2": 6.656 }, - "observed_at_utc": "2026-07-15T01:22:44Z", - "status": "pending-smoke-telemetry" + "observed_at_utc": "2026-07-17T19:42:44Z", + "status": "complete-from-accepted-smokes" }, "schema_version": 2, "supersedes": { "artifact": "retired 12-cell output-budget sweep estimate", - "description": "Replaces the four-cap, three-model sweep estimate with the registered ten-model fixed 1,024-token panel and its required smoke gate." + "description": "Replaces the four-cap, three-model sweep estimate with the registered 10-model fixed 4,096-token panel and its required smoke gate." } } diff --git a/scripts/analyze_publication_panel.py b/scripts/analyze_publication_panel.py index 2709dd7..e2159f2 100644 --- a/scripts/analyze_publication_panel.py +++ b/scripts/analyze_publication_panel.py @@ -215,10 +215,23 @@ def _registry_specs(registry: Mapping[str, Any]) -> tuple[list[dict[str, Any]], model = str(row.get("model") or "").strip() transport = str(row.get("transport") or "").strip() upstream_provider = str(row.get("upstream_provider") or "").strip() + upstream_provider_slug = str(row.get("upstream_provider_slug") or "").strip() + endpoint_tag = str(row.get("endpoint_tag") or "").strip() endpoint_name = str(row.get("endpoint_name") or "").strip() - if not all((model_id, provider, model, transport, upstream_provider, endpoint_name)): + if not all( + ( + model_id, + provider, + model, + transport, + upstream_provider, + upstream_provider_slug, + endpoint_tag, + endpoint_name, + ) + ): errors.append( - f"models[{index}] requires id, provider, model, transport, upstream_provider, and endpoint_name" + f"models[{index}] requires id, provider, model, transport, upstream provider identity, endpoint tag, and endpoint_name" ) continue if model_id in seen_ids: @@ -235,15 +248,23 @@ def _registry_specs(registry: Mapping[str, Any]) -> tuple[list[dict[str, Any]], "model": model, "transport": transport, "upstream_provider": upstream_provider, + "upstream_provider_slug": upstream_provider_slug, + "endpoint_tag": endpoint_tag, "endpoint_name": endpoint_name, "fixed_options": { **dict(shared_fixed_options), - "OPENROUTER_PROVIDER_ONLY": upstream_provider, + **dict(row.get("fixed_options") or {}), + "OPENROUTER_PROVIDER_ONLY": upstream_provider_slug, + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": upstream_provider, "OPENROUTER_EXPECTED_ENDPOINT_NAME": endpoint_name, "OPENROUTER_MAX_TOKENS": str(registry.get("output_token_cap")), "GM_BENCH_OUTPUT_BUDGET_CELL": str(registry.get("output_token_cap")), }, - "absent_options": [str(value) for value in shared_absent_options], + "absent_options": [ + str(value) + for value in [*shared_absent_options, *(row.get("absent_options") or [])] + if str(value) not in {**dict(shared_fixed_options), **dict(row.get("fixed_options") or {})} + ], } ) return specs, errors diff --git a/scripts/estimate_publication_cost.py b/scripts/estimate_publication_cost.py index f313f1a..09bd348 100644 --- a/scripts/estimate_publication_cost.py +++ b/scripts/estimate_publication_cost.py @@ -17,11 +17,16 @@ from gm_bench.benchmark_config import PRESETS # noqa: E402 from gm_bench.protocol import PHASES # noqa: E402 -RUNTIME_STATUS = "pending-smoke-telemetry" -RUNTIME_NOTE = ( +RUNTIME_STATUS_PENDING = "pending-smoke-telemetry" +RUNTIME_STATUS_COMPLETE = "complete-from-accepted-smokes" +RUNTIME_NOTE_PENDING = ( "Regenerate this artifact from accepted smoke telemetry before approving the full panel; " "latency is reported only for models with committed observations." ) +RUNTIME_NOTE_COMPLETE = ( + "Runtime observations are sourced from every currently registered model's accepted smoke; " + "recheck before paid full-panel runs if pricing, routes, or prompts change." +) def _read(path: Path) -> dict[str, Any]: @@ -60,12 +65,7 @@ def estimate( leaderboard = PRESETS["leaderboard"] smoke = PRESETS["smoke"] repeats = int(models_config["repeats"]) - panel_decisions_per_model = ( - len(leaderboard["seeds"]) - * int(leaderboard["seasons"]) - * len(PHASES) - * repeats - ) + panel_decisions_per_model = len(leaderboard["seeds"]) * int(leaderboard["seasons"]) * len(PHASES) * repeats smoke_decisions_per_run = len(smoke["seeds"]) * int(smoke["seasons"]) * len(PHASES) model_count = len(models) panel_calls = model_count * panel_decisions_per_model @@ -82,9 +82,7 @@ def estimate( rates = pricing["models"].get(model_name) if not rates: raise ValueError(f"missing pricing for {model_name}") - per_decision_cost = input_tokens * _decimal(rates["prompt"]) + output_tokens * _decimal( - rates["completion"] - ) + per_decision_cost = input_tokens * _decimal(rates["prompt"]) + output_tokens * _decimal(rates["completion"]) panel_cost = panel_decisions_per_model * per_decision_cost smoke_cost = smoke_decisions_per_run * per_decision_cost panel_costs.append(panel_cost) @@ -108,13 +106,19 @@ def estimate( smoke_cost = sum(smoke_costs, Decimal()) total_cost = panel_cost + smoke_cost contingency = _decimal(assumptions["cost_contingency_multiplier"]) + # "Complete" only once every currently registered model (not just a subset + # left over from a prior registry) has an observation; otherwise stay + # pending so a partial refresh can't be mistaken for full coverage. + runtime_complete = bool(models) and {model["model"] for model in models} <= set(observed_latency) + runtime_status = RUNTIME_STATUS_COMPLETE if runtime_complete else RUNTIME_STATUS_PENDING + runtime_note = RUNTIME_NOTE_COMPLETE if runtime_complete else RUNTIME_NOTE_PENDING return { "schema_version": 2, "supersedes": { "artifact": "retired 12-cell output-budget sweep estimate", "description": ( "Replaces the four-cap, three-model sweep estimate with the registered " - "ten-model fixed 1,024-token panel and its required smoke gate." + f"{len(models)}-model fixed {lane_config['output_token_cap']:,}-token panel and its required smoke gate." ), }, "pricing_checked_at_utc": pricing["checked_at_utc"], @@ -154,8 +158,8 @@ def estimate( "total_with_1_2x_contingency": float(total_cost * contingency), }, "runtime": { - "status": RUNTIME_STATUS, - "note": RUNTIME_NOTE, + "status": runtime_status, + "note": runtime_note, "observation_source": runtime_observations.get("source"), "observed_at_utc": runtime_observations.get("observed_at_utc"), "observed_api_seconds_per_decision_by_model": observed_latency, diff --git a/scripts/run_publication_matrix.py b/scripts/run_publication_matrix.py index c2595db..ab69100 100644 --- a/scripts/run_publication_matrix.py +++ b/scripts/run_publication_matrix.py @@ -15,6 +15,7 @@ import os import subprocess import sys +import time import urllib.error import urllib.request from dataclasses import dataclass @@ -29,12 +30,14 @@ LANE_CONFIG = ROOT / "config" / "sota_v2_lane.json" PRICING_CONFIG = ROOT / "config" / "openrouter_pricing_snapshot.json" SMOKE_MANIFEST = ROOT / "config" / "sota_v2_smoke_manifest.json" +RUN_STATE_FORMAT = "gm-bench-publication-run-v1" if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from gm_bench.benchmark_config import PRESETS # noqa: E402 from gm_bench.contract import contract_fingerprint, scaffold_fingerprint # noqa: E402 +from gm_bench.environment import load_environment_files # noqa: E402 from gm_bench.protocol import PHASES # noqa: E402 from gm_bench.publication import SMOKE_MANIFEST_FORMAT, smoke_manifest_issues # noqa: E402 @@ -48,6 +51,8 @@ class Cell: preset: str repeats: int cap: int | None + upstream_provider: str + endpoint_tag: str endpoint_name: str fixed_options: dict[str, str] absent_options: tuple[str, ...] @@ -64,7 +69,7 @@ def _read_json(path: Path) -> dict[str, Any]: return payload -def _validate_models(models: list[dict[str, Any]]) -> None: +def _validate_models(models: list[dict[str, Any]], *, exact_routes: bool = True) -> None: if not models: raise ValueError("publication matrix contains no models") ids = [str(model.get("id") or "") for model in models] @@ -73,6 +78,31 @@ def _validate_models(models: list[dict[str, Any]]) -> None: raise ValueError("every model requires non-empty id, provider, and model") if len(set(ids)) != len(ids) or len(set(identities)) != len(identities): raise ValueError("publication model ids and provider/model identities must be unique") + if not exact_routes: + return + for model in models: + required = ("upstream_provider", "upstream_provider_slug", "endpoint_tag", "endpoint_name") + if any(not str(model.get(key) or "").strip() for key in required): + raise ValueError(f"publication model {model.get('id')!r} is missing exact endpoint identity") + policy = model.get("reasoning_policy") + options = model.get("fixed_options") or {} + if policy == "disabled": + if ( + options.get("OPENROUTER_REASONING_ENABLED") != "false" + or options.get("OPENROUTER_REASONING_EFFORT") is not None + or model.get("reasoning_effort") is not None + ): + raise ValueError(f"publication model {model.get('id')!r} has an invalid disabled reasoning policy") + elif policy == "mandatory-minimum": + effort = model.get("reasoning_effort") + if ( + not effort + or options.get("OPENROUTER_REASONING_ENABLED") != "true" + or options.get("OPENROUTER_REASONING_EFFORT") != effort + ): + raise ValueError(f"publication model {model.get('id')!r} has an invalid mandatory reasoning policy") + else: + raise ValueError(f"publication model {model.get('id')!r} has an unknown reasoning policy") def _smoke_manifest_path(lane: dict[str, Any]) -> Path: @@ -80,6 +110,22 @@ def _smoke_manifest_path(lane: dict[str, Any]) -> Path: return ROOT / str(configured) if configured else SMOKE_MANIFEST +def _registered_fixed_options(config: dict[str, Any], model: dict[str, Any]) -> dict[str, str]: + return { + **{str(key): str(value) for key, value in (config.get("shared_fixed_options") or {}).items()}, + **{str(key): str(value) for key, value in (model.get("fixed_options") or {}).items()}, + "OPENROUTER_PROVIDER_ONLY": str(model["upstream_provider_slug"]), + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": str(model["upstream_provider"]), + "OPENROUTER_EXPECTED_ENDPOINT_NAME": str(model["endpoint_name"]), + } + + +def _registered_absent_options(config: dict[str, Any], model: dict[str, Any]) -> tuple[str, ...]: + fixed = _registered_fixed_options(config, model) + values = [*(config.get("shared_absent_options") or []), *(model.get("absent_options") or [])] + return tuple(dict.fromkeys(str(value) for value in values if str(value) not in fixed)) + + def _read_optional_json(path: Path) -> dict[str, Any] | None: try: return _read_json(path) @@ -94,13 +140,15 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) models = list(config.get("models") or []) _validate_models(models) frozen_cap = lane.get("output_token_cap") - if lane.get("output_policy_basis") != "fixed-safety-ceiling": + if lane.get("output_policy_basis") not in { + "fixed-safety-ceiling", + "common-safety-ceiling-with-native-minimum-reasoning", + }: raise ValueError("panel smoke is locked until the fixed safety ceiling is frozen") if not isinstance(frozen_cap, int) or frozen_cap < 1: raise ValueError("panel smoke requires a positive frozen output_token_cap") if cap is not None and cap != frozen_cap: raise ValueError(f"requested cap {cap} differs from frozen panel smoke cap {frozen_cap}") - shared = {str(key): str(value) for key, value in (config.get("shared_fixed_options") or {}).items()} cells = [ Cell( experiment_id=str(model["id"]), @@ -110,20 +158,18 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) preset="smoke", repeats=1, cap=frozen_cap, + upstream_provider=str(model["upstream_provider"]), + endpoint_tag=str(model["endpoint_tag"]), endpoint_name=str(model.get("endpoint_name") or ""), - fixed_options={ - **shared, - "OPENROUTER_PROVIDER_ONLY": str(model["upstream_provider"]), - "OPENROUTER_EXPECTED_ENDPOINT_NAME": str(model["endpoint_name"]), - }, - absent_options=tuple(str(value) for value in config.get("shared_absent_options") or []), + fixed_options=_registered_fixed_options(config, model), + absent_options=_registered_absent_options(config, model), ) for model in models ] elif phase == "sweep": config = _read_json(SWEEP_CONFIG) models = list(config.get("models") or []) - _validate_models(models) + _validate_models(models, exact_routes=False) configured_caps = list(config["output_token_caps"]) if cap is not None and cap not in configured_caps: raise ValueError(f"requested cap {cap} is not in the pre-registered sweep {configured_caps}") @@ -139,6 +185,8 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) preset=preset, repeats=repeats, cap=cell_cap, + upstream_provider=str(model["upstream_provider"]), + endpoint_tag=str(model.get("endpoint_tag") or ""), endpoint_name=str(model.get("endpoint_name") or ""), fixed_options={str(key): str(value) for key, value in (model.get("fixed_options") or {}).items()}, absent_options=tuple(str(value) for value in model.get("absent_options") or []), @@ -152,7 +200,11 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) models = list(config.get("models") or []) _validate_models(models) frozen_cap = lane.get("output_token_cap") - if lane.get("output_budget_status") not in {"frozen-saturation", "frozen-fixed-budget"}: + if lane.get("output_budget_status") not in { + "frozen-saturation", + "frozen-fixed-budget", + "frozen-native-reasoning-cap", + }: raise ValueError("full panel is locked until config/sota_v2_lane.json freezes the output-budget policy") if config.get("selection_status") != "frozen": raise ValueError("full panel is locked until config/sota_v2_models.json freezes the model registry") @@ -167,7 +219,6 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) ) if cap is not None and cap != frozen_cap: raise ValueError(f"requested cap {cap} differs from frozen panel cap {frozen_cap}") - shared = {str(key): str(value) for key, value in (config.get("shared_fixed_options") or {}).items()} cells = [ Cell( experiment_id=str(model["id"]), @@ -177,13 +228,11 @@ def build_cells(phase: str, model_id: str | None = None, cap: int | None = None) preset=str(config["preset"]), repeats=int(config["repeats"]), cap=frozen_cap, + upstream_provider=str(model["upstream_provider"]), + endpoint_tag=str(model["endpoint_tag"]), endpoint_name=str(model.get("endpoint_name") or ""), - fixed_options={ - **shared, - "OPENROUTER_PROVIDER_ONLY": str(model["upstream_provider"]), - "OPENROUTER_EXPECTED_ENDPOINT_NAME": str(model["endpoint_name"]), - }, - absent_options=tuple(str(value) for value in config.get("shared_absent_options") or []), + fixed_options=_registered_fixed_options(config, model), + absent_options=_registered_absent_options(config, model), ) for model in models ] @@ -264,19 +313,25 @@ def _openrouter_usage_usd(env: dict[str, str]) -> float: def _endpoint_issues(cell: Cell, payload: dict[str, Any]) -> list[str]: endpoints = (payload.get("data") or {}).get("endpoints") or [] - expected_provider = cell.fixed_options.get("OPENROUTER_PROVIDER_ONLY", "") + expected_provider = cell.upstream_provider matches = [ endpoint for endpoint in endpoints if endpoint.get("provider_name") == expected_provider + and endpoint.get("tag") == cell.endpoint_tag and endpoint.get("name") == cell.endpoint_name and endpoint.get("status") == 0 ] if not cell.endpoint_name: return ["pre-registered OpenRouter endpoint_name is empty"] if not matches: - return [f"no healthy OpenRouter endpoint matches provider={expected_provider!r} name={cell.endpoint_name!r}"] - required = {"max_tokens", "response_format", "reasoning"} + return [ + "no healthy OpenRouter endpoint matches " + f"provider={expected_provider!r} tag={cell.endpoint_tag!r} name={cell.endpoint_name!r}" + ] + required = {"max_tokens", "reasoning"} + if cell.fixed_options.get("OPENROUTER_JSON_MODE", "false").strip().lower() in {"1", "true", "yes", "on"}: + required.add("response_format") capable = [] for endpoint in matches: supported = set(endpoint.get("supported_parameters") or []) @@ -359,6 +414,272 @@ def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None: temporary.replace(path) +def _read_json_if_valid(path: Path) -> dict[str, Any] | None: + try: + return _read_json(path) + except (OSError, ValueError, json.JSONDecodeError): + return None + + +def _write_run_state(run_dir: Path, phase: str, cells: list[Cell], spend_ceiling_usd: float | None) -> None: + path = run_dir / "run-state.json" + existing = _read_json_if_valid(path) or {} + launched = {str(value) for value in existing.get("launched_model_ids") or [] if isinstance(value, str) and value} + launched.update(cell.experiment_id for cell in cells) + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + payload = { + "format": RUN_STATE_FORMAT, + "schema_version": 1, + "phase": phase, + "output_token_cap": cells[0].cap if cells else None, + "started_at_utc": existing.get("started_at_utc") or now, + "updated_at_utc": now, + "launched_model_ids": sorted(launched), + "spend_ceiling_usd": spend_ceiling_usd, + } + _write_json_atomic(path, payload) + + +def _checkpoint_process_alive(path: Path) -> bool: + import fcntl + + lock_path = path.with_suffix(path.suffix + ".lock") + try: + descriptor = os.open(lock_path, os.O_RDONLY) + except OSError: + return False + try: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return True + fcntl.flock(descriptor, fcntl.LOCK_UN) + return False + finally: + os.close(descriptor) + + +def _payload_cost_usd(payload: dict[str, Any] | None) -> float | None: + if not payload: + return None + candidate = payload.get("candidate") + if isinstance(candidate, dict): + summary = candidate.get("summary") + usage = summary.get("usage") if isinstance(summary, dict) else None + if isinstance(usage, dict) and usage.get("cost_usd") is not None: + try: + return float(usage["cost_usd"]) + except (TypeError, ValueError): + return None + costs = [] + for episode in payload.get("episodes") or []: + usage = episode.get("usage") if isinstance(episode, dict) else None + if isinstance(usage, dict) and usage.get("cost_usd") is not None: + try: + costs.append(float(usage["cost_usd"])) + except (TypeError, ValueError): + continue + return sum(costs) if costs else None + + +def _expected_progress( + run_state: dict[str, Any], checkpoint: dict[str, Any] | None, raw: dict[str, Any] | None +) -> tuple[int, int | None, int, int | None]: + source = checkpoint or raw or {} + completed = source.get("completed") + if isinstance(completed, list): + completed_episodes = len(completed) + else: + candidate = raw.get("candidate") if raw else None + episodes = candidate.get("episodes") if isinstance(candidate, dict) else None + completed_episodes = len(episodes) if isinstance(episodes, list) else 0 + + seeds = source.get("seeds") + repeats = source.get("repeats") + seasons = source.get("seasons") + if raw and isinstance(raw.get("candidate"), dict): + repeats = repeats or raw["candidate"].get("repeats") + seasons = seasons or raw["candidate"].get("seasons") + total_episodes = None + if isinstance(seeds, list) and isinstance(repeats, int): + total_episodes = len(seeds) * repeats + elif run_state.get("phase") == "smoke": + total_episodes = 1 + seasons = seasons or 1 + elif run_state.get("phase") == "panel": + registry = _read_json_if_valid(PANEL_CONFIG) or {} + preset = PRESETS.get(str(registry.get("preset") or "leaderboard"), {}) + total_episodes = len(preset.get("seeds") or []) * int(registry.get("repeats") or 0) + seasons = seasons or int(preset.get("seasons") or 0) + + decisions_per_episode = int(seasons or 0) * len(PHASES) + completed_decisions = completed_episodes * decisions_per_episode + total_decisions = total_episodes * decisions_per_episode if total_episodes is not None else None + return completed_episodes, total_episodes, completed_decisions, total_decisions + + +def publication_run_status(run_dir: Path, manifest_path: Path = SMOKE_MANIFEST) -> dict[str, Any]: + run_dir = run_dir.resolve() + registry = _read_json(PANEL_CONFIG) + run_state = _read_json_if_valid(run_dir / "run-state.json") or {} + cap = int(run_state.get("output_token_cap") or registry["output_token_cap"]) + manifest = _read_json_if_valid(manifest_path) or {} + manifest_entries = manifest.get("entries") if isinstance(manifest.get("entries"), dict) else {} + reservations_payload = _read_json_if_valid(run_dir / "openrouter-reservations.json") or {} + reservations = reservations_payload.get("cells") + reservations = reservations if isinstance(reservations, dict) else {} + + rows = [] + for model in registry.get("models") or []: + model_id = str(model["id"]) + stem = f"{model_id}--{cap}" + checkpoint_path = run_dir / "checkpoints" / f"{stem}.json" + raw_path = run_dir / "raw" / f"{stem}.json" + checkpoint = _read_json_if_valid(checkpoint_path) + raw = _read_json_if_valid(raw_path) + manifest_entry = manifest_entries.get(model_id) + reservation = reservations.get(stem) + smoke_accepted = isinstance(manifest_entry, dict) and manifest_entry.get("accepted") is True + + if raw_path.exists() and raw is None: + state = "invalid-raw" + elif raw is not None: + state = "complete" + elif checkpoint_path.exists() and checkpoint is None: + state = "invalid-checkpoint" + elif checkpoint is not None: + checkpoint_state = str(checkpoint.get("status") or "unknown") + if checkpoint_state == "running": + state = "running" if _checkpoint_process_alive(checkpoint_path) else "interrupted" + else: + state = checkpoint_state + elif isinstance(reservation, dict): + state = "reserved" + else: + state = "queued" + # A manifest acceptance only upgrades *this run's own* completed cell to + # "accepted"; it never substitutes for the current run/cap's raw + # artifact, so a stale or wrong-cap manifest entry cannot mask a cell + # that this run directory has not actually produced. + if ( + run_state.get("phase") == "smoke" + and state == "complete" + and smoke_accepted + and manifest_entry.get("output_token_cap") == cap + ): + state = "accepted" + + completed, total, completed_decisions, total_decisions = _expected_progress(run_state, checkpoint, raw) + if state in {"complete", "accepted"} and total is not None: + completed = total + completed_decisions = total_decisions or completed_decisions + mtimes = [path.stat().st_mtime for path in (checkpoint_path, raw_path) if path.exists()] + rows.append( + { + "model_id": model_id, + "model": model.get("model"), + "state": state, + "smoke_accepted": smoke_accepted, + "completed_episodes": completed, + "total_episodes": total, + "completed_decisions": completed_decisions, + "total_decisions": total_decisions, + "cost_usd": _payload_cost_usd(raw or checkpoint), + "reserved_usd": ( + float(reservation["reserved_usd"]) + if isinstance(reservation, dict) and reservation.get("reserved_usd") is not None + else None + ), + "updated_at_epoch": max(mtimes) if mtimes else None, + "error": checkpoint.get("error") if isinstance(checkpoint, dict) else None, + } + ) + + artifact_spend = _artifact_spend_usd(run_dir) + reserved_spend = sum( + float(value.get("reserved_usd") or 0.0) for value in reservations.values() if isinstance(value, dict) + ) + return { + "format": "gm-bench-publication-status-v1", + "run_dir": str(run_dir), + "phase": run_state.get("phase") or "unknown", + "output_token_cap": run_state.get("output_token_cap") or cap, + "started_at_utc": run_state.get("started_at_utc"), + "spend_ceiling_usd": run_state.get("spend_ceiling_usd"), + "artifact_spend_usd": round(artifact_spend, 6), + "reserved_spend_usd": round(reserved_spend, 6), + "total_cells": len(rows), + "accepted_smokes": sum(row["smoke_accepted"] for row in rows), + "active_cells": sum(row["state"] == "running" for row in rows), + "completed_cells": sum(row["state"] in {"complete", "accepted"} for row in rows), + "rows": rows, + } + + +def _format_progress(completed: int, total: int | None) -> str: + return f"{completed}/{total}" if total is not None else str(completed) + + +def _format_age(timestamp: float | None) -> str: + if timestamp is None: + return "-" + seconds = max(0, int(time.time() - timestamp)) + if seconds < 60: + return f"{seconds}s" + if seconds < 3600: + return f"{seconds // 60}m" + return f"{seconds // 3600}h" + + +def render_publication_status(status: dict[str, Any]) -> str: + ceiling = status.get("spend_ceiling_usd") + ceiling_text = f"${float(ceiling):.4f}" if ceiling is not None else "not recorded" + lines = [ + "GM-Bench publication run", + f"dir: {status['run_dir']}", + ( + f"phase: {status['phase']} cap: {status['output_token_cap']} " + f"active: {status['active_cells']} " + f"complete: {status['completed_cells']}/{status['total_cells']} " + f"accepted smokes: {status['accepted_smokes']}/{status['total_cells']}" + ), + ( + f"spend: ${status['artifact_spend_usd']:.4f} in artifacts " + f"${status['reserved_spend_usd']:.4f} reserved ceiling: {ceiling_text}" + ), + "", + f"{'MODEL':<42} {'STATE':<18} {'EPISODES':>9} {'DECISIONS':>10} {'COST':>9} {'UPDATED':>8}", + "-" * 102, + ] + for row in status["rows"]: + cost = f"${row['cost_usd']:.4f}" if row["cost_usd"] is not None else "-" + lines.append( + f"{row['model_id'][:42]:<42} {row['state']:<18} " + f"{_format_progress(row['completed_episodes'], row['total_episodes']):>9} " + f"{_format_progress(row['completed_decisions'], row['total_decisions']):>10} " + f"{cost:>9} {_format_age(row['updated_at_epoch']):>8}" + ) + if row.get("error"): + lines.append(f" error: {str(row['error']).splitlines()[0][:92]}") + return "\n".join(lines) + + +def _status_command(run_dir: Path, manifest_path: Path, *, watch: bool, interval: float, as_json: bool) -> int: + try: + while True: + status = publication_run_status(run_dir, manifest_path) + rendered = json.dumps(status, sort_keys=True) if as_json else render_publication_status(status) + if watch and sys.stdout.isatty() and not as_json: + print("\033[2J\033[H", end="") + print(rendered, flush=True) + if not watch: + return 0 + time.sleep(interval) + except KeyboardInterrupt: + print() + return 0 + + def _record_smoke_issues( artifact: dict[str, Any], model_id: str, @@ -396,14 +717,12 @@ def _record_smoke_issues( provider_options = run_info.get("provider_options") provider_options = provider_options if isinstance(provider_options, dict) else {} expected_options = { - **(registry.get("shared_fixed_options") or {}), - "OPENROUTER_PROVIDER_ONLY": entry.get("upstream_provider"), - "OPENROUTER_EXPECTED_ENDPOINT_NAME": entry.get("endpoint_name"), + **_registered_fixed_options(registry, entry), } for key, value in expected_options.items(): if provider_options.get(key) != value: issues.append(f"artifact provider option {key} does not match the registered value") - for key in registry.get("shared_absent_options") or []: + for key in _registered_absent_options(registry, entry): if provider_options.get(key) not in (None, ""): issues.append(f"artifact provider option {key} must be absent") @@ -431,9 +750,7 @@ def _record_smoke_issues( else: expected_pairs = {(seed, 1) for seed in expected_seeds} observed_pairs = { - (episode.get("seed"), episode.get("repeat", 1)) - for episode in episodes - if isinstance(episode, dict) + (episode.get("seed"), episode.get("repeat", 1)) for episode in episodes if isinstance(episode, dict) } if observed_pairs != expected_pairs: issues.append("artifact candidate episodes do not match the smoke seed/repeat panel") @@ -481,8 +798,12 @@ def _record_smoke_issues( if truncated_calls != 0: issues.append("artifact shows cap-induced truncation") reasoning_tokens = usage.get("reasoning_tokens") - if reasoning_tokens not in (None, 0): - issues.append("artifact recorded reasoning tokens in the reasoning-disabled lane") + if entry.get("reasoning_policy") == "disabled" and reasoning_tokens not in (None, 0): + issues.append("artifact recorded reasoning tokens for a reasoning-disabled model") + if entry.get("reasoning_policy") == "mandatory-minimum" and ( + not isinstance(reasoning_tokens, int) or isinstance(reasoning_tokens, bool) or reasoning_tokens < 0 + ): + issues.append("artifact is missing reasoning-token telemetry for a mandatory-reasoning model") max_output = usage.get("max_output_tokens_per_call") threshold = lane.get("cap_pressure_threshold_tokens") if not isinstance(max_output, int) or isinstance(max_output, bool): @@ -540,6 +861,8 @@ def _record_smoke(model_id: str, artifact_path: Path, manifest_path: Path) -> in "provider": entry["provider"], "model": entry["model"], "upstream_provider": entry["upstream_provider"], + "upstream_provider_slug": entry["upstream_provider_slug"], + "endpoint_tag": entry["endpoint_tag"], "endpoint_name": entry["endpoint_name"], "output_token_cap": int(lane["output_token_cap"]), "api_calls": usage["api_calls"], @@ -551,6 +874,8 @@ def _record_smoke(model_id: str, artifact_path: Path, manifest_path: Path) -> in "truncated_calls": usage["truncated_calls"], "max_output_tokens_per_call": usage["max_output_tokens_per_call"], "reasoning_tokens": usage.get("reasoning_tokens") or 0, + "reasoning_policy": entry["reasoning_policy"], + "reasoning_effort": entry.get("reasoning_effort"), "decision_failure_rate": artifact["candidate"]["summary"]["decision_failure_rate"], "contract_fingerprint": run_info["benchmark_contract"]["contract_fingerprint"], "scaffold_fingerprint": run_info["scaffold_fingerprint"], @@ -564,30 +889,53 @@ def _record_smoke(model_id: str, artifact_path: Path, manifest_path: Path) -> in return 0 +def _reusable_smoke_artifact(cell: Cell, run_dir: Path) -> Path | None: + """Return an existing raw smoke only when it still passes the current gate.""" + artifact_path = run_dir / "raw" / f"{cell.experiment_id}--{cell.cap_label}.json" + try: + artifact = _read_json(artifact_path) + registry = _read_json(PANEL_CONFIG) + lane = _read_json(LANE_CONFIG) + except (OSError, ValueError, json.JSONDecodeError): + return None + issues, _entry = _record_smoke_issues(artifact, cell.experiment_id, registry, lane) + return artifact_path if not issues else None + + def _reserve_cell(run_dir: Path, cell: Cell, measured_spend: float, ceiling: float) -> float: path = run_dir / "openrouter-reservations.json" payload = _read_json(path) if path.exists() else {"schema_version": 1, "cells": {}} reservations = payload.setdefault("cells", {}) stem = f"{cell.experiment_id}--{cell.cap_label}" - reserved_total = sum(float(value["reserved_usd"]) for value in reservations.values()) + # Only unsettled attempts remain a conservative liability. Successful + # cells are replaced by their measured account/artifact spend so historical + # worst-case reservations cannot prematurely stop a healthy serial panel. + reserved_total = sum( + float(value["reserved_usd"]) for value in reservations.values() if value.get("status") != "settled" + ) + reservation = _cell_reservation_usd(cell) if stem in reservations: - reservation = _cell_reservation_usd(cell) - committed = max(measured_spend, reserved_total) + committed = measured_spend + reserved_total if committed + reservation > ceiling: raise SystemExit( f"retry reservation would exceed spend ceiling: ${committed:.4f} + ${reservation:.4f} > ${ceiling:.4f}" ) stored = reservations[stem] - stored["reserved_usd"] = float(stored["reserved_usd"]) + reservation + stored["reserved_usd"] = ( + float(stored.get("reserved_usd") or 0) + reservation if stored.get("status") != "settled" else reservation + ) + stored["total_reserved_usd"] = float(stored.get("total_reserved_usd") or 0) + reservation stored["attempts"] = int(stored.get("attempts") or 1) + 1 + stored["status"] = "active" + stored.pop("settled_at_utc", None) + stored.pop("measured_run_spend_usd", None) _write_json_atomic(path, payload) print( f"reserved retry ${reservation:.4f} for {stem}; " f"cumulative conservative commitment ${committed + reservation:.4f}" ) return committed + reservation - reservation = _cell_reservation_usd(cell) - committed = max(measured_spend, reserved_total) + committed = measured_spend + reserved_total if committed + reservation > ceiling: raise SystemExit( f"cell reservation would exceed spend ceiling: ${committed:.4f} + ${reservation:.4f} > ${ceiling:.4f}" @@ -597,13 +945,32 @@ def _reserve_cell(run_dir: Path, cell: Cell, measured_spend: float, ceiling: flo "model": cell.model, "output_token_cap": cell.cap, "reserved_usd": reservation, + "total_reserved_usd": reservation, "attempts": 1, + "status": "active", } _write_json_atomic(path, payload) print(f"reserved ${reservation:.4f} for {stem}; cumulative conservative commitment ${committed + reservation:.4f}") return committed + reservation +def _settle_cell_reservation(run_dir: Path, cell: Cell, measured_spend: float) -> None: + """Release a successful cell's worst-case reservation in favor of measured spend.""" + path = run_dir / "openrouter-reservations.json" + if not path.exists(): + return + payload = _read_json(path) + stem = f"{cell.experiment_id}--{cell.cap_label}" + stored = (payload.get("cells") or {}).get(stem) + if not isinstance(stored, dict): + return + stored["reserved_usd"] = 0.0 + stored["status"] = "settled" + stored["settled_at_utc"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + stored["measured_run_spend_usd"] = measured_spend + _write_json_atomic(path, payload) + + def _print_command(cell: Cell, command: list[str]) -> None: options = {**cell.fixed_options, "GM_BENCH_OUTPUT_BUDGET_CELL": cell.cap_label, "GM_BENCH_WORKERS": "1"} if cell.provider == "openrouter" and cell.cap is not None: @@ -612,8 +979,12 @@ def _print_command(cell: Cell, command: list[str]) -> None: def main(argv: list[str] | None = None) -> int: + # The child CLI loads these files too, but the parent needs the provider key + # to query account usage and enforce the operator's spend ceiling before it + # launches any model process. + load_environment_files(ROOT) parser = argparse.ArgumentParser() - parser.add_argument("phase", choices=["smoke", "sweep", "panel", "record-smoke"]) + parser.add_argument("phase", choices=["smoke", "sweep", "panel", "record-smoke", "status"]) parser.add_argument("--model-id") parser.add_argument("--artifact", type=Path) parser.add_argument("--manifest", type=Path, default=SMOKE_MANIFEST) @@ -622,7 +993,14 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--dry-run", action="store_true") parser.add_argument("--preflight-only", action="store_true") parser.add_argument("--max-spend-usd", type=float) + parser.add_argument("--watch", action="store_true", help="refresh publication status until interrupted") + parser.add_argument("--interval", type=float, default=2.0, help="watch refresh interval in seconds") + parser.add_argument("--json", action="store_true", help="emit status as JSON") args = parser.parse_args(argv) + if args.phase == "status": + if args.interval <= 0: + parser.error("--interval must be positive") + return _status_command(args.run_dir, args.manifest, watch=args.watch, interval=args.interval, as_json=args.json) if args.phase == "record-smoke": if not args.model_id: parser.error("record-smoke requires --model-id") @@ -650,6 +1028,8 @@ def main(argv: list[str] | None = None) -> int: for directory in (run_dir / "raw", run_dir / "checkpoints"): if not args.dry_run: directory.mkdir(parents=True, exist_ok=True) + if not args.dry_run and not args.preflight_only: + _write_run_state(run_dir, args.phase, cells, args.max_spend_usd) budget_start: float | None = None for cell in cells: env = cell_environment(cell) @@ -657,6 +1037,11 @@ def main(argv: list[str] | None = None) -> int: _print_command(cell, command) if args.dry_run: continue + if args.phase == "smoke" and not args.preflight_only: + reusable = _reusable_smoke_artifact(cell, run_dir) + if reusable is not None: + print(f"reusing clean completed smoke without another reservation or provider call: {reusable}") + continue if cell.provider == "openrouter": try: _validate_openrouter_endpoint(cell) @@ -675,9 +1060,11 @@ def main(argv: list[str] | None = None) -> int: if spent >= args.max_spend_usd: raise SystemExit(f"spend ceiling reached: ${spent:.4f} >= ${args.max_spend_usd:.4f}") _reserve_cell(run_dir, cell, spent, args.max_spend_usd) + cell_succeeded = False try: try: subprocess.run(command, cwd=ROOT, env=env, check=True) + cell_succeeded = True except subprocess.CalledProcessError as exc: raise SystemExit( f"publication cell failed: {cell.experiment_id} cap={cell.cap_label} exit={exc.returncode}" @@ -687,6 +1074,8 @@ def main(argv: list[str] | None = None) -> int: # Report the post-cell delta even when the child exits nonzero. if args.max_spend_usd is not None and cell.provider == "openrouter": spent = _measured_spend_usd(run_dir, env, float(budget_start)) + if cell_succeeded: + _settle_cell_reservation(run_dir, cell, spent) print(f"measured OpenRouter spend for this run directory: ${spent:.4f}") if spent > args.max_spend_usd: raise SystemExit(f"spend ceiling exceeded after attempted cell: ${spent:.4f}") diff --git a/tests/test_publication.py b/tests/test_publication.py index 31798ab..12487e6 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -224,45 +224,50 @@ def test_budget_analysis_rejects_mixed_pre_registered_provenance(monkeypatch: py assert any("cost telemetry" in reason for reason in reasons) -def test_publication_model_registry_is_consistent_with_sweep() -> None: +def test_publication_model_registry_is_consistent_with_revised_lane() -> None: sweep = json.loads(Path("config/output_budget_sweep.json").read_text()) registry = json.loads(Path("config/sota_v2_models.json").read_text()) lane = json.loads(Path("config/sota_v2_lane.json").read_text()) models = registry["models"] identities = {(row["provider"], row["model"]): row for row in models} - assert 8 <= len(models) <= 12 + assert len(models) == 10 assert len({row["id"] for row in models}) == len(models) assert len(identities) == len(models) assert len({row["endpoint_name"] for row in models}) == len(models) - assert registry["selection_status"] == "provisional-awaiting-route-smokes" - assert registry["selection_frozen_at_utc"] is None - assert registry["output_token_cap"] == lane["output_token_cap"] == 1024 - assert registry["output_budget_status"] == lane["output_budget_status"] == "frozen-fixed-budget" - assert registry["output_policy_basis"] == lane["output_policy_basis"] == "fixed-safety-ceiling" - assert registry["shared_fixed_options"]["OPENROUTER_REASONING_ENABLED"] == "false" - assert "OPENROUTER_REASONING_EFFORT" in registry["shared_absent_options"] + assert registry["selection_status"] == "frozen" + assert registry["selection_frozen_at_utc"] == "2026-07-17T19:43:55Z" + assert registry["output_token_cap"] == lane["output_token_cap"] == 4096 + assert registry["output_budget_status"] == lane["output_budget_status"] == "frozen-native-reasoning-cap" + assert ( + registry["output_policy_basis"] + == lane["output_policy_basis"] + == "common-safety-ceiling-with-native-minimum-reasoning" + ) + assert {row["cohort"] for row in models} == {"big-american-lab-proprietary", "open-weight"} + assert sum(row["cohort"] == "big-american-lab-proprietary" for row in models) == 5 + assert sum(row["cohort"] == "open-weight" for row in models) == 5 assert {row["model"] for row in registry["explicit_exclusions"]} == { - "google/gemini-3.1-pro-preview", - "x-ai/grok-4.5", + "moonshotai/kimi-k3", + "nvidia/nemotron-3-ultra-550b-a55b:free", + "deepseek/deepseek-v4-pro", } assert set(registry["changed_routes_pending_smoke"]) <= {row["id"] for row in models} assert set(registry["required_smokes"]) == {row["id"] for row in models} assert lane["model_registry"] == "config/sota_v2_models.json" assert lane["minimum_headline_models"] >= 8 - for sweep_model in sweep["models"]: - registry_model = identities[(sweep_model["provider"], sweep_model["model"])] - assert sweep_model["id"] == registry_model["id"] - assert sweep_model["transport"] == registry_model["transport"] - assert sweep_model["upstream_provider"] == registry_model["upstream_provider"] - expected_options = { - **registry["shared_fixed_options"], - "OPENROUTER_PROVIDER_ONLY": registry_model["upstream_provider"], - "OPENROUTER_EXPECTED_ENDPOINT_NAME": registry_model["endpoint_name"], - } - assert sweep_model["fixed_options"] == expected_options - assert sweep_model["absent_options"] == registry["shared_absent_options"] + assert sweep["status"] == "retired-fixed-safety-cap" + for model in models: + assert model["upstream_provider_slug"] == model["endpoint_tag"] + assert model["fixed_options"]["OPENROUTER_REASONING_ENABLED"] in {"true", "false"} + if model["reasoning_policy"] == "mandatory-minimum": + assert model["reasoning_effort"] in {"minimal", "low", "max"} + assert model["fixed_options"]["OPENROUTER_REASONING_EFFORT"] == model["reasoning_effort"] + else: + assert model["reasoning_policy"] == "disabled" + assert model["reasoning_effort"] is None + assert "OPENROUTER_REASONING_EFFORT" in model["absent_options"] def _panel_analysis(rows: list[dict], *, family_size: int = 0) -> dict: @@ -286,6 +291,56 @@ def _panel_analysis(rows: list[dict], *, family_size: int = 0) -> dict: } +def test_publication_identity_issues_flags_missing_upstream_slug_without_crashing() -> None: + from web.scripts.build_leaderboard import _publication_identity_issues + + config = { + "profile": "compact", + "session": False, + "shared_fixed_options": {}, + "shared_absent_options": [], + "models": [ + { + "provider": "openrouter", + "model": "demo/model", + "transport": "gateway-api", + "upstream_provider": "Demo", + # upstream_provider_slug intentionally omitted: a registered + # model can declare upstream_provider without a slug. + "endpoint_name": "", + "fixed_options": {}, + "absent_options": [], + } + ], + } + payload = { + "run_info": { + "provider": "openrouter", + "model": "demo/model", + "transport": "gateway-api", + "profile": "compact", + "session": False, + "provider_options": { + "OPENROUTER_PROVIDER_ONLY": "demo", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "Demo", + }, + }, + "candidate": { + "summary": { + "decisions": 4, + "usage": { + "cost_usd": 0.01, + "cost_decisions": 4, + "upstream_providers": ["demo"], + }, + } + }, + } + + issues = _publication_identity_issues(payload, config) + assert any("OPENROUTER_PROVIDER_ONLY" in issue for issue in issues) + + def test_publication_gate_withholds_rows_until_minimum_panel_is_eligible() -> None: from web.scripts.build_leaderboard import publication_gate @@ -323,6 +378,38 @@ def test_publication_gate_withholds_rows_until_minimum_panel_is_eligible() -> No assert report["duplicate_headline_rows"] == 0 +def test_publication_gate_recognizes_frozen_native_reasoning_cap_status() -> None: + """A new frozen-* status string must not silently stop cap detection.""" + from web.scripts.build_leaderboard import publication_gate + + analysis = {"status": "retired"} + lane = { + "output_budget_status": "frozen-native-reasoning-cap", + "output_policy_basis": "common-safety-ceiling-with-native-minimum-reasoning", + "output_token_cap": 4096, + "minimum_headline_models": 1, + } + eligible = { + "id": "one", + "provider": "openrouter", + "model": "demo/one", + "lane": "api", + "publication_eligible": True, + "output_token_cap": 4096, + } + models, report = publication_gate( + [eligible], + analysis, + lane, + {"selection_status": "frozen"}, + panel_analysis=_panel_analysis([eligible]), + smoke_issues=[], + ) + assert models == [eligible] + assert report["frozen_output_token_cap"] == 4096 + assert report["publishable_ranking"] is True + + def test_publication_gate_rejects_wrong_cap_and_unregistered_rows() -> None: from web.scripts.build_leaderboard import publication_gate diff --git a/tests/test_publication_analysis.py b/tests/test_publication_analysis.py index 4ea7afc..42edad0 100644 --- a/tests/test_publication_analysis.py +++ b/tests/test_publication_analysis.py @@ -145,7 +145,11 @@ def _frozen_registry() -> dict: "model": "demo/model", "transport": "gateway-api", "upstream_provider": "DemoProvider", + "upstream_provider_slug": "demo-provider/fp8", + "endpoint_tag": "demo-provider/fp8", "endpoint_name": "DemoProvider | demo/model-20260716", + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + "absent_options": [], } ], } @@ -164,7 +168,8 @@ def _registered_payload() -> dict: "benchmark_contract": {"benchmark_version": "sota-v2"}, "provider_options": { "OPENROUTER_REASONING_ENABLED": "false", - "OPENROUTER_PROVIDER_ONLY": "DemoProvider", + "OPENROUTER_PROVIDER_ONLY": "demo-provider/fp8", + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": "DemoProvider", "OPENROUTER_EXPECTED_ENDPOINT_NAME": "DemoProvider | demo/model-20260716", "OPENROUTER_MAX_TOKENS": "1024", "GM_BENCH_OUTPUT_BUDGET_CELL": "1024", diff --git a/tests/test_publication_cost.py b/tests/test_publication_cost.py index 1db0615..0aaed14 100644 --- a/tests/test_publication_cost.py +++ b/tests/test_publication_cost.py @@ -5,7 +5,7 @@ import pytest -from scripts.estimate_publication_cost import RUNTIME_NOTE, RUNTIME_STATUS, estimate +from scripts.estimate_publication_cost import RUNTIME_NOTE_COMPLETE, RUNTIME_STATUS_COMPLETE, estimate def _committed_inputs() -> tuple[dict, dict, dict]: @@ -47,22 +47,19 @@ def test_costs_sum_unrounded_rows_before_contingency() -> None: assert costs["total_with_1_2x_contingency"] == pytest.approx(costs["total_unrounded"] * 1.2) -def test_runtime_is_pending_with_latency_only_where_observed() -> None: +def test_runtime_is_complete_once_every_registered_model_has_a_smoke_observation() -> None: result = estimate(*_committed_inputs()) runtime = result["runtime"] - observed = { - "openai/gpt-5.6-luna": 1.981, - "minimax/minimax-m3": 2.546, - "qwen/qwen3.5-9b": 11.622, - } - assert runtime["status"] == RUNTIME_STATUS == "pending-smoke-telemetry" - assert runtime["note"] == RUNTIME_NOTE - assert "before approving the full panel" in runtime["note"] - assert runtime["observed_api_seconds_per_decision_by_model"] == observed + assert runtime["status"] == RUNTIME_STATUS_COMPLETE == "complete-from-accepted-smokes" + assert runtime["note"] == RUNTIME_NOTE_COMPLETE + assert "4,096" in runtime["observation_source"] + assert "2026-07-17" in runtime["observation_source"] + model_names = {row["model"] for row in result["models"]} + assert set(runtime["observed_api_seconds_per_decision_by_model"]) == model_names rows_with_latency = { row["model"]: row["observed_api_seconds_per_decision"] for row in result["models"] if "observed_api_seconds_per_decision" in row } - assert rows_with_latency == observed + assert rows_with_latency == runtime["observed_api_seconds_per_decision_by_model"] diff --git a/tests/test_publication_runner.py b/tests/test_publication_runner.py index 0796b63..815162d 100644 --- a/tests/test_publication_runner.py +++ b/tests/test_publication_runner.py @@ -2,6 +2,8 @@ import hashlib import json +import os +import subprocess import sys from pathlib import Path @@ -14,10 +16,14 @@ _cell_reservation_usd, _endpoint_issues, _reserve_cell, + _settle_cell_reservation, + _write_run_state, build_cells, cell_command, cell_environment, main, + publication_run_status, + render_publication_status, ) @@ -28,6 +34,7 @@ def _frozen_panel_files( registry = json.loads(Path("config/sota_v2_models.json").read_text()) registry["selection_status"] = "frozen" lane = json.loads(Path("config/sota_v2_lane.json").read_text()) + lane["output_budget_status"] = "frozen-native-reasoning-cap" lane.pop("smoke_manifest", None) registry_path = tmp_path / "models.json" lane_path = tmp_path / "lane.json" @@ -47,7 +54,11 @@ def _valid_manifest(registry: dict, lane: dict) -> dict: "provider": model["provider"], "model": model["model"], "upstream_provider": model["upstream_provider"], + "upstream_provider_slug": model["upstream_provider_slug"], + "endpoint_tag": model["endpoint_tag"], "endpoint_name": model["endpoint_name"], + "reasoning_policy": model["reasoning_policy"], + "reasoning_effort": model["reasoning_effort"], "output_token_cap": lane["output_token_cap"], "api_calls": 4, "calls_with_finish_reason": 4, @@ -80,7 +91,9 @@ def _valid_smoke_artifact(registry: dict, lane: dict, model: dict) -> dict: "preset": "smoke", "provider_options": { **registry["shared_fixed_options"], - "OPENROUTER_PROVIDER_ONLY": model["upstream_provider"], + **model["fixed_options"], + "OPENROUTER_PROVIDER_ONLY": model["upstream_provider_slug"], + "OPENROUTER_EXPECTED_UPSTREAM_PROVIDER": model["upstream_provider"], "OPENROUTER_EXPECTED_ENDPOINT_NAME": model["endpoint_name"], "GM_BENCH_OUTPUT_BUDGET_CELL": str(lane["output_token_cap"]), }, @@ -152,26 +165,132 @@ def test_runner_rejects_cap_outside_pre_registered_sweep() -> None: def test_smoke_is_clean_and_resumes_existing_checkpoint(tmp_path: Path) -> None: assert len(build_cells("smoke")) == 10 - cell = build_cells("smoke", model_id="openrouter-qwen3.5-9b-deepinfra")[0] + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba")[0] command = cell_command(cell, tmp_path) assert cell.preset == "smoke" assert cell.repeats == 1 - assert cell.cap == 1024 + assert cell.cap == 4096 assert "--require-clean" in command - checkpoint = tmp_path / "checkpoints" / f"{cell.experiment_id}--1024.json" + checkpoint = tmp_path / "checkpoints" / f"{cell.experiment_id}--4096.json" checkpoint.parent.mkdir(parents=True) checkpoint.touch() assert "--resume" in cell_command(cell, tmp_path) +def test_smoke_reuses_only_existing_artifact_that_passes_current_gate(tmp_path: Path) -> None: + registry = json.loads(Path("config/sota_v2_models.json").read_text()) + lane = json.loads(Path("config/sota_v2_lane.json").read_text()) + model = registry["models"][0] + cell = build_cells("smoke", model_id=model["id"])[0] + raw = tmp_path / "raw" + raw.mkdir() + artifact_path = raw / f"{cell.experiment_id}--{cell.cap_label}.json" + + assert publication_runner._reusable_smoke_artifact(cell, tmp_path) is None + artifact = _valid_smoke_artifact(registry, lane, model) + artifact_path.write_text(json.dumps(artifact)) + assert publication_runner._reusable_smoke_artifact(cell, tmp_path) == artifact_path + + artifact["candidate"]["summary"]["failed_decisions"] = 1 + artifact_path.write_text(json.dumps(artifact)) + assert publication_runner._reusable_smoke_artifact(cell, tmp_path) is None + + +def test_preflight_only_still_validates_endpoint_despite_reusable_smoke_artifact( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """--preflight-only must never take the cached-artifact shortcut that skips it.""" + registry = json.loads(Path("config/sota_v2_models.json").read_text()) + lane = json.loads(Path("config/sota_v2_lane.json").read_text()) + model = registry["models"][0] + cell = build_cells("smoke", model_id=model["id"])[0] + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + artifact_path = raw_dir / f"{cell.experiment_id}--{cell.cap_label}.json" + artifact_path.write_text(json.dumps(_valid_smoke_artifact(registry, lane, model))) + assert publication_runner._reusable_smoke_artifact(cell, tmp_path) == artifact_path + + validated: list[str] = [] + monkeypatch.setattr( + publication_runner, + "_validate_openrouter_endpoint", + lambda cell: validated.append(cell.experiment_id), + ) + monkeypatch.setattr( + publication_runner.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args=args, returncode=0), + ) + + assert ( + main( + [ + "smoke", + "--model-id", + model["id"], + "--run-dir", + str(tmp_path), + "--preflight-only", + ] + ) + == 0 + ) + assert validated == [cell.experiment_id] + + def test_smoke_rejects_cap_that_differs_from_frozen_lane() -> None: with pytest.raises(ValueError, match="differs from frozen panel smoke cap"): - build_cells("smoke", cap=2048) + build_cells("smoke", cap=1024) + + +def test_smoke_applies_each_models_registered_reasoning_policy() -> None: + disabled = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai")[0] + mandatory = build_cells("smoke", model_id="openrouter-gemini-3.5-flash-google-ai-studio")[0] + + assert disabled.fixed_options["OPENROUTER_REASONING_ENABLED"] == "false" + assert "OPENROUTER_REASONING_EFFORT" in disabled.absent_options + assert mandatory.fixed_options["OPENROUTER_REASONING_ENABLED"] == "true" + assert mandatory.fixed_options["OPENROUTER_REASONING_EFFORT"] == "minimal" + assert mandatory.fixed_options["OPENROUTER_PROVIDER_ONLY"] == "google-ai-studio" + + +def _minimal_registered_model(**overrides: object) -> dict: + model = { + "id": "test-model", + "provider": "openrouter", + "model": "test/model", + "upstream_provider": "Test", + "upstream_provider_slug": "test", + "endpoint_tag": "test", + "endpoint_name": "Test | test/model", + "reasoning_policy": "disabled", + "reasoning_effort": None, + "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, + } + model.update(overrides) + return model -def test_panel_is_locked_until_model_registry_is_frozen() -> None: - with pytest.raises(ValueError, match="locked until"): - build_cells("panel") +def test_validate_models_rejects_disabled_policy_with_a_fixed_reasoning_effort() -> None: + model = _minimal_registered_model( + fixed_options={"OPENROUTER_REASONING_ENABLED": "false", "OPENROUTER_REASONING_EFFORT": "minimal"} + ) + with pytest.raises(ValueError, match="invalid disabled reasoning policy"): + publication_runner._validate_models([model]) + + +def test_validate_models_rejects_mandatory_minimum_policy_with_no_effort_declared() -> None: + model = _minimal_registered_model( + reasoning_policy="mandatory-minimum", + reasoning_effort=None, + fixed_options={"OPENROUTER_REASONING_ENABLED": "true"}, + ) + with pytest.raises(ValueError, match="invalid mandatory reasoning policy"): + publication_runner._validate_models([model]) + + +def test_committed_panel_is_unlocked_after_registry_and_smoke_freeze() -> None: + assert len(build_cells("panel")) == 10 def test_panel_stays_locked_when_frozen_registry_has_no_manifest( @@ -210,9 +329,9 @@ def test_panel_stays_locked_for_invalid_smoke_entry( elif mutation == "truncated": manifest["entries"][model_id]["truncated_calls"] = 1 elif mutation == "wrong-cap": - manifest["entries"][model_id]["output_token_cap"] = 2048 + manifest["entries"][model_id]["output_token_cap"] = 1024 elif mutation == "peak": - manifest["entries"][model_id]["max_output_tokens_per_call"] = 768 + manifest["entries"][model_id]["max_output_tokens_per_call"] = 3072 else: manifest["entries"][model_id]["scaffold_fingerprint"] = "wrong" manifest_path.write_text(json.dumps(manifest)) @@ -229,6 +348,22 @@ def test_panel_unlocks_with_complete_valid_smoke_manifest( assert len(build_cells("panel")) == 10 +@pytest.mark.parametrize("bad_reasoning_tokens", [True, -1]) +def test_panel_stays_locked_for_invalid_manifest_reasoning_tokens( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + bad_reasoning_tokens: object, +) -> None: + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + manifest = _valid_manifest(registry, lane) + model = next(m for m in registry["models"] if m["reasoning_policy"] == "mandatory-minimum") + manifest["entries"][model["id"]]["reasoning_tokens"] = bad_reasoning_tokens + manifest_path.write_text(json.dumps(manifest)) + + with pytest.raises(ValueError, match="missing reasoning-token telemetry"): + build_cells("panel") + + def test_record_smoke_writes_accepted_manifest_entry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -405,7 +540,7 @@ def test_record_smoke_requires_call_telemetry_for_protocol_repairs( ("field", "value", "message"), [ ("truncated_calls", 1, "cap-induced truncation"), - ("max_output_tokens_per_call", 768, "cap-pressure threshold"), + ("max_output_tokens_per_call", 3072, "cap-pressure threshold"), ("scaffold_fingerprint", "wrong", "different prompt scaffold"), ("decision_failure_rate", 0.1, "decision_failure_rate must be zero"), ], @@ -448,6 +583,38 @@ def test_record_smoke_refuses_invalid_artifact_without_writing_manifest( assert not manifest_path.exists() +@pytest.mark.parametrize("bad_reasoning_tokens", [True, -1]) +def test_record_smoke_refuses_invalid_reasoning_token_telemetry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + bad_reasoning_tokens: object, +) -> None: + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + model = next(m for m in registry["models"] if m["reasoning_policy"] == "mandatory-minimum") + artifact = _valid_smoke_artifact(registry, lane, model) + artifact["candidate"]["summary"]["usage"]["reasoning_tokens"] = bad_reasoning_tokens + artifact_path = tmp_path / "bad-reasoning-tokens.json" + artifact_path.write_text(json.dumps(artifact)) + + assert ( + main( + [ + "record-smoke", + "--model-id", + model["id"], + "--artifact", + str(artifact_path), + "--manifest", + str(manifest_path), + ] + ) + == 1 + ) + assert "missing reasoning-token telemetry" in capsys.readouterr().err + assert not manifest_path.exists() + + def test_artifact_spend_uses_completed_result_telemetry(tmp_path: Path) -> None: raw = tmp_path / "raw" raw.mkdir() @@ -464,7 +631,7 @@ def test_paid_openrouter_run_requires_explicit_spend_ceiling( [ "smoke", "--model-id", - "openrouter-qwen3.5-9b-deepinfra", + "openrouter-qwen3.7-plus-alibaba", "--run-dir", str(tmp_path), ] @@ -481,7 +648,7 @@ def test_paid_sweep_is_locked_after_policy_is_retired(tmp_path: Path, capsys: py def test_cell_reservation_blocks_launch_before_ceiling_overrun(tmp_path: Path) -> None: - cell = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai", cap=1024)[0] + cell = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai", cap=4096)[0] reservation = _cell_reservation_usd(cell) assert 0 < reservation < 1 with pytest.raises(SystemExit, match="reservation would exceed"): @@ -490,7 +657,7 @@ def test_cell_reservation_blocks_launch_before_ceiling_overrun(tmp_path: Path) - def test_retry_reservation_accounts_for_fresh_full_attempt(tmp_path: Path) -> None: - cell = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai", cap=1024)[0] + cell = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai", cap=4096)[0] reservation = _cell_reservation_usd(cell) path = tmp_path / "openrouter-reservations.json" path.write_text( @@ -498,10 +665,10 @@ def test_retry_reservation_accounts_for_fresh_full_attempt(tmp_path: Path) -> No { "schema_version": 1, "cells": { - f"{cell.experiment_id}--1024": { + f"{cell.experiment_id}--4096": { "experiment_id": cell.experiment_id, "model": cell.model, - "output_token_cap": 1024, + "output_token_cap": 4096, "reserved_usd": reservation, "attempts": 1, } @@ -513,30 +680,236 @@ def test_retry_reservation_accounts_for_fresh_full_attempt(tmp_path: Path) -> No _reserve_cell(tmp_path, cell, measured_spend=0.9, ceiling=0.9 + reservation / 2) committed = _reserve_cell(tmp_path, cell, measured_spend=0.1, ceiling=reservation * 2 + 0.1) - stored = json.loads(path.read_text())["cells"][f"{cell.experiment_id}--1024"] - assert committed == pytest.approx(0.1 + reservation) + stored = json.loads(path.read_text())["cells"][f"{cell.experiment_id}--4096"] + assert committed == pytest.approx(reservation * 2 + 0.1) assert stored["reserved_usd"] == pytest.approx(reservation * 2) + assert stored["status"] == "active" assert stored["attempts"] == 2 +def test_successful_cell_settlement_releases_reservation_for_next_cell(tmp_path: Path) -> None: + first = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai")[0] + second = build_cells("smoke", model_id="openrouter-claude-sonnet-5-bedrock")[0] + first_reservation = _cell_reservation_usd(first) + second_reservation = _cell_reservation_usd(second) + + _reserve_cell(tmp_path, first, measured_spend=0.0, ceiling=first_reservation + 0.01) + _settle_cell_reservation(tmp_path, first, measured_spend=0.02) + committed = _reserve_cell( + tmp_path, + second, + measured_spend=0.02, + ceiling=0.02 + second_reservation + 0.01, + ) + + cells = json.loads((tmp_path / "openrouter-reservations.json").read_text())["cells"] + assert cells[f"{first.experiment_id}--4096"]["reserved_usd"] == 0 + assert cells[f"{first.experiment_id}--4096"]["status"] == "settled" + assert cells[f"{second.experiment_id}--4096"]["status"] == "active" + assert committed == pytest.approx(0.02 + second_reservation) + + +def test_unsettled_failed_attempt_remains_part_of_next_reservation_guard(tmp_path: Path) -> None: + first = build_cells("smoke", model_id="openrouter-gpt-5.6-luna-openai")[0] + second = build_cells("smoke", model_id="openrouter-claude-sonnet-5-bedrock")[0] + first_reservation = _cell_reservation_usd(first) + second_reservation = _cell_reservation_usd(second) + _reserve_cell(tmp_path, first, measured_spend=0.0, ceiling=first_reservation + 0.01) + + with pytest.raises(SystemExit, match="reservation would exceed"): + _reserve_cell( + tmp_path, + second, + measured_spend=0.02, + ceiling=0.02 + second_reservation + first_reservation / 2, + ) + + def test_endpoint_preflight_requires_frozen_healthy_capable_route() -> None: - cell = build_cells("smoke", model_id="openrouter-qwen3.5-9b-deepinfra", cap=1024)[0] + cell = build_cells("smoke", model_id="openrouter-qwen3.7-plus-alibaba", cap=4096)[0] valid = { "data": { "endpoints": [ { - "provider_name": "DeepInfra", + "provider_name": "Alibaba", + "tag": "alibaba", "name": cell.endpoint_name, "status": 0, - "max_completion_tokens": 4096, + "max_completion_tokens": 65536, "supported_parameters": ["max_tokens", "response_format", "reasoning"], } ] } } assert _endpoint_issues(cell, valid) == [] - valid["data"]["endpoints"][0]["name"] = "DeepInfra | replaced-snapshot" + valid["data"]["endpoints"][0]["tag"] = "alibaba/other-tier" + assert "no healthy OpenRouter endpoint" in _endpoint_issues(cell, valid)[0] + valid["data"]["endpoints"][0]["tag"] = "alibaba" + valid["data"]["endpoints"][0]["name"] = "Alibaba | replaced-snapshot" assert "no healthy OpenRouter endpoint" in _endpoint_issues(cell, valid)[0] valid["data"]["endpoints"][0]["name"] = cell.endpoint_name valid["data"]["endpoints"][0]["supported_parameters"] = ["max_tokens", "response_format"] assert "cannot honor required parameters" in _endpoint_issues(cell, valid)[0] + + +def test_endpoint_preflight_allows_registered_prompt_only_json_route() -> None: + cell = build_cells("smoke", model_id="openrouter-tencent-hy3-free-novita", cap=4096)[0] + assert cell.fixed_options["OPENROUTER_JSON_MODE"] == "false" + payload = { + "data": { + "endpoints": [ + { + "provider_name": "Novita", + "tag": "novita", + "name": cell.endpoint_name, + "status": 0, + "max_completion_tokens": 262144, + "supported_parameters": ["max_tokens", "reasoning", "structured_outputs"], + } + ] + } + } + assert _endpoint_issues(cell, payload) == [] + + +def test_publication_status_tracks_active_progress_spend_and_ceiling( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import fcntl + + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + cell = build_cells("smoke", model_id=registry["models"][0]["id"])[0] + _write_run_state(tmp_path, "smoke", [cell], 1.0) + checkpoint_path = tmp_path / "checkpoints" / f"{cell.experiment_id}--{lane['output_token_cap']}.json" + checkpoint_path.parent.mkdir(parents=True) + checkpoint_path.write_text( + json.dumps( + { + "format": "gm-bench-model-checkpoint-v1", + "status": "running", + "seeds": [1], + "seasons": 1, + "repeats": 1, + "completed": [], + "episodes": [], + } + ) + ) + lock_path = checkpoint_path.with_suffix(".json.lock") + lock_path.write_text(f"pid={os.getpid()}\n") + lock_descriptor = os.open(lock_path, os.O_RDONLY) + fcntl.flock(lock_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + reservations = tmp_path / "openrouter-reservations.json" + reservations.write_text( + json.dumps( + { + "schema_version": 1, + "cells": { + f"{cell.experiment_id}--{cell.cap_label}": { + "experiment_id": cell.experiment_id, + "reserved_usd": 0.01, + } + }, + } + ) + ) + + try: + status = publication_run_status(tmp_path, manifest_path) + row = next(row for row in status["rows"] if row["model_id"] == cell.experiment_id) + assert status["phase"] == "smoke" + assert status["spend_ceiling_usd"] == 1.0 + assert status["active_cells"] == 1 + assert status["reserved_spend_usd"] == 0.01 + assert row["state"] == "running" + assert row["reserved_usd"] == pytest.approx(0.01) + assert (row["completed_episodes"], row["total_episodes"]) == (0, 1) + assert (row["completed_decisions"], row["total_decisions"]) == (0, 4) + finally: + os.close(lock_descriptor) + + +def test_publication_status_distinguishes_complete_and_accepted_smokes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + first, second = registry["models"][:2] + _write_run_state(tmp_path, "smoke", [build_cells("smoke")[0]], 1.0) + raw_dir = tmp_path / "raw" + raw_dir.mkdir(parents=True) + for model, cost in ((first, 0.01), (second, 0.02)): + artifact = _valid_smoke_artifact(registry, lane, model) + artifact["candidate"]["summary"]["usage"]["cost_usd"] = cost + (raw_dir / f"{model['id']}--{lane['output_token_cap']}.json").write_text(json.dumps(artifact)) + manifest_path.write_text(json.dumps(_valid_manifest(registry, lane))) + + status = publication_run_status(tmp_path, manifest_path) + rows = {row["model_id"]: row for row in status["rows"]} + assert rows[first["id"]]["state"] == "accepted" + assert rows[first["id"]]["completed_decisions"] == 4 + assert status["artifact_spend_usd"] == pytest.approx(0.03) + assert status["accepted_smokes"] == 10 + assert "accepted smokes: 10/10" in render_publication_status(status) + + +def test_publication_status_does_not_mark_accepted_without_this_runs_raw_artifact( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An accepted manifest entry must not stand in for this run/cap's own artifact.""" + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + model = registry["models"][0] + _write_run_state(tmp_path, "smoke", [build_cells("smoke")[0]], 1.0) + manifest_path.write_text(json.dumps(_valid_manifest(registry, lane))) + + status = publication_run_status(tmp_path, manifest_path) + row = next(row for row in status["rows"] if row["model_id"] == model["id"]) + assert row["state"] == "queued" + assert row["smoke_accepted"] is True + + +def test_publication_status_does_not_promote_complete_state_on_stale_manifest_cap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A manifest entry accepted at a different cap must not upgrade this cell's state.""" + registry, lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + model = registry["models"][0] + _write_run_state(tmp_path, "smoke", [build_cells("smoke")[0]], 1.0) + raw_dir = tmp_path / "raw" + raw_dir.mkdir(parents=True) + artifact = _valid_smoke_artifact(registry, lane, model) + (raw_dir / f"{model['id']}--{lane['output_token_cap']}.json").write_text(json.dumps(artifact)) + + manifest = _valid_manifest(registry, lane) + manifest["entries"][model["id"]]["output_token_cap"] = lane["output_token_cap"] + 1 + manifest_path.write_text(json.dumps(manifest)) + + status = publication_run_status(tmp_path, manifest_path) + row = next(row for row in status["rows"] if row["model_id"] == model["id"]) + assert row["state"] == "complete" + assert row["smoke_accepted"] is True + + +def test_panel_status_keeps_smoke_acceptance_separate_from_panel_progress( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, _lane, manifest_path = _frozen_panel_files(tmp_path, monkeypatch) + cell = build_cells("smoke")[0] + _write_run_state(tmp_path, "panel", [cell], 60.0) + manifest_path.write_text(json.dumps(_valid_manifest(registry, _lane))) + + status = publication_run_status(tmp_path, manifest_path) + assert status["accepted_smokes"] == 10 + assert status["completed_cells"] == 0 + assert {row["state"] for row in status["rows"]} == {"queued"} + assert {row["total_episodes"] for row in status["rows"]} == {24} + assert {row["total_decisions"] for row in status["rows"]} == {480} + + +def test_status_command_prints_snapshot_without_creating_run_files( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert main(["status", "--run-dir", str(tmp_path)]) == 0 + output = capsys.readouterr().out + assert "GM-Bench publication run" in output + assert "openrouter-gpt-5.6-luna-openai" in output + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_smoke_manifest.py b/tests/test_smoke_manifest.py index 0f0bf76..4d8cd08 100644 --- a/tests/test_smoke_manifest.py +++ b/tests/test_smoke_manifest.py @@ -25,7 +25,11 @@ def _valid_manifest(registry: dict, lane: dict) -> dict: "provider": model["provider"], "model": model["model"], "upstream_provider": model["upstream_provider"], + "upstream_provider_slug": model["upstream_provider_slug"], + "endpoint_tag": model["endpoint_tag"], "endpoint_name": model["endpoint_name"], + "reasoning_policy": model["reasoning_policy"], + "reasoning_effort": model["reasoning_effort"], "output_token_cap": lane["output_token_cap"], "api_calls": 4, "calls_with_finish_reason": 4, diff --git a/web/scripts/build_leaderboard.py b/web/scripts/build_leaderboard.py index 8a56da0..d7bc443 100644 --- a/web/scripts/build_leaderboard.py +++ b/web/scripts/build_leaderboard.py @@ -54,6 +54,10 @@ # a fallback for external `--agent-cmd` rows with no `run_info.transport`; # built-in providers should always carry a transport (see gm_bench/providers.py). CLI_HARNESS_PROVIDERS = {"codex", "claude", "cursor", "opencode"} +# Output-policy bases that a completed smoke gate can freeze the lane under; +# kept in sync with the equivalent allow-list in build_cells() (the "smoke" +# branch) in scripts/run_publication_matrix.py. +FROZEN_OUTPUT_POLICY_BASES = {"fixed-safety-ceiling", "common-safety-ceiling-with-native-minimum-reasoning"} def _lane(provider: str, transport: str | None) -> str: @@ -113,15 +117,23 @@ def _publication_identity_issues(payload: dict[str, Any], config: dict[str, Any] options = run_info.get("provider_options") or {} expected_options = { **(config.get("shared_fixed_options") or {}), + **(registered.get("fixed_options") or {}), } if registered.get("upstream_provider") not in (None, ""): - expected_options["OPENROUTER_PROVIDER_ONLY"] = registered["upstream_provider"] + # .get(), not indexing: a registered model can declare upstream_provider + # without a slug, and that mismatch must surface as a gate issue below + # rather than crash the builder or silently skip the check. + expected_options["OPENROUTER_PROVIDER_ONLY"] = registered.get("upstream_provider_slug") + expected_options["OPENROUTER_EXPECTED_UPSTREAM_PROVIDER"] = registered["upstream_provider"] if registered.get("endpoint_name") not in (None, ""): expected_options["OPENROUTER_EXPECTED_ENDPOINT_NAME"] = registered["endpoint_name"] for key, expected in expected_options.items(): if str(options.get(key, "")) != str(expected): issues.append(f"provider option {key} does not match the pre-registered value") - for key in config.get("shared_absent_options") or []: + absent_options = [*(config.get("shared_absent_options") or []), *(registered.get("absent_options") or [])] + for key in absent_options: + if key in expected_options: + continue if options.get(str(key)) not in (None, ""): issues.append(f"provider option {key} must be absent for the headline lane") observed = sorted({str(value) for value in usage.get("upstream_providers") or [] if value}) @@ -368,16 +380,19 @@ def publication_gate( ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Return only publishable rows plus an explicit gate report.""" - frozen_statuses = {"frozen-saturation", "frozen-fixed-budget"} frozen_cap = lane_config.get("output_token_cap") policy_basis = lane_config.get("output_policy_basis") - fixed_safety_ceiling = policy_basis == "fixed-safety-ceiling" + policy_frozen = ( + policy_basis in FROZEN_OUTPUT_POLICY_BASES or analysis.get("status") == "complete-needs-interpretation" + ) registry_frozen = model_config.get("selection_status") == "frozen" + # Recognize any "frozen-*" status rather than an enumerated set: the lane + # has moved through frozen-saturation, frozen-fixed-budget, and now + # frozen-native-reasoning-cap, and an enumerated set silently stops + # detecting freezes each time a new frozen status string is introduced. + output_budget_status = str(lane_config.get("output_budget_status") or "") lane_frozen = ( - (fixed_safety_ceiling or analysis.get("status") == "complete-needs-interpretation") - and lane_config.get("output_budget_status") in frozen_statuses - and isinstance(frozen_cap, int) - and frozen_cap > 0 + policy_frozen and output_budget_status.startswith("frozen") and isinstance(frozen_cap, int) and frozen_cap > 0 ) candidates = [ row @@ -420,6 +435,13 @@ def publication_gate( "smoke_gate_issues": smoke_gate_issues, "panel_analysis_ready": panel_analysis_ready, "panel_analysis_issues": panel_analysis_issues[:10], + # `analysis` is the retired output-budget sweep artifact; its own + # "models" field is that sweep's three-model panel and must not leak + # into the gate report as if it described the current registry. + "models": [ + {"id": model.get("id"), "provider": model.get("provider"), "model": model.get("model")} + for model in model_config.get("models") or [] + ], } if not registry_frozen: publication["reason"] = "model registry remains provisional until every registered route passes its smoke" diff --git a/web/src/data/leaderboard.json b/web/src/data/leaderboard.json index 7ec4d3f..b2b397f 100644 --- a/web/src/data/leaderboard.json +++ b/web/src/data/leaderboard.json @@ -80,91 +80,63 @@ "duplicate_cells": [], "duplicate_headline_rows": 0, "eligible_headline_models": 0, - "frozen_output_token_cap": 1024, + "frozen_output_token_cap": 4096, "minimum_headline_models": 8, "missing": [], - "model_registry_frozen": false, + "model_registry_frozen": true, "models": [ { - "absent_options": [ - "OPENROUTER_REASONING_EFFORT", - "OPENROUTER_REASONING_MAX_TOKENS", - "OPENROUTER_TEMPERATURE", - "OPENROUTER_QUANTIZATIONS", - "OPENROUTER_ZDR" - ], - "endpoint_name": "DeepInfra | qwen/qwen3.5-9b-20260310", - "fixed_options": { - "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", - "OPENROUTER_ALLOW_FALLBACKS": "false", - "OPENROUTER_DATA_COLLECTION": "deny", - "OPENROUTER_EXPECTED_ENDPOINT_NAME": "DeepInfra | qwen/qwen3.5-9b-20260310", - "OPENROUTER_JSON_MODE": "true", - "OPENROUTER_PROVIDER_ONLY": "DeepInfra", - "OPENROUTER_PROVIDER_SORT": "price", - "OPENROUTER_REASONING_ENABLED": "false", - "OPENROUTER_REQUIRE_PARAMETERS": "true" - }, - "id": "openrouter-qwen3.5-9b-deepinfra", - "model": "qwen/qwen3.5-9b", - "provider": "openrouter", - "transport": "gateway-api", - "upstream_provider": "DeepInfra" - }, - { - "absent_options": [ - "OPENROUTER_REASONING_EFFORT", - "OPENROUTER_REASONING_MAX_TOKENS", - "OPENROUTER_TEMPERATURE", - "OPENROUTER_QUANTIZATIONS", - "OPENROUTER_ZDR" - ], - "endpoint_name": "OpenAI | openai/gpt-5.6-luna-20260709", - "fixed_options": { - "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", - "OPENROUTER_ALLOW_FALLBACKS": "false", - "OPENROUTER_DATA_COLLECTION": "deny", - "OPENROUTER_EXPECTED_ENDPOINT_NAME": "OpenAI | openai/gpt-5.6-luna-20260709", - "OPENROUTER_JSON_MODE": "true", - "OPENROUTER_PROVIDER_ONLY": "OpenAI", - "OPENROUTER_PROVIDER_SORT": "price", - "OPENROUTER_REASONING_ENABLED": "false", - "OPENROUTER_REQUIRE_PARAMETERS": "true" - }, "id": "openrouter-gpt-5.6-luna-openai", "model": "openai/gpt-5.6-luna", - "provider": "openrouter", - "transport": "gateway-api", - "upstream_provider": "OpenAI" + "provider": "openrouter" + }, + { + "id": "openrouter-claude-sonnet-5-bedrock", + "model": "anthropic/claude-sonnet-5", + "provider": "openrouter" + }, + { + "id": "openrouter-gemini-3.5-flash-google-ai-studio", + "model": "google/gemini-3.5-flash", + "provider": "openrouter" + }, + { + "id": "openrouter-grok-4.5-xai", + "model": "x-ai/grok-4.5", + "provider": "openrouter" + }, + { + "id": "openrouter-muse-spark-1.1-meta", + "model": "meta/muse-spark-1.1", + "provider": "openrouter" + }, + { + "id": "openrouter-glm-5.2-zai", + "model": "z-ai/glm-5.2", + "provider": "openrouter" }, { - "absent_options": [ - "OPENROUTER_REASONING_EFFORT", - "OPENROUTER_REASONING_MAX_TOKENS", - "OPENROUTER_TEMPERATURE", - "OPENROUTER_QUANTIZATIONS", - "OPENROUTER_ZDR" - ], - "endpoint_name": "Minimax | minimax/minimax-m3-20260531", - "fixed_options": { - "GM_BENCH_PROTOCOL_REPAIR_ATTEMPTS": "1", - "OPENROUTER_ALLOW_FALLBACKS": "false", - "OPENROUTER_DATA_COLLECTION": "deny", - "OPENROUTER_EXPECTED_ENDPOINT_NAME": "Minimax | minimax/minimax-m3-20260531", - "OPENROUTER_JSON_MODE": "true", - "OPENROUTER_PROVIDER_ONLY": "Minimax", - "OPENROUTER_PROVIDER_SORT": "price", - "OPENROUTER_REASONING_ENABLED": "false", - "OPENROUTER_REQUIRE_PARAMETERS": "true" - }, "id": "openrouter-minimax-m3-minimax", "model": "minimax/minimax-m3", - "provider": "openrouter", - "transport": "gateway-api", - "upstream_provider": "Minimax" + "provider": "openrouter" + }, + { + "id": "openrouter-qwen3.7-plus-alibaba", + "model": "qwen/qwen3.7-plus", + "provider": "openrouter" + }, + { + "id": "openrouter-mistral-medium-3.5-mistral", + "model": "mistralai/mistral-medium-3-5", + "provider": "openrouter" + }, + { + "id": "openrouter-tencent-hy3-free-novita", + "model": "tencent/hy3:free", + "provider": "openrouter" } ], - "output_policy_basis": "fixed-safety-ceiling", + "output_policy_basis": "common-safety-ceiling-with-native-minimum-reasoning", "panel_analysis_issues": [ "publication panel analysis artifact is missing" ], @@ -177,12 +149,10 @@ ], "points": [], "publishable_ranking": false, - "reason": "model registry remains provisional until every registered route passes its smoke", + "reason": "frozen lane has 0 eligible headline models; at least 8 are required", "rejected_artifacts": [], "schema_version": 1, - "smoke_gate_issues": [ - "smoke manifest is missing; record every registered-model smoke before the panel" - ], + "smoke_gate_issues": [], "status": "retired" }, "updated": ""