From 3ed3d3e4606cebb59ee6aa9be12459cf76e3d47a Mon Sep 17 00:00:00 2001 From: Ned Cutler Date: Sat, 18 Jul 2026 15:56:52 -0400 Subject: [PATCH] Replace unhealthy GLM route with Novita --- config/openrouter_pricing_snapshot.json | 10 +++--- config/sota_v2_models.json | 20 ++++++------ config/sota_v2_smoke_manifest.json | 18 +++++------ docs/PUBLISH_READINESS.md | 11 ++++--- ...sota-v2-glm-novita-amendment-2026-07-18.md | 32 +++++++++++++++++++ docs/run_logs/sota-v2-smokes-2026-07-17.md | 4 ++- .../analysis/output-budget-cost-estimate.json | 26 +++++++-------- tests/test_publication.py | 8 ++++- web/src/data/leaderboard.json | 2 +- 9 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md diff --git a/config/openrouter_pricing_snapshot.json b/config/openrouter_pricing_snapshot.json index 4cd38b1..99aa31f 100644 --- a/config/openrouter_pricing_snapshot.json +++ b/config/openrouter_pricing_snapshot.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "checked_at_utc": "2026-07-17T19:37:57Z", + "checked_at_utc": "2026-07-18T19:50:59Z", "source": "https://openrouter.ai/api/v1/models and per-model /endpoints responses", "currency": "USD", "rates_are_per_token": true, @@ -11,8 +11,8 @@ "runtime_contingency_multiplier": 1.5 }, "runtime_observations": { - "source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md", - "observed_at_utc": "2026-07-17T19:42:44Z", + "source": "accepted 4,096-token native-minimum-reasoning smokes from 2026-07-17 plus the 2026-07-18 GLM 5.2 Novita route amendment; see docs/run_logs/sota-v2-smokes-2026-07-17.md and docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md", + "observed_at_utc": "2026-07-18T19:52:17Z", "api_seconds_per_decision": { "anthropic/claude-sonnet-5": 8.657, "google/gemini-3.5-flash": 1.497, @@ -23,7 +23,7 @@ "qwen/qwen3.7-plus": 3.586, "tencent/hy3:free": 4.087, "x-ai/grok-4.5": 7.683, - "z-ai/glm-5.2": 6.656 + "z-ai/glm-5.2": 8.045 } }, "models": { @@ -39,6 +39,6 @@ "qwen/qwen3.7-plus": {"prompt": 0.00000032, "completion": 0.00000128}, "tencent/hy3:free": {"prompt": 0, "completion": 0}, "x-ai/grok-4.5": {"prompt": 0.000002, "completion": 0.000006}, - "z-ai/glm-5.2": {"prompt": 0.0000014, "completion": 0.0000044} + "z-ai/glm-5.2": {"prompt": 0.0000002758, "completion": 0.0000008668} } } diff --git a/config/sota_v2_models.json b/config/sota_v2_models.json index 2f1957a..290ecd5 100644 --- a/config/sota_v2_models.json +++ b/config/sota_v2_models.json @@ -8,9 +8,9 @@ "preset": "leaderboard", "repeats": 3, "selection_status": "frozen", - "selection_frozen_at_utc": "2026-07-17T19:43:55Z", - "selection_revision": "2026-07-17-phase-one-after-smoke-exclusions", - "catalog_checked_at_utc": "2026-07-17T19:37:57Z", + "selection_frozen_at_utc": "2026-07-18T19:50:59Z", + "selection_revision": "2026-07-18-glm-novita-route-amendment", + "catalog_checked_at_utc": "2026-07-18T19:50:59Z", "selection_policy": "User-curated current frontier panel: five large American-lab proprietary models and five leading open-weight models. Every OpenRouter route pins an exact provider slug, endpoint tag, and dated endpoint name with fallbacks disabled. Models run at native-minimum reasoning: disabled when optional and the lowest catalog-supported effort when mandatory. Kimi K3 is parked after its clean smoke failed and demonstrated cap pressure; the exact Nemotron 3 Ultra and DeepSeek V4 Pro routes are parked after bounded HTTP 404 infrastructure failures.", "models": [ { @@ -94,20 +94,20 @@ "role": "Meta proprietary frontier anchor" }, { - "id": "openrouter-glm-5.2-zai", + "id": "openrouter-glm-5.2-novita", "provider": "openrouter", "model": "z-ai/glm-5.2", "transport": "gateway-api", "cohort": "open-weight", - "upstream_provider": "Z.AI", - "upstream_provider_slug": "z-ai/fp8", - "endpoint_tag": "z-ai/fp8", - "endpoint_name": "Z.AI | z-ai/glm-5.2-20260616", + "upstream_provider": "Novita", + "upstream_provider_slug": "novita/fp8", + "endpoint_tag": "novita/fp8", + "endpoint_name": "Novita | z-ai/glm-5.2-20260616", "reasoning_policy": "disabled", "reasoning_effort": null, "fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"}, "absent_options": ["OPENROUTER_REASONING_EFFORT"], - "role": "Z.ai open-weight anchor on first-party FP8 route" + "role": "Z.ai open-weight anchor on healthy Novita FP8 route; replaces the unhealthy first-party endpoint" }, { "id": "openrouter-minimax-m3-minimax", @@ -191,7 +191,7 @@ "openrouter-gemini-3.5-flash-google-ai-studio", "openrouter-grok-4.5-xai", "openrouter-muse-spark-1.1-meta", - "openrouter-glm-5.2-zai", + "openrouter-glm-5.2-novita", "openrouter-minimax-m3-minimax", "openrouter-qwen3.7-plus-alibaba", "openrouter-mistral-medium-3.5-mistral", diff --git a/config/sota_v2_smoke_manifest.json b/config/sota_v2_smoke_manifest.json index 17d7b60..22b29cd 100644 --- a/config/sota_v2_smoke_manifest.json +++ b/config/sota_v2_smoke_manifest.json @@ -54,19 +54,19 @@ "upstream_provider": "Google AI Studio", "upstream_provider_slug": "google-ai-studio" }, - "openrouter-glm-5.2-zai": { + "openrouter-glm-5.2-novita": { "accepted": true, "api_calls": 4, - "artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-glm-5.2-zai--4096.json", - "artifact_sha256": "37da29291723f868a10668cbbe9c667108693800227d56f058db38a1ec61cb18", + "artifact_path": "data/publication-runs/smoke-glm-novita-4096-2026-07-18/raw/openrouter-glm-5.2-novita--4096.json", + "artifact_sha256": "244ae595503e2288ac4051c5f8c10b9ebff46eb159924b14a668c43541e88c52", "calls_with_finish_reason": 4, "contract_fingerprint": "558e8f35ea1d66b9", "cost_decisions": 4, "decision_failure_rate": 0.0, "decisions_with_usage": 4, - "endpoint_name": "Z.AI | z-ai/glm-5.2-20260616", - "endpoint_tag": "z-ai/fp8", - "max_output_tokens_per_call": 311, + "endpoint_name": "Novita | z-ai/glm-5.2-20260616", + "endpoint_tag": "novita/fp8", + "max_output_tokens_per_call": 325, "model": "z-ai/glm-5.2", "output_token_cap": 4096, "protocol_repair_attempts": 0, @@ -75,11 +75,11 @@ "reasoning_effort": null, "reasoning_policy": "disabled", "reasoning_tokens": 0, - "recorded_at_utc": "2026-07-17T19:43:25+00:00", + "recorded_at_utc": "2026-07-18T19:52:17+00:00", "scaffold_fingerprint": "d7321ad9d0a739b4", "truncated_calls": 0, - "upstream_provider": "Z.AI", - "upstream_provider_slug": "z-ai/fp8" + "upstream_provider": "Novita", + "upstream_provider_slug": "novita/fp8" }, "openrouter-gpt-5.6-luna-openai": { "accepted": true, diff --git a/docs/PUBLISH_READINESS.md b/docs/PUBLISH_READINESS.md index 2663203..05b60b0 100644 --- a/docs/PUBLISH_READINESS.md +++ b/docs/PUBLISH_READINESS.md @@ -10,10 +10,12 @@ **Last reviewed:** 2026-07-18 **Current target:** Publish a validated `sota-v2` leaderboard, accompanying blog post, GitHub release, and public site. -**Current state:** The phase-one ten-model registry is frozen as of 2026-07-17, +**Current state:** The phase-one ten-model registry is frozen as amended on 2026-07-18, uses exact OpenRouter endpoint tags, and completed its fresh 4,096-token native-minimum-reasoning smoke gate; all ten registered routes are accepted with -zero failures, zero truncations, and no cap-pressure trigger. Earlier 1,024-token +zero failures, zero truncations, and no cap-pressure trigger. GLM 5.2 moved from +the unhealthy first-party Z.AI endpoint to an exact Novita FP8 route and passed +a fresh replacement smoke. Earlier 1,024-token smokes are retained only as superseded operational evidence. The full panel run is the remaining blocker before any eligible model row exists. **Current weekly focus:** [#63 — Publication sprint: freeze and ship GM-Bench @@ -264,7 +266,7 @@ The provisional restarted headline panel is: | Gemini 3.5 Flash | Google AI Studio | Google fast frontier anchor; mandatory `minimal` reasoning | | Grok 4.5 | xAI | xAI frontier anchor; mandatory `low` reasoning | | Muse Spark 1.1 | Meta | Meta frontier anchor; mandatory `minimal` reasoning | -| GLM 5.2 | Z.AI FP8 | First-party open-weight anchor | +| GLM 5.2 | Novita FP8 | Open-weight anchor; replacement for the unhealthy first-party Z.AI route | | MiniMax M3 | MiniMax FP8 | First-party open-weight anchor | | Qwen 3.7 Plus | Alibaba | Qwen frontier open-weight anchor | | Mistral Medium 3.5 | Mistral | European open-weight anchor | @@ -559,8 +561,9 @@ decision and why. | 2026-07-17 | Switch Nemotron 3 Ultra from paid Together to the exact free Nvidia route before its first smoke. | OpenRouter's live catalog exposed `nvidia/nemotron-3-ultra-550b-a55b:free` at zero input/output cost on one healthy first-party Nvidia endpoint with no listed expiration. It supports optional reasoning but does not advertise `response_format`. | Pin the dated Nvidia free endpoint with fallbacks and reasoning disabled, use prompt-only JSON plus the clean-smoke gate, and regenerate the cost plan before continuing. | | 2026-07-17 | Park the listed-free Nemotron 3 Ultra route after bounded infrastructure failure. | The exact Nvidia endpoint passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint as infrastructure evidence, do not retry or silently substitute the paid Together route, reduce the phase-one registry and Holm family to eleven, and continue the untouched models. | | 2026-07-17 | Park DeepSeek V4 Pro after the same bounded infrastructure failure. | The exact first-party DeepSeek route passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint, do not retry or substitute a different route, reduce the phase-one registry and Holm family to ten, and continue Mistral and HY3. | -| 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Artifact-reported spend was $0.728909 against a $2 ceiling. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. | +| 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Accepted-route artifact spend was $0.427613; total campaign spend was $0.728909 including the excluded Kimi diagnostic. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. | | 2026-07-18 | Settle successful serial-cell reservations against measured spend. | The runner retained every historical worst-case reservation, so a healthy panel could stop against cumulative hypothetical spend even after completed artifacts and the OpenRouter account established a much lower real cost. | Mark successful-cell reservations settled after post-cell spend measurement, keep failed/interrupted reservations active, and evaluate each next cell against measured spend plus only unresolved liabilities. | +| 2026-07-18 | Amend GLM 5.2 from the unhealthy first-party Z.AI FP8 endpoint to Novita FP8. | The frozen `z-ai/fp8` endpoint remained at OpenRouter status `-2` across repeated launch preflights, while the exact dated Novita FP8 endpoint was healthy and advertised the common lane parameters. No full-panel GLM result existed. | Pin `novita/fp8`, replace rather than reuse the Z.AI smoke entry, refresh route pricing/runtime evidence, and require a clean exact-route smoke before restoring panel unlock. The replacement smoke completed 4/4 decisions with zero failures or truncations for $0.009225. | ## Experiment and release log diff --git a/docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md b/docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md new file mode 100644 index 0000000..cc1a4e1 --- /dev/null +++ b/docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md @@ -0,0 +1,32 @@ +# sota-v2 GLM 5.2 route amendment — 2026-07-18 + +Repeated launch preflights found the frozen first-party Z.AI FP8 endpoint for +`z-ai/glm-5.2` unhealthy in OpenRouter's live catalog (`status: -2`). No +full-panel GLM result existed, so the registry was amended before panel data to +the exact dated Novita FP8 endpoint: + +- provider slug and endpoint tag: `novita/fp8` +- endpoint name: `Novita | z-ai/glm-5.2-20260616` +- reasoning: disabled, unchanged from the prior route +- output cap: 4,096 tokens, unchanged +- fallbacks: disabled + +The replacement endpoint passed live catalog and authentication preflight. Its +serial benchmark smoke then completed all four decisions with zero failed +decisions, zero truncations, complete usage and finish-reason telemetry, and +exact Novita provenance. Peak output was 325 tokens, safely below the 3,072-token +cap-pressure threshold. Artifact-reported spend was **$0.009225** and API time +was 32.1795 seconds, or 8.045 seconds per decision. + +The accepted artifact is retained outside Git at +`data/publication-runs/smoke-glm-novita-4096-2026-07-18/raw/openrouter-glm-5.2-novita--4096.json` +with SHA-256 +`244ae595503e2288ac4051c5f8c10b9ebff46eb159924b14a668c43541e88c52`. +The superseded Z.AI artifact remains retained as historical evidence but no +longer unlocks the current registry. + +Replacing the $0.047053 Z.AI smoke with this $0.009225 Novita smoke brings the +current ten-route accepted-smoke total to **$0.389785**. Scaling observed smoke +spend by the panel's 120x decision-count ratio gives an empirical panel estimate +of **$46.7742**; the operator ceiling remains authoritative and may stop the run +before completion if measured spend diverges upward. diff --git a/docs/run_logs/sota-v2-smokes-2026-07-17.md b/docs/run_logs/sota-v2-smokes-2026-07-17.md index a28bf98..c9d8d9a 100644 --- a/docs/run_logs/sota-v2-smokes-2026-07-17.md +++ b/docs/run_logs/sota-v2-smokes-2026-07-17.md @@ -8,7 +8,9 @@ Git under `data/publication-runs/smoke-frontier-4096-2026-07-17/`. All ten registered models completed four decisions with zero failed decisions, zero truncations, complete cost/finish-reason telemetry, and the exact pinned -upstream route. Artifact-reported spend was **$0.728909**. +upstream route. Accepted-route artifact spend was **$0.427613**. Total smoke +campaign spend was **$0.728909** including the excluded $0.301296 Kimi K3 +diagnostic described below. | Model | Cost | Peak output tokens per call | | --- | ---: | ---: | diff --git a/results/analysis/output-budget-cost-estimate.json b/results/analysis/output-budget-cost-estimate.json index e058b53..cfab669 100644 --- a/results/analysis/output-budget-cost-estimate.json +++ b/results/analysis/output-budget-cost-estimate.json @@ -25,10 +25,10 @@ "total_calls": 4840 }, "costs_usd": { - "panel": 140.8533504, - "smoke": 1.17377792, - "total_unrounded": 142.02712832, - "total_with_1_2x_contingency": 170.432553984 + "panel": 129.589868544, + "smoke": 1.0799155712, + "total_unrounded": 130.6697841152, + "total_with_1_2x_contingency": 156.80374093824 }, "models": [ { @@ -82,14 +82,14 @@ "smoke_cost_usd": 0.109632 }, { - "cost_per_decision_usd": 0.0292224, - "experiment_id": "openrouter-glm-5.2-zai", + "cost_per_decision_usd": 0.0057568128, + "experiment_id": "openrouter-glm-5.2-novita", "model": "z-ai/glm-5.2", - "observed_api_seconds_per_decision": 6.656, + "observed_api_seconds_per_decision": 8.045, "panel_calls": 480, - "panel_cost_usd": 14.026752, + "panel_cost_usd": 2.763270144, "smoke_calls": 4, - "smoke_cost_usd": 0.1168896 + "smoke_cost_usd": 0.0230272512 }, { "cost_per_decision_usd": 0.0073152, @@ -132,10 +132,10 @@ "smoke_cost_usd": 0.0 } ], - "pricing_checked_at_utc": "2026-07-17T19:37:57Z", + "pricing_checked_at_utc": "2026-07-18T19:50:59Z", "runtime": { "note": "Runtime observations are sourced from every currently registered model's accepted smoke; recheck before paid full-panel runs if pricing, routes, or prompts change.", - "observation_source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md", + "observation_source": "accepted 4,096-token native-minimum-reasoning smokes from 2026-07-17 plus the 2026-07-18 GLM 5.2 Novita route amendment; see docs/run_logs/sota-v2-smokes-2026-07-17.md and docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md", "observed_api_seconds_per_decision_by_model": { "anthropic/claude-sonnet-5": 8.657, "google/gemini-3.5-flash": 1.497, @@ -146,9 +146,9 @@ "qwen/qwen3.7-plus": 3.586, "tencent/hy3:free": 4.087, "x-ai/grok-4.5": 7.683, - "z-ai/glm-5.2": 6.656 + "z-ai/glm-5.2": 8.045 }, - "observed_at_utc": "2026-07-17T19:42:44Z", + "observed_at_utc": "2026-07-18T19:52:17Z", "status": "complete-from-accepted-smokes" }, "schema_version": 2, diff --git a/tests/test_publication.py b/tests/test_publication.py index 12487e6..2353b61 100644 --- a/tests/test_publication.py +++ b/tests/test_publication.py @@ -236,7 +236,8 @@ def test_publication_model_registry_is_consistent_with_revised_lane() -> None: assert len(identities) == len(models) assert len({row["endpoint_name"] for row in models}) == len(models) assert registry["selection_status"] == "frozen" - assert registry["selection_frozen_at_utc"] == "2026-07-17T19:43:55Z" + assert registry["selection_frozen_at_utc"] == "2026-07-18T19:50:59Z" + assert registry["selection_revision"] == "2026-07-18-glm-novita-route-amendment" assert registry["output_token_cap"] == lane["output_token_cap"] == 4096 assert registry["output_budget_status"] == lane["output_budget_status"] == "frozen-native-reasoning-cap" assert ( @@ -257,6 +258,11 @@ def test_publication_model_registry_is_consistent_with_revised_lane() -> None: assert lane["model_registry"] == "config/sota_v2_models.json" assert lane["minimum_headline_models"] >= 8 + glm = identities[("openrouter", "z-ai/glm-5.2")] + assert glm["id"] == "openrouter-glm-5.2-novita" + assert glm["upstream_provider"] == "Novita" + assert glm["endpoint_tag"] == "novita/fp8" + assert sweep["status"] == "retired-fixed-safety-cap" for model in models: assert model["upstream_provider_slug"] == model["endpoint_tag"] diff --git a/web/src/data/leaderboard.json b/web/src/data/leaderboard.json index b2b397f..1c84f76 100644 --- a/web/src/data/leaderboard.json +++ b/web/src/data/leaderboard.json @@ -111,7 +111,7 @@ "provider": "openrouter" }, { - "id": "openrouter-glm-5.2-zai", + "id": "openrouter-glm-5.2-novita", "model": "z-ai/glm-5.2", "provider": "openrouter" },