Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions config/openrouter_pricing_snapshot.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"schema_version": 1,
"checked_at_utc": "2026-07-17T19:37:57Z",
"checked_at_utc": "2026-07-18T19:50:59Z",
"source": "https://openrouter.ai/api/v1/models and per-model /endpoints responses",
"currency": "USD",
"rates_are_per_token": true,
Expand All @@ -11,8 +11,8 @@
"runtime_contingency_multiplier": 1.5
},
"runtime_observations": {
"source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md",
"observed_at_utc": "2026-07-17T19:42:44Z",
"source": "accepted 4,096-token native-minimum-reasoning smokes from 2026-07-17 plus the 2026-07-18 GLM 5.2 Novita route amendment; see docs/run_logs/sota-v2-smokes-2026-07-17.md and docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md",
"observed_at_utc": "2026-07-18T19:52:17Z",
"api_seconds_per_decision": {
"anthropic/claude-sonnet-5": 8.657,
"google/gemini-3.5-flash": 1.497,
Expand All @@ -23,7 +23,7 @@
"qwen/qwen3.7-plus": 3.586,
"tencent/hy3:free": 4.087,
"x-ai/grok-4.5": 7.683,
"z-ai/glm-5.2": 6.656
"z-ai/glm-5.2": 8.045
}
},
"models": {
Expand All @@ -39,6 +39,6 @@
"qwen/qwen3.7-plus": {"prompt": 0.00000032, "completion": 0.00000128},
"tencent/hy3:free": {"prompt": 0, "completion": 0},
"x-ai/grok-4.5": {"prompt": 0.000002, "completion": 0.000006},
"z-ai/glm-5.2": {"prompt": 0.0000014, "completion": 0.0000044}
"z-ai/glm-5.2": {"prompt": 0.0000002758, "completion": 0.0000008668}
}
}
20 changes: 10 additions & 10 deletions config/sota_v2_models.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
"preset": "leaderboard",
"repeats": 3,
"selection_status": "frozen",
"selection_frozen_at_utc": "2026-07-17T19:43:55Z",
"selection_revision": "2026-07-17-phase-one-after-smoke-exclusions",
"catalog_checked_at_utc": "2026-07-17T19:37:57Z",
"selection_frozen_at_utc": "2026-07-18T19:50:59Z",
"selection_revision": "2026-07-18-glm-novita-route-amendment",
"catalog_checked_at_utc": "2026-07-18T19:50:59Z",
"selection_policy": "User-curated current frontier panel: five large American-lab proprietary models and five leading open-weight models. Every OpenRouter route pins an exact provider slug, endpoint tag, and dated endpoint name with fallbacks disabled. Models run at native-minimum reasoning: disabled when optional and the lowest catalog-supported effort when mandatory. Kimi K3 is parked after its clean smoke failed and demonstrated cap pressure; the exact Nemotron 3 Ultra and DeepSeek V4 Pro routes are parked after bounded HTTP 404 infrastructure failures.",
"models": [
{
Expand Down Expand Up @@ -94,20 +94,20 @@
"role": "Meta proprietary frontier anchor"
},
{
"id": "openrouter-glm-5.2-zai",
"id": "openrouter-glm-5.2-novita",
"provider": "openrouter",
"model": "z-ai/glm-5.2",
"transport": "gateway-api",
"cohort": "open-weight",
"upstream_provider": "Z.AI",
"upstream_provider_slug": "z-ai/fp8",
"endpoint_tag": "z-ai/fp8",
"endpoint_name": "Z.AI | z-ai/glm-5.2-20260616",
"upstream_provider": "Novita",
"upstream_provider_slug": "novita/fp8",
"endpoint_tag": "novita/fp8",
"endpoint_name": "Novita | z-ai/glm-5.2-20260616",
"reasoning_policy": "disabled",
"reasoning_effort": null,
"fixed_options": {"OPENROUTER_REASONING_ENABLED": "false"},
"absent_options": ["OPENROUTER_REASONING_EFFORT"],
"role": "Z.ai open-weight anchor on first-party FP8 route"
"role": "Z.ai open-weight anchor on healthy Novita FP8 route; replaces the unhealthy first-party endpoint"
},
{
"id": "openrouter-minimax-m3-minimax",
Expand Down Expand Up @@ -191,7 +191,7 @@
"openrouter-gemini-3.5-flash-google-ai-studio",
"openrouter-grok-4.5-xai",
"openrouter-muse-spark-1.1-meta",
"openrouter-glm-5.2-zai",
"openrouter-glm-5.2-novita",
"openrouter-minimax-m3-minimax",
"openrouter-qwen3.7-plus-alibaba",
"openrouter-mistral-medium-3.5-mistral",
Expand Down
18 changes: 9 additions & 9 deletions config/sota_v2_smoke_manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,19 @@
"upstream_provider": "Google AI Studio",
"upstream_provider_slug": "google-ai-studio"
},
"openrouter-glm-5.2-zai": {
"openrouter-glm-5.2-novita": {
"accepted": true,
"api_calls": 4,
"artifact_path": "data/publication-runs/smoke-frontier-4096-2026-07-17/raw/openrouter-glm-5.2-zai--4096.json",
"artifact_sha256": "37da29291723f868a10668cbbe9c667108693800227d56f058db38a1ec61cb18",
"artifact_path": "data/publication-runs/smoke-glm-novita-4096-2026-07-18/raw/openrouter-glm-5.2-novita--4096.json",
"artifact_sha256": "244ae595503e2288ac4051c5f8c10b9ebff46eb159924b14a668c43541e88c52",
"calls_with_finish_reason": 4,
"contract_fingerprint": "558e8f35ea1d66b9",
"cost_decisions": 4,
"decision_failure_rate": 0.0,
"decisions_with_usage": 4,
"endpoint_name": "Z.AI | z-ai/glm-5.2-20260616",
"endpoint_tag": "z-ai/fp8",
"max_output_tokens_per_call": 311,
"endpoint_name": "Novita | z-ai/glm-5.2-20260616",
"endpoint_tag": "novita/fp8",
"max_output_tokens_per_call": 325,
"model": "z-ai/glm-5.2",
"output_token_cap": 4096,
"protocol_repair_attempts": 0,
Expand All @@ -75,11 +75,11 @@
"reasoning_effort": null,
"reasoning_policy": "disabled",
"reasoning_tokens": 0,
"recorded_at_utc": "2026-07-17T19:43:25+00:00",
"recorded_at_utc": "2026-07-18T19:52:17+00:00",
"scaffold_fingerprint": "d7321ad9d0a739b4",
"truncated_calls": 0,
"upstream_provider": "Z.AI",
"upstream_provider_slug": "z-ai/fp8"
"upstream_provider": "Novita",
"upstream_provider_slug": "novita/fp8"
},
"openrouter-gpt-5.6-luna-openai": {
"accepted": true,
Expand Down
11 changes: 7 additions & 4 deletions docs/PUBLISH_READINESS.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
**Last reviewed:** 2026-07-18
**Current target:** Publish a validated `sota-v2` leaderboard, accompanying blog
post, GitHub release, and public site.
**Current state:** The phase-one ten-model registry is frozen as of 2026-07-17,
**Current state:** The phase-one ten-model registry is frozen as amended on 2026-07-18,
uses exact OpenRouter endpoint tags, and completed its fresh 4,096-token
native-minimum-reasoning smoke gate; all ten registered routes are accepted with
zero failures, zero truncations, and no cap-pressure trigger. Earlier 1,024-token
zero failures, zero truncations, and no cap-pressure trigger. GLM 5.2 moved from
the unhealthy first-party Z.AI endpoint to an exact Novita FP8 route and passed
a fresh replacement smoke. Earlier 1,024-token
smokes are retained only as superseded operational evidence. The full panel run
is the remaining blocker before any eligible model row exists.
**Current weekly focus:** [#63 — Publication sprint: freeze and ship GM-Bench
Expand Down Expand Up @@ -264,7 +266,7 @@ The provisional restarted headline panel is:
| Gemini 3.5 Flash | Google AI Studio | Google fast frontier anchor; mandatory `minimal` reasoning |
| Grok 4.5 | xAI | xAI frontier anchor; mandatory `low` reasoning |
| Muse Spark 1.1 | Meta | Meta frontier anchor; mandatory `minimal` reasoning |
| GLM 5.2 | Z.AI FP8 | First-party open-weight anchor |
| GLM 5.2 | Novita FP8 | Open-weight anchor; replacement for the unhealthy first-party Z.AI route |
| MiniMax M3 | MiniMax FP8 | First-party open-weight anchor |
| Qwen 3.7 Plus | Alibaba | Qwen frontier open-weight anchor |
| Mistral Medium 3.5 | Mistral | European open-weight anchor |
Expand Down Expand Up @@ -559,8 +561,9 @@ decision and why.
| 2026-07-17 | Switch Nemotron 3 Ultra from paid Together to the exact free Nvidia route before its first smoke. | OpenRouter's live catalog exposed `nvidia/nemotron-3-ultra-550b-a55b:free` at zero input/output cost on one healthy first-party Nvidia endpoint with no listed expiration. It supports optional reasoning but does not advertise `response_format`. | Pin the dated Nvidia free endpoint with fallbacks and reasoning disabled, use prompt-only JSON plus the clean-smoke gate, and regenerate the cost plan before continuing. |
| 2026-07-17 | Park the listed-free Nemotron 3 Ultra route after bounded infrastructure failure. | The exact Nvidia endpoint passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint as infrastructure evidence, do not retry or silently substitute the paid Together route, reduce the phase-one registry and Holm family to eleven, and continue the untouched models. |
| 2026-07-17 | Park DeepSeek V4 Pro after the same bounded infrastructure failure. | The exact first-party DeepSeek route passed live catalog and authentication preflight, then both permitted real chat-completion attempts returned `HTTP 404 Not Found`. Fail-fast stopped before a complete episode and OpenRouter reported no incremental spend. | Preserve the checkpoint, do not retry or substitute a different route, reduce the phase-one registry and Holm family to ten, and continue Mistral and HY3. |
| 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Artifact-reported spend was $0.728909 against a $2 ceiling. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. |
| 2026-07-17 | Freeze the ten-model phase-one registry and 4,096-token lane after the accepted smoke gate. | All ten registered models completed four decisions with zero failed decisions and zero truncations. Peak per-call output was 1,432 tokens, below the 3,072 cap-pressure trigger. Accepted-route artifact spend was $0.427613; total campaign spend was $0.728909 including the excluded Kimi diagnostic. | Record all ten manifest entries, freeze the registry and native-reasoning cap, retain excluded-model diagnostics, regenerate the cost plan, and unlock panel dry-runs without starting paid panel cells. |
| 2026-07-18 | Settle successful serial-cell reservations against measured spend. | The runner retained every historical worst-case reservation, so a healthy panel could stop against cumulative hypothetical spend even after completed artifacts and the OpenRouter account established a much lower real cost. | Mark successful-cell reservations settled after post-cell spend measurement, keep failed/interrupted reservations active, and evaluate each next cell against measured spend plus only unresolved liabilities. |
| 2026-07-18 | Amend GLM 5.2 from the unhealthy first-party Z.AI FP8 endpoint to Novita FP8. | The frozen `z-ai/fp8` endpoint remained at OpenRouter status `-2` across repeated launch preflights, while the exact dated Novita FP8 endpoint was healthy and advertised the common lane parameters. No full-panel GLM result existed. | Pin `novita/fp8`, replace rather than reuse the Z.AI smoke entry, refresh route pricing/runtime evidence, and require a clean exact-route smoke before restoring panel unlock. The replacement smoke completed 4/4 decisions with zero failures or truncations for $0.009225. |

## Experiment and release log

Expand Down
32 changes: 32 additions & 0 deletions docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# sota-v2 GLM 5.2 route amendment — 2026-07-18

Repeated launch preflights found the frozen first-party Z.AI FP8 endpoint for
`z-ai/glm-5.2` unhealthy in OpenRouter's live catalog (`status: -2`). No
full-panel GLM result existed, so the registry was amended before panel data to
the exact dated Novita FP8 endpoint:

- provider slug and endpoint tag: `novita/fp8`
- endpoint name: `Novita | z-ai/glm-5.2-20260616`
- reasoning: disabled, unchanged from the prior route
- output cap: 4,096 tokens, unchanged
- fallbacks: disabled

The replacement endpoint passed live catalog and authentication preflight. Its
serial benchmark smoke then completed all four decisions with zero failed
decisions, zero truncations, complete usage and finish-reason telemetry, and
exact Novita provenance. Peak output was 325 tokens, safely below the 3,072-token
cap-pressure threshold. Artifact-reported spend was **$0.009225** and API time
was 32.1795 seconds, or 8.045 seconds per decision.

The accepted artifact is retained outside Git at
`data/publication-runs/smoke-glm-novita-4096-2026-07-18/raw/openrouter-glm-5.2-novita--4096.json`
with SHA-256
`244ae595503e2288ac4051c5f8c10b9ebff46eb159924b14a668c43541e88c52`.
The superseded Z.AI artifact remains retained as historical evidence but no
longer unlocks the current registry.

Replacing the $0.047053 Z.AI smoke with this $0.009225 Novita smoke brings the
current ten-route accepted-smoke total to **$0.389785**. Scaling observed smoke
spend by the panel's 120x decision-count ratio gives an empirical panel estimate
of **$46.7742**; the operator ceiling remains authoritative and may stop the run
before completion if measured spend diverges upward.
4 changes: 3 additions & 1 deletion docs/run_logs/sota-v2-smokes-2026-07-17.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ Git under `data/publication-runs/smoke-frontier-4096-2026-07-17/`.

All ten registered models completed four decisions with zero failed decisions,
zero truncations, complete cost/finish-reason telemetry, and the exact pinned
upstream route. Artifact-reported spend was **$0.728909**.
upstream route. Accepted-route artifact spend was **$0.427613**. Total smoke
campaign spend was **$0.728909** including the excluded $0.301296 Kimi K3
diagnostic described below.

| Model | Cost | Peak output tokens per call |
| --- | ---: | ---: |
Expand Down
26 changes: 13 additions & 13 deletions results/analysis/output-budget-cost-estimate.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
"total_calls": 4840
},
"costs_usd": {
"panel": 140.8533504,
"smoke": 1.17377792,
"total_unrounded": 142.02712832,
"total_with_1_2x_contingency": 170.432553984
"panel": 129.589868544,
"smoke": 1.0799155712,
"total_unrounded": 130.6697841152,
"total_with_1_2x_contingency": 156.80374093824
},
"models": [
{
Expand Down Expand Up @@ -82,14 +82,14 @@
"smoke_cost_usd": 0.109632
},
{
"cost_per_decision_usd": 0.0292224,
"experiment_id": "openrouter-glm-5.2-zai",
"cost_per_decision_usd": 0.0057568128,
"experiment_id": "openrouter-glm-5.2-novita",
"model": "z-ai/glm-5.2",
"observed_api_seconds_per_decision": 6.656,
"observed_api_seconds_per_decision": 8.045,
"panel_calls": 480,
"panel_cost_usd": 14.026752,
"panel_cost_usd": 2.763270144,
"smoke_calls": 4,
"smoke_cost_usd": 0.1168896
"smoke_cost_usd": 0.0230272512
},
{
"cost_per_decision_usd": 0.0073152,
Expand Down Expand Up @@ -132,10 +132,10 @@
"smoke_cost_usd": 0.0
}
],
"pricing_checked_at_utc": "2026-07-17T19:37:57Z",
"pricing_checked_at_utc": "2026-07-18T19:50:59Z",
"runtime": {
"note": "Runtime observations are sourced from every currently registered model's accepted smoke; recheck before paid full-panel runs if pricing, routes, or prompts change.",
"observation_source": "accepted 4,096-token native-minimum-reasoning smokes, 2026-07-17 (data/publication-runs/smoke-frontier-4096-2026-07-17/); see docs/run_logs/sota-v2-smokes-2026-07-17.md",
"observation_source": "accepted 4,096-token native-minimum-reasoning smokes from 2026-07-17 plus the 2026-07-18 GLM 5.2 Novita route amendment; see docs/run_logs/sota-v2-smokes-2026-07-17.md and docs/run_logs/sota-v2-glm-novita-amendment-2026-07-18.md",
"observed_api_seconds_per_decision_by_model": {
"anthropic/claude-sonnet-5": 8.657,
"google/gemini-3.5-flash": 1.497,
Expand All @@ -146,9 +146,9 @@
"qwen/qwen3.7-plus": 3.586,
"tencent/hy3:free": 4.087,
"x-ai/grok-4.5": 7.683,
"z-ai/glm-5.2": 6.656
"z-ai/glm-5.2": 8.045
},
"observed_at_utc": "2026-07-17T19:42:44Z",
"observed_at_utc": "2026-07-18T19:52:17Z",
"status": "complete-from-accepted-smokes"
},
"schema_version": 2,
Expand Down
8 changes: 7 additions & 1 deletion tests/test_publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ def test_publication_model_registry_is_consistent_with_revised_lane() -> None:
assert len(identities) == len(models)
assert len({row["endpoint_name"] for row in models}) == len(models)
assert registry["selection_status"] == "frozen"
assert registry["selection_frozen_at_utc"] == "2026-07-17T19:43:55Z"
assert registry["selection_frozen_at_utc"] == "2026-07-18T19:50:59Z"
assert registry["selection_revision"] == "2026-07-18-glm-novita-route-amendment"
assert registry["output_token_cap"] == lane["output_token_cap"] == 4096
assert registry["output_budget_status"] == lane["output_budget_status"] == "frozen-native-reasoning-cap"
assert (
Expand All @@ -257,6 +258,11 @@ def test_publication_model_registry_is_consistent_with_revised_lane() -> None:
assert lane["model_registry"] == "config/sota_v2_models.json"
assert lane["minimum_headline_models"] >= 8

glm = identities[("openrouter", "z-ai/glm-5.2")]
assert glm["id"] == "openrouter-glm-5.2-novita"
assert glm["upstream_provider"] == "Novita"
assert glm["endpoint_tag"] == "novita/fp8"

assert sweep["status"] == "retired-fixed-safety-cap"
for model in models:
assert model["upstream_provider_slug"] == model["endpoint_tag"]
Expand Down
2 changes: 1 addition & 1 deletion web/src/data/leaderboard.json
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@
"provider": "openrouter"
},
{
"id": "openrouter-glm-5.2-zai",
"id": "openrouter-glm-5.2-novita",
"model": "z-ai/glm-5.2",
"provider": "openrouter"
},
Expand Down