Fix issue #84 P0 defects: non-finite inputs, walk-away windows, artifact integrity

docs/blog/sota-v2-findings.md

@@ -101,6 +101,20 @@ front office.
   actual model token use still ranged from 7,611 to 10,764 tokens per decision.
 - The result evaluates model-plus-standardized-scaffold systems on this task;
   it is not a claim about general intelligence or universal strategic ability.
+- The published panel and scripted references are committed and public. That
+  makes replay easy, but it also permits benchmark-specific adaptation; future
+  generalization claims need a private or prospective panel.
+- Opponents are scripted front-office policies, not autonomous learning GMs.
+  Morale is shown as decorative state but does not currently affect strength,
+  development, or score; readers should not treat it as a decision-relevant mechanic.
+
+## Decision history
+
+The protocol was frozen through an iterative process, not discovered whole.
+The public [publish-readiness decision log](../PUBLISH_READINESS.md) records
+panel reshuffles, cap-policy changes, route substitutions, and evidence resets.
+“Frozen” describes the released protocol and artifact set; it does not erase
+those pre-freeze researcher decisions.
 
 ## Audit and reproduce
 

examples/gm_agent_common.py

@@ -233,15 +233,15 @@ def parse_actions(text: Any) -> list[dict[str, Any]]:
         raise ValueError(f"model response content must be a string, got {type(text).__name__}")
     stripped = strip_terminal_codes(text).strip()
     try:
-        parsed = json.loads(stripped)
+        parsed = json.loads(stripped, parse_constant=_reject_non_finite_json_constant)
         return _actions_from_json(parsed)
     except json.JSONDecodeError:
         pass
 
     fenced = re.search(r"```(?:json)?\s*(.*?)\s*```", stripped, re.DOTALL)
     if fenced:
         try:
-            parsed = json.loads(fenced.group(1))
+            parsed = json.loads(fenced.group(1), parse_constant=_reject_non_finite_json_constant)
             return _actions_from_json(parsed)
         except json.JSONDecodeError:
             pass
@@ -260,6 +260,10 @@ def parse_actions(text: Any) -> list[dict[str, Any]]:
     raise ValueError("model did not return a JSON action array")
 
 
+def _reject_non_finite_json_constant(value: str) -> None:
+    raise ValueError(f"non-finite JSON number {value!r} is not allowed")
+
+
 def _actions_from_json(parsed: Any) -> list[dict[str, Any]]:
     if isinstance(parsed, dict) and isinstance(parsed.get("actions"), list):
         parsed = parsed["actions"]
@@ -327,7 +331,7 @@ def _has_usable_value(value: Any) -> bool:
 
 
 def _scan_json_values(text: str, opener: str) -> list[Any]:
-    decoder = json.JSONDecoder()
+    decoder = json.JSONDecoder(parse_constant=_reject_non_finite_json_constant)
     values: list[Any] = []
     for match in re.finditer(re.escape(opener), text):
         try:

gm_bench/official.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import copy
+import math
 import os
 import re
 from dataclasses import dataclass
@@ -332,6 +333,9 @@ def validate_leaderboard_payload(
         elif expected_seeds is not None:
             _validate_episode_panel(errors, f"baseline[{name}]", baseline, expected_seeds, expected_seasons, repeats=1)
 
+    if isinstance(payload.get("publication"), dict) and not redacted_private:
+        _validate_compact_integrity(errors, payload, candidate, baselines, expected_seeds or [])
+
     normalized = _dict(payload.get("normalized"))
     paired = _dict(payload.get("paired"))
     if not normalized:
@@ -656,6 +660,69 @@ def _validate_episode_panel(
         errors.append(f"{label}.episodes missing seed/repeat pairs: {missing}")
 
 
+def _validate_compact_integrity(
+    errors: list[str],
+    payload: dict[str, Any],
+    candidate: dict[str, Any],
+    baselines: list[dict[str, Any]],
+    seeds: list[int],
+) -> None:
+    """Treat compact summaries as derived views, never independent evidence."""
+    _validate_finite_numbers(errors, payload)
+    if not candidate or not baselines or not seeds:
+        return
+    from gm_bench.runner import _paired_analysis, _precise_mean_score, summarize_episodes
+
+    rebuilt_candidate = {**candidate, "summary": summarize_episodes(candidate.get("episodes") or [])}
+    rebuilt_baselines = [
+        {**baseline, "summary": summarize_episodes(baseline.get("episodes") or [])} for baseline in baselines
+    ]
+    _compare_present_values(errors, "candidate.summary", candidate.get("summary"), rebuilt_candidate["summary"])
+    for original, rebuilt in zip(baselines, rebuilt_baselines, strict=True):
+        _compare_present_values(
+            errors, f"baseline[{original.get('agent', 'unknown')}].summary", original.get("summary"), rebuilt["summary"]
+        )
+    baseline_mean = sum(_precise_mean_score(block) for block in rebuilt_baselines) / len(rebuilt_baselines)
+    candidate_mean = _precise_mean_score(rebuilt_candidate)
+    expected_normalized = {
+        "candidate_mean_score": round(candidate_mean, 3),
+        "baseline_panel_mean_score": round(baseline_mean, 3),
+        "score_lift": round(candidate_mean - baseline_mean, 3),
+        "score_lift_pct": round(((candidate_mean / baseline_mean) - 1.0) * 100.0, 2) if baseline_mean else 0.0,
+        "candidate_illegal_actions": rebuilt_candidate["summary"]["illegal_actions"],
+        "baseline_illegal_actions": sum(block["summary"]["illegal_actions"] for block in rebuilt_baselines),
+    }
+    _compare_present_values(errors, "normalized", payload.get("normalized"), expected_normalized)
+    _compare_present_values(
+        errors, "paired", payload.get("paired"), _paired_analysis(seeds, rebuilt_candidate, rebuilt_baselines)
+    )
+
+
+def _validate_finite_numbers(errors: list[str], value: Any, path: str = "payload") -> None:
+    if isinstance(value, float) and not math.isfinite(value):
+        errors.append(f"{path} must not contain a non-finite number")
+    elif isinstance(value, dict):
+        for key, child in value.items():
+            _validate_finite_numbers(errors, child, f"{path}.{key}")
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            _validate_finite_numbers(errors, child, f"{path}[{index}]")
+
+
+def _compare_present_values(errors: list[str], path: str, actual: Any, expected: Any) -> None:
+    if not isinstance(actual, dict) or not isinstance(expected, dict):
+        return
+    for key, value in actual.items():
+        if key not in expected:
+            continue
+        expected_value = expected[key]
+        child_path = f"{path}.{key}"
+        if isinstance(value, dict) and isinstance(expected_value, dict):
+            _compare_present_values(errors, child_path, value, expected_value)
+        elif value != expected_value:
+            errors.append(f"{child_path} does not match episode-derived value")
+
+
 def _expect_equal(errors: list[str], name: str, actual: Any, expected: Any) -> None:
     if actual != expected:
         errors.append(f"{name} must be {expected!r}, got {actual!r}")

gm_bench/publication.py

@@ -17,7 +17,7 @@
 
 
 def canonical_sha256(payload: dict[str, Any]) -> str:
-    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode()
     return hashlib.sha256(encoded).hexdigest()
 
 

gm_bench/runner.py

@@ -166,6 +166,9 @@ def run_decision_point(
     timed into harness latency, usages are collected, and the decision is
     marked failed if ANY round's actions fail.
     """
+    # Interaction rounds are one decision window.  Negotiation walk-aways must
+    # survive query follow-ups and reopen only at the next decision point.
+    league.begin_decision_window()
     tier = _observation_tier_for_agent(agent, config)
     action_results: list[dict[str, Any]] | None = None
     last_results: list[dict[str, Any]] = []

gm_bench/simulator.py

@@ -199,9 +199,6 @@ def _available_actions(self, phase: str) -> list[str]:
         return actions
 
     def apply_actions(self, actions: list[dict[str, Any]], phase: str) -> list[ActionResult]:
-        # Walk-aways last one decision window: each call to apply_actions is one
-        # window, so counterparties who broke off talks come back next window.
-        self.window_walkaways = {}
         self._action_results = []
         if not isinstance(actions, list):
             self._record({}, phase, False, "agent response must be a list of actions")
@@ -217,6 +214,10 @@ def apply_actions(self, actions: list[dict[str, Any]], phase: str) -> list[Actio
             self._dispatch_action(action_type, action, phase)
         return self._action_results
 
+    def begin_decision_window(self) -> None:
+        """Reset negotiation walk-aways once before an agent decision."""
+        self.window_walkaways = {}
+
     def _dispatch_action(self, action_type: str, action: dict[str, Any], phase: str) -> ActionResult:
         if action_type == "noop":
             return self._record(action, phase, True, "no-op", penalize=False)
@@ -488,7 +489,12 @@ def _inspect_player(self, action: dict[str, Any], phase: str) -> ActionResult:
 
     def _list_free_agents(self, action: dict[str, Any], phase: str) -> ActionResult:
         position = action.get("position")
-        min_overall = float(action.get("min_overall", 0.0))
+        try:
+            min_overall = float(action.get("min_overall", 0.0))
+        except (TypeError, ValueError):
+            return self._record(action, phase, False, "min_overall must be a finite number", penalize=False)
+        if not math.isfinite(min_overall):
+            return self._record(action, phase, False, "min_overall must be a finite number", penalize=False)
         limit = min(24, max(1, int(action.get("limit", 12))))
         players = [self.players[pid] for pid in self.free_agents]
         if position in {"F", "D", "G"}:
@@ -510,9 +516,13 @@ def _memo(self, action: dict[str, Any], phase: str) -> None:
         self._record({**action, "text": memo}, phase, True, "memo saved")
 
     def _sign_free_agent(self, action: dict[str, Any], phase: str) -> None:
-        player_id = int(action.get("player_id", -1))
-        years = int(action.get("years", 1))
-        salary = float(action.get("salary", 0.0))
+        try:
+            player_id = int(action.get("player_id", -1))
+            years = int(action.get("years", 1))
+            salary = float(action.get("salary", 0.0))
+        except (TypeError, ValueError):
+            self._record(action, phase, False, "signing fields must be valid numbers")
+            return
         if player_id not in self.free_agents or player_id not in self.players:
             self._record(action, phase, False, "player is not an available free agent")
             return
@@ -521,7 +531,7 @@ def _sign_free_agent(self, action: dict[str, Any], phase: str) -> None:
             return
         # A non-positive salary is not a negotiable bid — it is a malformed
         # action, so it must not ride the penalty-free rejected-offer path.
-        if salary <= 0:
+        if not math.isfinite(salary) or salary <= 0:
             self._record(action, phase, False, "salary must be a positive amount")
             return
         player = self.players[player_id]

scripts/analyze_publication_panel.py

@@ -309,12 +309,18 @@ def _registered_lane_issues(
     return issues
 
 
-def analyze(registry: Mapping[str, Any], payloads: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+def analyze(
+    registry: Mapping[str, Any],
+    payloads: Sequence[Mapping[str, Any]],
+    *,
+    raw_payloads: Sequence[Mapping[str, Any]] = (),
+) -> dict[str, Any]:
     """Analyze valid artifacts matching the registry and report missing/rejected rows."""
     specs, config_errors = _registry_specs(registry)
     specs_by_identity = {(spec["provider"], spec["model"]): spec for spec in specs}
     candidates: dict[str, list[dict[str, Any]]] = {}
     rejected: list[dict[str, Any]] = []
+    raw_by_hash = {canonical_sha256(dict(raw)): raw for raw in raw_payloads}
 
     for index, payload in enumerate(payloads):
         artifact_sha256 = canonical_sha256(dict(payload))
@@ -334,6 +340,8 @@ def analyze(registry: Mapping[str, Any], payloads: Sequence[Mapping[str, Any]])
             not isinstance(raw_artifact_sha256, str) or re.fullmatch(r"[0-9a-f]{64}", raw_artifact_sha256) is None
         ):
             reasons.append("publication.raw_artifact_sha256 must be a 64-character lowercase hex digest")
+        elif raw_artifact_sha256 is not None and raw_by_hash and raw_artifact_sha256 not in raw_by_hash:
+            reasons.append("publication.raw_artifact_sha256 does not match any supplied raw artifact")
         try:
             per_seed = per_seed_pick_trader_lifts(
                 payload,
@@ -426,17 +434,23 @@ def main() -> int:
     parser.add_argument("artifacts", nargs="*", type=Path)
     parser.add_argument("--registry", type=Path, default=REGISTRY_PATH)
     parser.add_argument("--artifacts-dir", type=Path, default=DEFAULT_ARTIFACT_DIR)
+    parser.add_argument("--raw-artifacts-dir", type=Path, help="raw JSON evidence used to verify compact hash links")
     parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH)
     args = parser.parse_args()
 
     artifact_paths = args.artifacts or sorted(args.artifacts_dir.glob("*.json"))
     try:
         registry = _read_json(args.registry)
         payloads = [_read_json(path) for path in artifact_paths]
+        raw_payloads = (
+            [_read_json(path) for path in sorted(args.raw_artifacts_dir.glob("*.json"))]
+            if args.raw_artifacts_dir
+            else []
+        )
     except (OSError, ValueError, json.JSONDecodeError) as exc:
         sys.exit(f"analyze_publication_panel: {exc}")
 
-    result = analyze(registry, payloads)
+    result = analyze(registry, payloads, raw_payloads=raw_payloads)
     text = json.dumps(result, indent=2, sort_keys=True) + "\n"
     if result["eligible_model_count"] == 0:
         print(text, end="")

tests/test_action_validation.py

@@ -2,8 +2,12 @@
 
 from __future__ import annotations
 
+import json
 import random
 
+import pytest
+
+from examples.gm_agent_common import parse_actions
 from gm_bench.simulator import League
 
 
@@ -51,3 +55,26 @@ def test_trade_rejected_when_give_value_too_low() -> None:
     assert league.illegal_actions == illegal_before
     assert league.rejected_offers > rejected_before
     assert league.transactions[-1].accepted is False
+
+
+@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"])
+def test_non_finite_action_numbers_are_rejected_at_parser_and_simulator_boundaries(value: str) -> None:
+    with pytest.raises(ValueError, match="non-finite"):
+        parse_actions(f'{{"actions":[{{"type":"sign_free_agent","salary":{value}}}]}}')
+
+    league = League.new(seed=7)
+    player_id = league.free_agents[0]
+    league.apply_actions(
+        [{"type": "sign_free_agent", "player_id": player_id, "years": 1, "salary": json.loads(value)}], "preseason"
+    )
+    assert not league.transactions[-1].accepted
+    assert player_id in league.free_agents
+    assert league.user_team.roster.count(player_id) == 0
+
+
+@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
+def test_non_finite_query_threshold_is_rejected(value: float) -> None:
+    league = League.new(seed=7)
+    league.apply_actions([{"type": "list_free_agents", "min_overall": value}], "preseason")
+    assert not league.transactions[-1].accepted
+    assert "finite number" in league.transactions[-1].message

tests/test_negotiation.py

@@ -84,6 +84,7 @@ def test_free_agent_walks_away_after_repeated_lowballs() -> None:
     assert league.rejected_offers == REJECTED_OFFER_LIMIT_PER_WINDOW + 1
     assert league.illegal_actions == 0
     # Talks reopen at the next decision window.
+    league.begin_decision_window()
     league.apply_actions([{"type": "sign_free_agent", "player_id": player_id, "years": 1, "salary": ask}], "preseason")
     assert league.transactions[-1].accepted is True
 

tests/test_official_results.py

@@ -19,6 +19,7 @@
     redact_leaderboard_payload,
     validate_leaderboard_payload,
 )
+from gm_bench.publication import compact_result
 from scripts.analyze_output_budget import analyze
 from web.scripts.build_leaderboard import model_row
 
@@ -136,6 +137,16 @@ def test_public_leaderboard_policy_accepts_single_repeat_payload() -> None:
     assert report.ok
 
 
+def test_compact_artifact_rejects_tampered_episode_and_aggregate() -> None:
+    payload = compact_result(_official_payload(repeats=3))
+    payload["candidate"]["episodes"][0]["final_score"] = 9999.0
+    payload["candidate"]["summary"]["mean_score"] = 9999.0
+    payload["normalized"]["candidate_mean_score"] = 9999.0
+    report = validate_leaderboard_payload(payload, policy=SOTA_V2_POLICY)
+    assert not report.ok
+    assert any("episode-derived" in error for error in report.errors)
+
+
 def test_output_budget_policy_preserves_high_failure_cells() -> None:
     payload = _official_payload(repeats=3, failure_rate=1.0)
     report = validate_leaderboard_payload(payload, policy=OUTPUT_BUDGET_SWEEP_POLICY)

tests/test_pick_trading.py

@@ -30,6 +30,7 @@ def find_accepted_pick_sale(league: League) -> dict | None:
                 "receive_pick_seasons": [league.season + 1],
             }
             before = league.illegal_actions
+            league.begin_decision_window()  # each probe is its own window
             league.apply_actions([action], "preseason")
             if league.transactions[-1].accepted:
                 return action

tests/test_publication.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import math
 import subprocess
 from pathlib import Path
 
@@ -64,6 +65,11 @@ def test_compact_result_rejects_already_compacted_artifact() -> None:
         compact_result(compact)
 
 
+def test_canonical_hash_refuses_non_finite_publication_json() -> None:
+    with pytest.raises(ValueError, match="Out of range float values"):
+        canonical_sha256({"score": math.nan})
+
+
 def test_budget_analysis_refuses_empty_sweep(tmp_path: Path) -> None:
     output = tmp_path / "analysis.json"
     subprocess.run(["python3", "scripts/analyze_output_budget.py", "--output", str(output)], check=True)