diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 4045d457..25d3c393 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -653,6 +653,80 @@ def adversarial_probe_source_receipt_error( return "" +def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | Any: + """Bind trusted source receipts and locations into otherwise valid evidence. + + Models sometimes place the exact changed-file path and positive line in the + structured ``path``/``line`` fields but omit the duplicate ``path:line`` text + from ``evidence``. Restore only that redundant location after the existing + single receipt already matches the sealed current-head line. Missing, + duplicate, or mismatched receipts, unsafe paths, missing changed-file + evidence, and circular or unobserved claims remain unmodified and fail closed. + """ + if not isinstance(value, dict): + return value + validation = value.get("adversarial_validation") + if not isinstance(validation, dict): + return value + probes = validation.get("probes") + if not isinstance(probes, list): + return value + + changed_files = current_changed_files() + repaired_probes: list[Any] = [] + changed = False + for probe in probes: + if not isinstance(probe, dict): + repaired_probes.append(probe) + continue + path = probe.get("path") + line = probe.get("line") + evidence = probe.get("evidence") + if ( + not isinstance(path, str) + or path not in changed_files + or isinstance(line, bool) + or not isinstance(line, int) + or line <= 0 + or not isinstance(evidence, str) + or not evidence.strip() + or adversarial_probe_location_error(path, line) + ): + repaired_probes.append(probe) + continue + + receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) + if len(receipts) != 1 or adversarial_probe_source_receipt_error( + evidence, path, line + ): + repaired_probes.append(probe) + continue + repaired_evidence = evidence + rejection = adversarial_evidence_rejection_reason(repaired_evidence, path, line) + if rejection == "must cite the exact probe path and positive line": + repaired_evidence = f"{path}:{line} {repaired_evidence}" + elif rejection: + repaired_probes.append(probe) + continue + if adversarial_probe_source_receipt_error( + repaired_evidence, path, line + ) or adversarial_evidence_rejection_reason(repaired_evidence, path, line): + repaired_probes.append(probe) + continue + if repaired_evidence == evidence: + repaired_probes.append(probe) + else: + repaired_probes.append({**probe, "evidence": repaired_evidence}) + changed = True + + if not changed: + return value + return { + **value, + "adversarial_validation": {**validation, "probes": repaired_probes}, + } + + def adversarial_validation_error( value: Any, *, @@ -1267,6 +1341,7 @@ def reject(reason: str) -> None: return reject("APPROVE cannot contain findings") if result == "REQUEST_CHANGES" and not findings: return reject("REQUEST_CHANGES requires at least one finding") + value = repair_adversarial_probe_evidence_bindings(value) adversarial_error = adversarial_validation_error( value.get("adversarial_validation"), result=result, diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..ad78fd73 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -14,6 +14,8 @@ import pytest +from scripts.ci import opencode_review_normalize_output as normalizer + ROOT = Path(__file__).resolve().parents[1] RUNNER = ROOT / "scripts" / "ci" / "run_opencode_review_model_pool.sh" @@ -29,6 +31,16 @@ } +@pytest.fixture(autouse=True) +def clear_normalizer_artifact_caches(): + """Keep trusted artifact caches isolated from later review-gate tests.""" + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + yield + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + + def bash_command() -> str: """Return a Bash executable that can run repository shell scripts locally.""" if os.name == "nt": @@ -87,6 +99,170 @@ def seal_artifacts( return hashlib.sha256(manifest.read_bytes()).hexdigest() +def prepare_probe_binding_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, int, str]: + """Create one sealed changed source line for normalizer binding tests.""" + runner_temp = tmp_path / "binding-runner-temp" + source_root = tmp_path / "binding-source" + source_path = source_root / "scripts" / "ci" / "example.py" + runner_temp.mkdir() + source_path.parent.mkdir(parents=True) + source_path.write_text("return False\n", encoding="utf-8") + changed_files = runner_temp / "opencode-changed-files.txt" + changed_files.write_text("scripts/ci/example.py\n", encoding="utf-8") + manifest_digest = seal_artifacts( + runner_temp, + head_sha="binding-head", + run_id="binding-run", + run_attempt="1", + paths=(changed_files,), + ) + monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) + monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) + monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) + monkeypatch.setenv("OPENCODE_ARTIFACT_MANIFEST_SHA256", manifest_digest) + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + digest = hashlib.sha256(b"return False").hexdigest() + return "scripts/ci/example.py", 1, f"source-line-sha256={digest}" + + +def test_normalizer_binds_only_a_verified_structured_probe_location( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A matching receipt may restore only redundant path:line evidence text.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + evidence = ( + f"Regression command rejected malformed input with exit code 1; {receipt}" + ) + value = { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": evidence, + } + ] + } + } + + repaired = normalizer.repair_adversarial_probe_evidence_bindings(value) + + assert repaired is not value + assert repaired["adversarial_validation"]["probes"][0]["evidence"] == ( + f"{path}:{line} {evidence}" + ) + + +def test_normalizer_probe_binding_repair_remains_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Malformed, untrusted, circular, and already-bound probes are not rewritten.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + assert normalizer.repair_adversarial_probe_evidence_bindings(None) is None + + missing_validation: dict[str, object] = {} + assert ( + normalizer.repair_adversarial_probe_evidence_bindings(missing_validation) + is missing_validation + ) + malformed = {"adversarial_validation": {"probes": "not-a-list"}} + assert normalizer.repair_adversarial_probe_evidence_bindings(malformed) is malformed + + invalid_values = [ + { + "adversarial_validation": { + "probes": [ + "not-an-object", + {"path": path, "line": 0, "evidence": receipt}, + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + "source-line-sha256=" + "0" * 64 + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt} {receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/not-changed.py", + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": receipt, + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": f"Source inspection properly handles all cases; {receipt}", + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + f"Regression command at {path}:{line} rejected malformed input " + f"with exit code 1; {receipt}" + ), + } + ] + } + }, + ] + for invalid in invalid_values: + assert normalizer.repair_adversarial_probe_evidence_bindings(invalid) is invalid + + def skip_if_windows_bash_is_unresponsive(command: str) -> None: """Skip with a visible reason when local Git Bash cannot start on Windows.""" if os.name != "nt":