From 33a3fcd11e2456c549fcf8648f79819f9dbbde39 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 15 Jul 2026 00:45:02 +0900 Subject: [PATCH 1/5] fix(review): repair verified adversarial line bindings --- .../ci/opencode_review_normalize_output.py | 64 +++++++++++++++++++ scripts/ci/run_opencode_review_model_pool.sh | 3 +- tests/test_opencode_model_pool_runner.py | 8 +++ .../test_opencode_review_normalize_output.py | 56 ++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 4045d457..c116bccd 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -653,6 +653,69 @@ def adversarial_probe_source_receipt_error( return "" +def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | Any: + """Bind a verified structured probe location into otherwise valid evidence. + + Models sometimes place the exact changed-file path and positive line in the + structured ``path``/``line`` fields and copy the correct trusted source-line + receipt, but omit the duplicate ``path:line`` text from ``evidence``. This + repair is deliberately narrower than the validator: it only prefixes that + structured location after the receipt matches the current-head source bytes + and the resulting evidence satisfies every independent-proof and observed- + result check. Invalid digests, unsafe paths, missing changed-file evidence, + and circular or unobserved claims remain unmodified and fail closed. + """ + if not isinstance(value, dict): + return value + validation = value.get("adversarial_validation") + if not isinstance(validation, dict): + return value + probes = validation.get("probes") + if not isinstance(probes, list): + return value + + changed_files = current_changed_files() + repaired_probes: list[Any] = [] + changed = False + for probe in probes: + if not isinstance(probe, dict): + repaired_probes.append(probe) + continue + path = probe.get("path") + line = probe.get("line") + evidence = probe.get("evidence") + if ( + not isinstance(path, str) + or path not in changed_files + or isinstance(line, bool) + or not isinstance(line, int) + or line <= 0 + or not isinstance(evidence, str) + or not evidence.strip() + or adversarial_probe_location_error(path, line) + or adversarial_probe_source_receipt_error(evidence, path, line) + ): + repaired_probes.append(probe) + continue + rejection = adversarial_evidence_rejection_reason(evidence, path, line) + if rejection != "must cite the exact probe path and positive line": + repaired_probes.append(probe) + continue + repaired_evidence = f"{path}:{line} {evidence.strip()}" + if adversarial_evidence_rejection_reason(repaired_evidence, path, line): + repaired_probes.append(probe) + continue + repaired_probes.append({**probe, "evidence": repaired_evidence}) + changed = True + + if not changed: + return value + return { + **value, + "adversarial_validation": {**validation, "probes": repaired_probes}, + } + + def adversarial_validation_error( value: Any, *, @@ -1267,6 +1330,7 @@ def reject(reason: str) -> None: return reject("APPROVE cannot contain findings") if result == "REQUEST_CHANGES" and not findings: return reject("REQUEST_CHANGES requires at least one finding") + value = repair_adversarial_probe_evidence_bindings(value) adversarial_error = adversarial_validation_error( value.get("adversarial_validation"), result=result, diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 85e122ab..9e70f073 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -186,9 +186,10 @@ write_prompt() { printf 'Do not request changes solely because your tool call, MCP call, or full-file read was not executed. Treat that as a review source limitation unless current-head evidence explicitly reports a materialization failure; any such finding must be tied to that evidence, not a generic model-exhaustion message. REQUEST_CHANGES findings must cite a positive source/evidence line; never use line 0.\n' printf 'Always return a final control block instead of a progress summary. Return only the final review body.\n\n' printf 'Adversarial evidence must state a concrete observed pass, failure, rejection, return value, exit code, or trace outcome and exactly one source-line-sha256=<64 lowercase hex> digest computed from the cited current-head line bytes without its line ending; generic source-inspection or coverage-verification claims are invalid.\n' + printf 'Every adversarial_validation.probes[].evidence string must literally include the same path:positive-line declared by that probe before the observed result and receipt; the separate path and line JSON fields do not replace this evidence citation.\n' printf 'Required control block shape:\n' printf '```json\n' - printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"executed command or source-backed trace, observed outcome, and source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" + printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"exact/current-head/changed-file:1 source trace or executed command observed a concrete outcome; source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" printf '```\n' if [ -s "$evidence_excerpt_file" ]; then printf '\nCurrent-head evidence packet:\n\n' diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..bd52751f 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -704,3 +704,11 @@ def test_deepseek_prompt_still_inlines_bounded_evidence_excerpt(tmp_path: Path) prompt = prompt_capture.read_text(encoding="utf-8") assert evidence_excerpt in prompt assert "Evidence excerpt omitted" not in prompt + assert ( + "Every adversarial_validation.probes[].evidence string must literally include " + "the same path:positive-line" in prompt + ) + assert ( + '"evidence":"exact/current-head/changed-file:1 source trace or executed command ' + "observed a concrete outcome; source-line-sha256=" in prompt + ) diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index 590fb3e5..d8755ea3 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -400,6 +400,62 @@ def test_adversarial_validation_rejects_unbound_or_mismatched_source_receipts( assert "does not match the cited current-head line" in mismatch_reasons[-1] +def test_valid_control_repairs_only_verified_structured_probe_location_binding( + tmp_path, monkeypatch +): + """A verified receipt may restore missing path:line prose without weakening proof.""" + require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") + validation = adversarial_validation() + unbound_probes = [] + for probe in validation["probes"]: + unbound = dict(probe) + unbound["evidence"] = re.sub( + rf"Focused source trace at {re.escape(probe['path'])}:{probe['line']} and ", + "Regression command ", + probe["evidence"], + ) + unbound_probes.append(unbound) + + normalized = norm.valid_control( + control( + adversarial_validation={ + **validation, + "probes": unbound_probes, + } + ), + expected_head_sha="head", + expected_run_id="run", + expected_run_attempt="attempt", + ) + + assert normalized is not None + repaired_probes = normalized["adversarial_validation"]["probes"] + assert repaired_probes[0]["evidence"].startswith("scripts/ci/example.py:7 ") + assert repaired_probes[1]["evidence"].startswith("scripts/ci/example.py:8 ") + + +def test_adversarial_probe_binding_repair_fails_closed_for_malformed_or_unobserved_input(): + """Malformed shapes and receipt-only prose remain unchanged and unpublishable.""" + assert norm.repair_adversarial_probe_evidence_bindings(None) is None + + malformed = {"adversarial_validation": {"probes": "not-an-array"}} + assert norm.repair_adversarial_probe_evidence_bindings(malformed) is malformed + + receipt_only = { + "adversarial_validation": { + "probes": [ + "not-an-object", + { + "path": "scripts/ci/example.py", + "line": 7, + "evidence": source_line_receipt("line 7"), + }, + ] + } + } + assert norm.repair_adversarial_probe_evidence_bindings(receipt_only) is receipt_only + + def test_adversarial_source_receipt_helpers_fail_closed_at_trust_boundaries( tmp_path, monkeypatch ): From 02c3370ee3f533d5df1b91ee96a17fb535e7f58f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 15 Jul 2026 17:54:53 +0900 Subject: [PATCH 2/5] fix(review): derive trusted probe receipts --- .../ci/opencode_review_normalize_output.py | 48 +++++++++++++------ tests/test_opencode_existing_approval_gate.py | 2 + .../test_opencode_review_normalize_output.py | 45 ++++++++++++----- 3 files changed, 67 insertions(+), 28 deletions(-) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index c116bccd..3a18997b 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -654,16 +654,15 @@ def adversarial_probe_source_receipt_error( def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | Any: - """Bind a verified structured probe location into otherwise valid evidence. + """Bind trusted source receipts and locations into otherwise valid evidence. Models sometimes place the exact changed-file path and positive line in the - structured ``path``/``line`` fields and copy the correct trusted source-line - receipt, but omit the duplicate ``path:line`` text from ``evidence``. This - repair is deliberately narrower than the validator: it only prefixes that - structured location after the receipt matches the current-head source bytes - and the resulting evidence satisfies every independent-proof and observed- - result check. Invalid digests, unsafe paths, missing changed-file evidence, - and circular or unobserved claims remain unmodified and fail closed. + structured ``path``/``line`` fields but either miscompute the receipt or omit + the duplicate ``path:line`` text from ``evidence``. The receipt is a trusted + binding rather than independent proof, so derive it from the sealed current- + head tree only after validating the changed path and positive line. Missing + or duplicate receipts, unsafe paths, missing changed-file evidence, and + circular or unobserved claims remain unmodified and fail closed. """ if not isinstance(value, dict): return value @@ -693,20 +692,39 @@ def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | A or not isinstance(evidence, str) or not evidence.strip() or adversarial_probe_location_error(path, line) - or adversarial_probe_source_receipt_error(evidence, path, line) ): repaired_probes.append(probe) continue - rejection = adversarial_evidence_rejection_reason(evidence, path, line) - if rejection != "must cite the exact probe path and positive line": + + receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) + expected_digest = adversarial_probe_source_line_digest(path, line) + if len(receipts) != 1 or expected_digest is None: repaired_probes.append(probe) continue - repaired_evidence = f"{path}:{line} {evidence.strip()}" - if adversarial_evidence_rejection_reason(repaired_evidence, path, line): + repaired_evidence = SOURCE_LINE_RECEIPT_RE.sub( + f"source-line-sha256={expected_digest}", + evidence.strip(), + count=1, + ) + rejection = adversarial_evidence_rejection_reason( + repaired_evidence, path, line + ) + if rejection == "must cite the exact probe path and positive line": + repaired_evidence = f"{path}:{line} {repaired_evidence}" + elif rejection: repaired_probes.append(probe) continue - repaired_probes.append({**probe, "evidence": repaired_evidence}) - changed = True + if ( + adversarial_probe_source_receipt_error(repaired_evidence, path, line) + or adversarial_evidence_rejection_reason(repaired_evidence, path, line) + ): + repaired_probes.append(probe) + continue + if repaired_evidence == evidence: + repaired_probes.append(probe) + else: + repaired_probes.append({**probe, "evidence": repaired_evidence}) + changed = True if not changed: return value diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 7602b2a1..9f6aea4f 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -68,6 +68,8 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): "OPENCODE_ARTIFACT_MANIFEST_SHA256", hashlib.sha256(manifest.read_bytes()).hexdigest(), ) + opencode_review_normalize_output.current_changed_files.cache_clear() + opencode_review_normalize_output.trusted_execution_receipts.cache_clear() def valid_body(head: str = HEAD) -> str: diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index d8755ea3..a2a71fbc 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -346,10 +346,10 @@ def test_adversarial_validation_canonicalizes_case_and_whitespace_for_duplicates assert "duplicates an earlier probe" in reasons[-1] -def test_adversarial_validation_rejects_unbound_or_mismatched_source_receipts( +def test_adversarial_validation_rejects_missing_and_repairs_mismatched_receipts( tmp_path, monkeypatch ): - """Lexical proof prose cannot authorize approval without exact line binding.""" + """Missing receipts fail closed while trusted source bytes repair one mismatch.""" require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") validation = adversarial_validation() @@ -386,18 +386,16 @@ def test_adversarial_validation_rejects_unbound_or_mismatched_source_receipts( "probes": [mismatched, validation["probes"][1]], } ) - mismatch_reasons: list[str] = [] - assert ( - norm.valid_control( - invalid, - expected_head_sha="head", - expected_run_id="run", - expected_run_attempt="attempt", - rejection_reasons=mismatch_reasons, - ) - is None + normalized = norm.valid_control( + invalid, + expected_head_sha="head", + expected_run_id="run", + expected_run_attempt="attempt", ) - assert "does not match the cited current-head line" in mismatch_reasons[-1] + assert normalized is not None + repaired_evidence = normalized["adversarial_validation"]["probes"][0]["evidence"] + assert "source-line-sha256=" + "0" * 64 not in repaired_evidence + assert source_line_receipt("line 7") in repaired_evidence def test_valid_control_repairs_only_verified_structured_probe_location_binding( @@ -455,6 +453,27 @@ def test_adversarial_probe_binding_repair_fails_closed_for_malformed_or_unobserv } assert norm.repair_adversarial_probe_evidence_bindings(receipt_only) is receipt_only + duplicate_receipt = { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/example.py", + "line": 7, + "evidence": ( + "Focused source trace at scripts/ci/example.py:7 confirmed " + "the guard; " + f"{source_line_receipt('line 7')} " + f"{source_line_receipt('line 7')}" + ), + } + ] + } + } + assert ( + norm.repair_adversarial_probe_evidence_bindings(duplicate_receipt) + is duplicate_receipt + ) + def test_adversarial_source_receipt_helpers_fail_closed_at_trust_boundaries( tmp_path, monkeypatch From 4ff50eca109bb1410275e024c92fef348f278147 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 15 Jul 2026 18:19:46 +0900 Subject: [PATCH 3/5] refactor(review): isolate trusted receipt repair --- scripts/ci/opencode_review_normalize_output.py | 11 ++++------- scripts/ci/run_opencode_review_model_pool.sh | 3 +-- tests/test_opencode_existing_approval_gate.py | 3 +-- tests/test_opencode_model_pool_runner.py | 8 -------- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 3a18997b..07437735 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -706,18 +706,15 @@ def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | A evidence.strip(), count=1, ) - rejection = adversarial_evidence_rejection_reason( - repaired_evidence, path, line - ) + rejection = adversarial_evidence_rejection_reason(repaired_evidence, path, line) if rejection == "must cite the exact probe path and positive line": repaired_evidence = f"{path}:{line} {repaired_evidence}" elif rejection: repaired_probes.append(probe) continue - if ( - adversarial_probe_source_receipt_error(repaired_evidence, path, line) - or adversarial_evidence_rejection_reason(repaired_evidence, path, line) - ): + if adversarial_probe_source_receipt_error( + repaired_evidence, path, line + ) or adversarial_evidence_rejection_reason(repaired_evidence, path, line): repaired_probes.append(probe) continue if repaired_evidence == evidence: diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 9e70f073..85e122ab 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -186,10 +186,9 @@ write_prompt() { printf 'Do not request changes solely because your tool call, MCP call, or full-file read was not executed. Treat that as a review source limitation unless current-head evidence explicitly reports a materialization failure; any such finding must be tied to that evidence, not a generic model-exhaustion message. REQUEST_CHANGES findings must cite a positive source/evidence line; never use line 0.\n' printf 'Always return a final control block instead of a progress summary. Return only the final review body.\n\n' printf 'Adversarial evidence must state a concrete observed pass, failure, rejection, return value, exit code, or trace outcome and exactly one source-line-sha256=<64 lowercase hex> digest computed from the cited current-head line bytes without its line ending; generic source-inspection or coverage-verification claims are invalid.\n' - printf 'Every adversarial_validation.probes[].evidence string must literally include the same path:positive-line declared by that probe before the observed result and receipt; the separate path and line JSON fields do not replace this evidence citation.\n' printf 'Required control block shape:\n' printf '```json\n' - printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"exact/current-head/changed-file:1 source trace or executed command observed a concrete outcome; source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" + printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"executed command or source-backed trace, observed outcome, and source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" printf '```\n' if [ -s "$evidence_excerpt_file" ]; then printf '\nCurrent-head evidence packet:\n\n' diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 9f6aea4f..5f676b94 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -85,8 +85,7 @@ def valid_body(head: str = HEAD) -> str: "evidence": ( f"Source trace at .github/workflows/opencode-review.yml:{line} " "confirmed the gate rejected the forged evidence. " - "source-line-sha256=" - + hashlib.sha256(source_line).hexdigest() + "source-line-sha256=" + hashlib.sha256(source_line).hexdigest() ), "outcome": "falsified", } diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index bd52751f..71e96add 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -704,11 +704,3 @@ def test_deepseek_prompt_still_inlines_bounded_evidence_excerpt(tmp_path: Path) prompt = prompt_capture.read_text(encoding="utf-8") assert evidence_excerpt in prompt assert "Evidence excerpt omitted" not in prompt - assert ( - "Every adversarial_validation.probes[].evidence string must literally include " - "the same path:positive-line" in prompt - ) - assert ( - '"evidence":"exact/current-head/changed-file:1 source trace or executed command ' - "observed a concrete outcome; source-line-sha256=" in prompt - ) From c28eb67360c7ac7add2e7bfbe5826e2c314f95da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 15 Jul 2026 18:29:16 +0900 Subject: [PATCH 4/5] test(review): restore normalizer coverage gate --- .../test_opencode_review_normalize_output.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index a2a71fbc..e3490d2a 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -453,6 +453,43 @@ def test_adversarial_probe_binding_repair_fails_closed_for_malformed_or_unobserv } assert norm.repair_adversarial_probe_evidence_bindings(receipt_only) is receipt_only + unchanged_location = { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/not-changed.py", + "line": 7, + "evidence": ( + "Focused source trace at scripts/ci/not-changed.py:7 confirmed " + f"the guard; {source_line_receipt('line 7')}" + ), + } + ] + } + } + assert ( + norm.repair_adversarial_probe_evidence_bindings(unchanged_location) + is unchanged_location + ) + + unproved_binding = { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/example.py", + "line": 7, + "evidence": ( + f"scripts/ci/example.py:7 {source_line_receipt('line 7')}" + ), + } + ] + } + } + assert ( + norm.repair_adversarial_probe_evidence_bindings(unproved_binding) + is unproved_binding + ) + duplicate_receipt = { "adversarial_validation": { "probes": [ @@ -498,6 +535,31 @@ def test_adversarial_source_receipt_helpers_fail_closed_at_trust_boundaries( norm.adversarial_probe_source_receipt_error(receipt, "missing.py", 1) == "source-line receipt could not be verified from the trusted tree" ) + assert ( + norm.adversarial_probe_source_receipt_error(receipt, "one_line.py", 1) + == "source-line-sha256 receipt does not match the cited current-head line" + ) + + +def test_adversarial_validation_still_rejects_unrepaired_receipt_mismatch(): + """The terminal validator rejects a bad digest when no trusted repair occurred.""" + validation = adversarial_validation() + mismatched = dict(validation["probes"][0]) + mismatched["evidence"] = re.sub( + r"source-line-sha256=[0-9a-f]{64}", + "source-line-sha256=" + "0" * 64, + mismatched["evidence"], + ) + validation["probes"][0] = mismatched + + assert norm.adversarial_validation_error( + validation, + result="APPROVE", + findings=[], + ) == ( + "adversarial probe 1 evidence source-line-sha256 receipt does not match " + "the cited current-head line" + ) def test_adversarial_request_changes_requires_confirmed_probe_at_finding( From 8deac92bf9ce2c8ed03a05fe2d66c301cd2a27b3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 15 Jul 2026 19:23:41 +0900 Subject: [PATCH 5/5] fix(review): bind only verified probe locations --- .../ci/opencode_review_normalize_output.py | 22 +-- tests/test_opencode_existing_approval_gate.py | 5 +- tests/test_opencode_model_pool_runner.py | 176 ++++++++++++++++++ .../test_opencode_review_normalize_output.py | 161 ++-------------- 4 files changed, 199 insertions(+), 165 deletions(-) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 07437735..25d3c393 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -657,12 +657,11 @@ def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | A """Bind trusted source receipts and locations into otherwise valid evidence. Models sometimes place the exact changed-file path and positive line in the - structured ``path``/``line`` fields but either miscompute the receipt or omit - the duplicate ``path:line`` text from ``evidence``. The receipt is a trusted - binding rather than independent proof, so derive it from the sealed current- - head tree only after validating the changed path and positive line. Missing - or duplicate receipts, unsafe paths, missing changed-file evidence, and - circular or unobserved claims remain unmodified and fail closed. + structured ``path``/``line`` fields but omit the duplicate ``path:line`` text + from ``evidence``. Restore only that redundant location after the existing + single receipt already matches the sealed current-head line. Missing, + duplicate, or mismatched receipts, unsafe paths, missing changed-file + evidence, and circular or unobserved claims remain unmodified and fail closed. """ if not isinstance(value, dict): return value @@ -697,15 +696,12 @@ def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | A continue receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) - expected_digest = adversarial_probe_source_line_digest(path, line) - if len(receipts) != 1 or expected_digest is None: + if len(receipts) != 1 or adversarial_probe_source_receipt_error( + evidence, path, line + ): repaired_probes.append(probe) continue - repaired_evidence = SOURCE_LINE_RECEIPT_RE.sub( - f"source-line-sha256={expected_digest}", - evidence.strip(), - count=1, - ) + repaired_evidence = evidence rejection = adversarial_evidence_rejection_reason(repaired_evidence, path, line) if rejection == "must cite the exact probe path and positive line": repaired_evidence = f"{path}:{line} {repaired_evidence}" diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 5f676b94..7602b2a1 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -68,8 +68,6 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): "OPENCODE_ARTIFACT_MANIFEST_SHA256", hashlib.sha256(manifest.read_bytes()).hexdigest(), ) - opencode_review_normalize_output.current_changed_files.cache_clear() - opencode_review_normalize_output.trusted_execution_receipts.cache_clear() def valid_body(head: str = HEAD) -> str: @@ -85,7 +83,8 @@ def valid_body(head: str = HEAD) -> str: "evidence": ( f"Source trace at .github/workflows/opencode-review.yml:{line} " "confirmed the gate rejected the forged evidence. " - "source-line-sha256=" + hashlib.sha256(source_line).hexdigest() + "source-line-sha256=" + + hashlib.sha256(source_line).hexdigest() ), "outcome": "falsified", } diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..ad78fd73 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -14,6 +14,8 @@ import pytest +from scripts.ci import opencode_review_normalize_output as normalizer + ROOT = Path(__file__).resolve().parents[1] RUNNER = ROOT / "scripts" / "ci" / "run_opencode_review_model_pool.sh" @@ -29,6 +31,16 @@ } +@pytest.fixture(autouse=True) +def clear_normalizer_artifact_caches(): + """Keep trusted artifact caches isolated from later review-gate tests.""" + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + yield + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + + def bash_command() -> str: """Return a Bash executable that can run repository shell scripts locally.""" if os.name == "nt": @@ -87,6 +99,170 @@ def seal_artifacts( return hashlib.sha256(manifest.read_bytes()).hexdigest() +def prepare_probe_binding_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, int, str]: + """Create one sealed changed source line for normalizer binding tests.""" + runner_temp = tmp_path / "binding-runner-temp" + source_root = tmp_path / "binding-source" + source_path = source_root / "scripts" / "ci" / "example.py" + runner_temp.mkdir() + source_path.parent.mkdir(parents=True) + source_path.write_text("return False\n", encoding="utf-8") + changed_files = runner_temp / "opencode-changed-files.txt" + changed_files.write_text("scripts/ci/example.py\n", encoding="utf-8") + manifest_digest = seal_artifacts( + runner_temp, + head_sha="binding-head", + run_id="binding-run", + run_attempt="1", + paths=(changed_files,), + ) + monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) + monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) + monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) + monkeypatch.setenv("OPENCODE_ARTIFACT_MANIFEST_SHA256", manifest_digest) + normalizer.current_changed_files.cache_clear() + normalizer.trusted_execution_receipts.cache_clear() + digest = hashlib.sha256(b"return False").hexdigest() + return "scripts/ci/example.py", 1, f"source-line-sha256={digest}" + + +def test_normalizer_binds_only_a_verified_structured_probe_location( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A matching receipt may restore only redundant path:line evidence text.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + evidence = ( + f"Regression command rejected malformed input with exit code 1; {receipt}" + ) + value = { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": evidence, + } + ] + } + } + + repaired = normalizer.repair_adversarial_probe_evidence_bindings(value) + + assert repaired is not value + assert repaired["adversarial_validation"]["probes"][0]["evidence"] == ( + f"{path}:{line} {evidence}" + ) + + +def test_normalizer_probe_binding_repair_remains_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Malformed, untrusted, circular, and already-bound probes are not rewritten.""" + path, line, receipt = prepare_probe_binding_artifacts(tmp_path, monkeypatch) + assert normalizer.repair_adversarial_probe_evidence_bindings(None) is None + + missing_validation: dict[str, object] = {} + assert ( + normalizer.repair_adversarial_probe_evidence_bindings(missing_validation) + is missing_validation + ) + malformed = {"adversarial_validation": {"probes": "not-a-list"}} + assert normalizer.repair_adversarial_probe_evidence_bindings(malformed) is malformed + + invalid_values = [ + { + "adversarial_validation": { + "probes": [ + "not-an-object", + {"path": path, "line": 0, "evidence": receipt}, + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + "source-line-sha256=" + "0" * 64 + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt} {receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": "scripts/ci/not-changed.py", + "line": line, + "evidence": ( + "Regression command rejected malformed input with exit code 1; " + f"{receipt}" + ), + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": receipt, + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": f"Source inspection properly handles all cases; {receipt}", + } + ] + } + }, + { + "adversarial_validation": { + "probes": [ + { + "path": path, + "line": line, + "evidence": ( + f"Regression command at {path}:{line} rejected malformed input " + f"with exit code 1; {receipt}" + ), + } + ] + } + }, + ] + for invalid in invalid_values: + assert normalizer.repair_adversarial_probe_evidence_bindings(invalid) is invalid + + def skip_if_windows_bash_is_unresponsive(command: str) -> None: """Skip with a visible reason when local Git Bash cannot start on Windows.""" if os.name != "nt": diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index e3490d2a..590fb3e5 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -346,10 +346,10 @@ def test_adversarial_validation_canonicalizes_case_and_whitespace_for_duplicates assert "duplicates an earlier probe" in reasons[-1] -def test_adversarial_validation_rejects_missing_and_repairs_mismatched_receipts( +def test_adversarial_validation_rejects_unbound_or_mismatched_source_receipts( tmp_path, monkeypatch ): - """Missing receipts fail closed while trusted source bytes repair one mismatch.""" + """Lexical proof prose cannot authorize approval without exact line binding.""" require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") validation = adversarial_validation() @@ -386,130 +386,18 @@ def test_adversarial_validation_rejects_missing_and_repairs_mismatched_receipts( "probes": [mismatched, validation["probes"][1]], } ) - normalized = norm.valid_control( - invalid, - expected_head_sha="head", - expected_run_id="run", - expected_run_attempt="attempt", - ) - assert normalized is not None - repaired_evidence = normalized["adversarial_validation"]["probes"][0]["evidence"] - assert "source-line-sha256=" + "0" * 64 not in repaired_evidence - assert source_line_receipt("line 7") in repaired_evidence - - -def test_valid_control_repairs_only_verified_structured_probe_location_binding( - tmp_path, monkeypatch -): - """A verified receipt may restore missing path:line prose without weakening proof.""" - require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") - validation = adversarial_validation() - unbound_probes = [] - for probe in validation["probes"]: - unbound = dict(probe) - unbound["evidence"] = re.sub( - rf"Focused source trace at {re.escape(probe['path'])}:{probe['line']} and ", - "Regression command ", - probe["evidence"], - ) - unbound_probes.append(unbound) - - normalized = norm.valid_control( - control( - adversarial_validation={ - **validation, - "probes": unbound_probes, - } - ), - expected_head_sha="head", - expected_run_id="run", - expected_run_attempt="attempt", - ) - - assert normalized is not None - repaired_probes = normalized["adversarial_validation"]["probes"] - assert repaired_probes[0]["evidence"].startswith("scripts/ci/example.py:7 ") - assert repaired_probes[1]["evidence"].startswith("scripts/ci/example.py:8 ") - - -def test_adversarial_probe_binding_repair_fails_closed_for_malformed_or_unobserved_input(): - """Malformed shapes and receipt-only prose remain unchanged and unpublishable.""" - assert norm.repair_adversarial_probe_evidence_bindings(None) is None - - malformed = {"adversarial_validation": {"probes": "not-an-array"}} - assert norm.repair_adversarial_probe_evidence_bindings(malformed) is malformed - - receipt_only = { - "adversarial_validation": { - "probes": [ - "not-an-object", - { - "path": "scripts/ci/example.py", - "line": 7, - "evidence": source_line_receipt("line 7"), - }, - ] - } - } - assert norm.repair_adversarial_probe_evidence_bindings(receipt_only) is receipt_only - - unchanged_location = { - "adversarial_validation": { - "probes": [ - { - "path": "scripts/ci/not-changed.py", - "line": 7, - "evidence": ( - "Focused source trace at scripts/ci/not-changed.py:7 confirmed " - f"the guard; {source_line_receipt('line 7')}" - ), - } - ] - } - } + mismatch_reasons: list[str] = [] assert ( - norm.repair_adversarial_probe_evidence_bindings(unchanged_location) - is unchanged_location - ) - - unproved_binding = { - "adversarial_validation": { - "probes": [ - { - "path": "scripts/ci/example.py", - "line": 7, - "evidence": ( - f"scripts/ci/example.py:7 {source_line_receipt('line 7')}" - ), - } - ] - } - } - assert ( - norm.repair_adversarial_probe_evidence_bindings(unproved_binding) - is unproved_binding - ) - - duplicate_receipt = { - "adversarial_validation": { - "probes": [ - { - "path": "scripts/ci/example.py", - "line": 7, - "evidence": ( - "Focused source trace at scripts/ci/example.py:7 confirmed " - "the guard; " - f"{source_line_receipt('line 7')} " - f"{source_line_receipt('line 7')}" - ), - } - ] - } - } - assert ( - norm.repair_adversarial_probe_evidence_bindings(duplicate_receipt) - is duplicate_receipt + norm.valid_control( + invalid, + expected_head_sha="head", + expected_run_id="run", + expected_run_attempt="attempt", + rejection_reasons=mismatch_reasons, + ) + is None ) + assert "does not match the cited current-head line" in mismatch_reasons[-1] def test_adversarial_source_receipt_helpers_fail_closed_at_trust_boundaries( @@ -535,31 +423,6 @@ def test_adversarial_source_receipt_helpers_fail_closed_at_trust_boundaries( norm.adversarial_probe_source_receipt_error(receipt, "missing.py", 1) == "source-line receipt could not be verified from the trusted tree" ) - assert ( - norm.adversarial_probe_source_receipt_error(receipt, "one_line.py", 1) - == "source-line-sha256 receipt does not match the cited current-head line" - ) - - -def test_adversarial_validation_still_rejects_unrepaired_receipt_mismatch(): - """The terminal validator rejects a bad digest when no trusted repair occurred.""" - validation = adversarial_validation() - mismatched = dict(validation["probes"][0]) - mismatched["evidence"] = re.sub( - r"source-line-sha256=[0-9a-f]{64}", - "source-line-sha256=" + "0" * 64, - mismatched["evidence"], - ) - validation["probes"][0] = mismatched - - assert norm.adversarial_validation_error( - validation, - result="APPROVE", - findings=[], - ) == ( - "adversarial probe 1 evidence source-line-sha256 receipt does not match " - "the cited current-head line" - ) def test_adversarial_request_changes_requires_confirmed_probe_at_finding(