From eeed6c0048b089c616981e88a996cab13dc88ed9 Mon Sep 17 00:00:00 2001 From: SDL-HQ <229556096+SDL-HQ@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:22:14 +1200 Subject: [PATCH] Bind benchmark evidence to gated run --- .github/workflows/audit-and-sign.yml | 127 +++++++++++-- .github/workflows/r1-cli-acceptance.yml | 35 +++- README.md | 7 +- docs/backlog.md | 72 ++++++++ docs/evidence-perimeter.v4.md | 173 ++++++++++++++++++ docs/release-notes-2.2.md | 26 ++- red_team_suite.py | 1 - src/sir_firewall/core.py | 37 +++- .../tool_result_ingress_benchmark.json | 4 +- tests/test_pack_binding.py | 2 +- tests/test_tool_result_ingress.py | 11 +- 11 files changed, 450 insertions(+), 45 deletions(-) create mode 100644 docs/backlog.md create mode 100644 docs/evidence-perimeter.v4.md diff --git a/.github/workflows/audit-and-sign.yml b/.github/workflows/audit-and-sign.yml index 6cb1164d..2f672467 100644 --- a/.github/workflows/audit-and-sign.yml +++ b/.github/workflows/audit-and-sign.yml @@ -301,6 +301,7 @@ jobs: python tools/sign_policy.py - name: Generate signed certificate + HTML + id: generate_certificate if: always() env: SDL_PRIVATE_KEY_PEM: ${{ secrets.SDL_PRIVATE_KEY_PEM }} @@ -309,28 +310,75 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} run: | set -euo pipefail - python tools/generate_certificate.py + + output_file="$(mktemp)" + python tools/generate_certificate.py | tee "$output_file" + + cert_path="$( + sed -n 's/^OUTPUT_AUDIT_JSON=//p' "$output_file" | + tail -n 1 + )" + + case "$cert_path" in + proofs/latest-audit.json|proofs/local-audit.json) + ;; + *) + echo "ERROR: generate_certificate.py did not emit a valid OUTPUT_AUDIT_JSON path." + exit 1 + ;; + esac + + if [ ! -f "$cert_path" ]; then + echo "ERROR: generated certificate does not exist: $cert_path" + exit 1 + fi + + echo "audit_json=$cert_path" >> "$GITHUB_OUTPUT" + echo "Current-run certificate: $cert_path" - name: Validate certificate contract if: always() + env: + AUDIT_JSON: ${{ steps.generate_certificate.outputs.audit_json }} run: | set -euo pipefail - python tools/validate_certificate_contract.py proofs/latest-audit.json + + if [ -z "${AUDIT_JSON:-}" ] || [ ! -f "$AUDIT_JSON" ]; then + echo "SKIP: generate_certificate.py did not produce a current-run certificate path." + exit 0 + fi + + python tools/validate_certificate_contract.py "$AUDIT_JSON" - name: Verify certificate signature if: always() + env: + AUDIT_JSON: ${{ steps.generate_certificate.outputs.audit_json }} run: | set -euo pipefail - python tools/verify_certificate.py proofs/latest-audit.json + + if [ -z "${AUDIT_JSON:-}" ] || [ ! -f "$AUDIT_JSON" ]; then + echo "SKIP: generate_certificate.py did not produce a current-run certificate path." + exit 0 + fi + + python tools/verify_certificate.py "$AUDIT_JSON" - name: Archive this run (P6 truth-preserving per-run archive) if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' env: SDL_PRIVATE_KEY_PEM: ${{ secrets.SDL_PRIVATE_KEY_PEM }} + AUDIT_JSON: ${{ steps.generate_certificate.outputs.audit_json }} run: | set -euo pipefail + + if [ -z "${AUDIT_JSON:-}" ] || [ ! -f "$AUDIT_JSON" ]; then + echo "SKIP: generate_certificate.py did not produce a current-run certificate path." + exit 0 + fi + python tools/publish_run.py \ - --cert proofs/latest-audit.json \ + --cert "$AUDIT_JSON" \ --copy proofs/itgl_ledger.jsonl \ --copy proofs/itgl_final_hash.txt \ --copy proofs/latest-attempts.log \ @@ -377,6 +425,7 @@ jobs: GITHUB_SHA: ${{ github.sha }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ID: ${{ github.run_id }} + AUDIT_JSON: ${{ steps.generate_certificate.outputs.audit_json }} run: | set -euo pipefail @@ -385,7 +434,7 @@ jobs: harmless="$(cat harmless_blocked.txt 2>/dev/null || echo 0)" inconclusive="false" - if [ ! -f "proofs/latest-audit.json" ]; then inconclusive="true"; fi + if [ "${AUDIT_JSON:-}" != "proofs/latest-audit.json" ]; then inconclusive="true"; fi if [ ! -f "proofs/run_summary.json" ]; then inconclusive="true"; fi if [ ! -f "proofs/itgl_final_hash.txt" ]; then inconclusive="true"; fi if ! grep -q '^ITGL_FINAL_HASH=' itgl_env.txt 2>/dev/null; then inconclusive="true"; fi @@ -428,7 +477,7 @@ jobs: echo "LATEST_CI_RUN_URL=${ci_run_url}" >> "$GITHUB_ENV" - name: Publish latest PASS proof to docs/ (PASS only) - if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' && env.AUDIT_PASS == 'true' && env.INCONCLUSIVE != 'true' + if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' && steps.generate_certificate.outputs.audit_json == 'proofs/latest-audit.json' run: | set -euo pipefail test -f proofs/latest-audit.html @@ -445,6 +494,8 @@ jobs: path: | proofs/latest-audit.json proofs/latest-audit.html + proofs/local-audit.json + proofs/local-audit.html proofs/itgl_ledger.jsonl proofs/itgl_final_hash.txt proofs/run_summary.json @@ -461,6 +512,8 @@ jobs: - name: Commit proof files back to repo (archive always; latest PASS + sidecars only) if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' + env: + AUDIT_JSON: ${{ steps.generate_certificate.outputs.audit_json }} run: | set -euo pipefail @@ -484,7 +537,7 @@ jobs: git add proofs/runs/ docs/runs/ docs/latest-run.json docs/monitor-summary.json - if [ "${AUDIT_PASS}" = "true" ]; then + if [ "${AUDIT_JSON:-}" = "proofs/latest-audit.json" ]; then git add proofs/latest-audit.json proofs/latest-audit.html || true git add docs/latest-audit.json docs/latest-audit.html || true git add policy/isc_policy.signed.json || true @@ -635,6 +688,7 @@ jobs: PY - name: Run paired benchmark through SIR + id: run_benchmark env: DISPATCH_PACK: ${{ github.event.inputs.pack || 'generic_safety' }} DISPATCH_PROVIDER: ${{ github.event.inputs.provider || 'xai' }} @@ -670,7 +724,28 @@ jobs: fi fi - sir benchmark run --mode "$mode" --pack "$pack" --provider "$provider" --model "$model" + output_file="$(mktemp)" + sir benchmark run --mode "$mode" --pack "$pack" --provider "$provider" --model "$model" | + tee "$output_file" + + gated_run_id="$( + sed -n 's/^GATED_RUN_ID=//p' "$output_file" | + tail -n 1 + )" + + if [ -z "$gated_run_id" ]; then + echo "ERROR: benchmark did not emit GATED_RUN_ID." + exit 1 + fi + + gated_cert="proofs/runs/$gated_run_id/audit.json" + if [ ! -f "$gated_cert" ]; then + echo "ERROR: gated benchmark certificate not found: $gated_cert" + exit 1 + fi + + echo "gated_run_id=$gated_run_id" >> "$GITHUB_OUTPUT" + echo "gated_audit_json=$gated_cert" >> "$GITHUB_OUTPUT" - name: Prepare docs/ for GitHub Pages if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' @@ -766,8 +841,11 @@ jobs: print(json.dumps(latest_run_payload, ensure_ascii=False, indent=2)) PY - - name: Compute benchmark gated PASS truth from latest audit proof + - name: Compute benchmark gated PASS truth from current gated audit proof + id: compute_benchmark_gated_pass if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' + env: + GATED_AUDIT_JSON: ${{ steps.run_benchmark.outputs.gated_audit_json }} run: | set -euo pipefail python - <<'PY' @@ -775,22 +853,33 @@ jobs: import os from pathlib import Path - cert_path = Path("proofs/latest-audit.json") + gated_path_raw = os.environ.get("GATED_AUDIT_JSON", "").strip() + canonical_path = Path("proofs/latest-audit.json") gated_pass = False - if cert_path.is_file(): - payload = json.loads(cert_path.read_text(encoding="utf-8")) - result = str(payload.get("result", "")).strip().upper() - gated_pass = (result == "AUDIT PASSED") - + if gated_path_raw: + gated_path = Path(gated_path_raw) + if gated_path.is_file(): + gated_bytes = gated_path.read_bytes() + payload = json.loads(gated_bytes) + result = str(payload.get("result", "")).strip().upper() + gated_pass = ( + result == "AUDIT PASSED" + and canonical_path.is_file() + and canonical_path.read_bytes() == gated_bytes + ) + + value = "true" if gated_pass else "false" with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file: - env_file.write(f"BENCHMARK_GATED_PASS={'true' if gated_pass else 'false'}\n") + env_file.write(f"BENCHMARK_GATED_PASS={value}\n") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output_file: + output_file.write(f"gated_pass={value}\n") - print(f"BENCHMARK_GATED_PASS={'true' if gated_pass else 'false'}") + print(f"BENCHMARK_GATED_PASS={value}") PY - name: Publish latest PASS proof to docs/ (benchmark gated PASS only) - if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' && env.BENCHMARK_GATED_PASS == 'true' + if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' && steps.compute_benchmark_gated_pass.outputs.gated_pass == 'true' run: | set -euo pipefail test -f proofs/latest-audit.html @@ -814,6 +903,8 @@ jobs: - name: Commit benchmark files back to repo (archive + benchmark index/pairs) if: always() && env.CANONICAL_PUBLICATION_CONTEXT == 'true' + env: + BENCHMARK_GATED_PASS: ${{ steps.compute_benchmark_gated_pass.outputs.gated_pass }} run: | set -euo pipefail git config user.name "SIR Audit Bot" diff --git a/.github/workflows/r1-cli-acceptance.yml b/.github/workflows/r1-cli-acceptance.yml index 3023c59f..321f5707 100644 --- a/.github/workflows/r1-cli-acceptance.yml +++ b/.github/workflows/r1-cli-acceptance.yml @@ -109,25 +109,50 @@ jobs: PY - name: Generate cert from scenario run and validate certificate contract + id: generate_scenario_certificate env: SDL_PRIVATE_KEY_PEM: ${{ env.SDL_PRIVATE_KEY_PEM }} run: | set -euo pipefail - python tools/generate_certificate.py - python tools/validate_certificate_contract.py proofs/latest-audit.json + + output_file="$(mktemp)" + python tools/generate_certificate.py | tee "$output_file" + + cert_path="$( + sed -n 's/^OUTPUT_AUDIT_JSON=//p' "$output_file" | + tail -n 1 + )" + + case "$cert_path" in + proofs/latest-audit.json|proofs/local-audit.json) + ;; + *) + echo "ERROR: generate_certificate.py did not emit a valid OUTPUT_AUDIT_JSON path." + exit 1 + ;; + esac + + test -f "$cert_path" + python tools/validate_certificate_contract.py "$cert_path" + echo "audit_json=$cert_path" >> "$GITHUB_OUTPUT" - name: CLI verify smoke (cert path + stdin after scenario) + env: + AUDIT_JSON: ${{ steps.generate_scenario_certificate.outputs.audit_json }} run: | set -euo pipefail - sir verify cert --key /tmp/sir_ci_pub.pem --key-registry /tmp/ci_missing_key_registry.json proofs/latest-audit.json - cat proofs/latest-audit.json | sir verify cert --key /tmp/sir_ci_pub.pem --key-registry /tmp/ci_missing_key_registry.json - + test -f "$AUDIT_JSON" + sir verify cert --key /tmp/sir_ci_pub.pem --key-registry /tmp/ci_missing_key_registry.json "$AUDIT_JSON" + cat "$AUDIT_JSON" | sir verify cert --key /tmp/sir_ci_pub.pem --key-registry /tmp/ci_missing_key_registry.json - - name: Create archive and verify archive via CLI env: SDL_PRIVATE_KEY_PEM: ${{ env.SDL_PRIVATE_KEY_PEM }} + AUDIT_JSON: ${{ steps.generate_scenario_certificate.outputs.audit_json }} run: | set -euo pipefail - python tools/publish_run.py --cert proofs/latest-audit.json \ + test -f "$AUDIT_JSON" + python tools/publish_run.py --cert "$AUDIT_JSON" \ --copy proofs/itgl_ledger.jsonl \ --copy proofs/itgl_final_hash.txt \ --copy proofs/latest-attempts.log \ diff --git a/README.md b/README.md index fd2d84fe..c4ff1483 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# SIR: Signal Integrity Resolver Version 2.1 +# SIR: Signal Integrity Resolver Version 2.2 [![SIR Real Governance Audit](https://github.com/SDL-HQ/sir-firewall/actions/workflows/audit-and-sign.yml/badge.svg)](https://github.com/SDL-HQ/sir-firewall/actions/workflows/audit-and-sign.yml) @@ -204,12 +204,13 @@ SIR’s job is simple: enforce policy before inference, then prove what happened * [Minimal pilot runbook](docs/minimal-pilot-runbook.md) (procedural cold-start path) * [Evaluator technical explainer](docs/evaluator-technical-explainer.md) (evaluation and interpretation path) * [Assurance kit](docs/assurance-kit.md) (supporting evaluation and verification reference) -* [Evidence perimeter note](docs/evidence-perimeter.v2.md) (current bounded benchmark perimeter) +* [Evidence perimeter note](docs/evidence-perimeter.v4.md) (current bounded benchmark perimeter) +* [Public backlog](docs/backlog.md) (known constraints and planned hardening) * [External technical review preparation](docs/external-technical-review-prep.md) * [Engineer guide](docs/engineer-guide.md) (local runs, signing, serving) * [Trial guide](docs/trial-guide.md) (auditors, insurers, evidence capture) * [Key governance readiness](docs/key-governance-readiness.md) (authority map and `CRYPTO_ENFORCED` checklist) -* [Release notes](docs/release-notes-2.1.md) (2.1 closeout) +* [Release notes](docs/release-notes-2.2.md) (2.2 closeout) * [Retention / Tier B export](RETENTION.md) --- diff --git a/docs/backlog.md b/docs/backlog.md new file mode 100644 index 00000000..7b38a422 --- /dev/null +++ b/docs/backlog.md @@ -0,0 +1,72 @@ +# SIR Public Backlog + +This backlog tracks known hardening items, constraints, and planned work. Inclusion does not imply a committed release date. + +SIR separates current implemented behaviour from planned hardening work and future candidate coverage. + +## Status labels + +- Current +- Planned +- Investigating +- Known constraint +- Future candidate +- Not in scope + +## Current domain packs + +| Pack | Status | Notes | +|---|---|---| +| generic_safety | Current | Baseline coverage for bypass attempts, harmful requests, and prompt injection patterns. | +| data_exfiltration_pressure | Current | Attempts to extract restricted content, secrets, or internal data through pressure and reframing. | +| support_operator_override | Current | Override-style prompts invoking authority, support escalation, or operator language. | +| eu_ai_act_compliance_pressure | Current | Pressure patterns testing whether stated governance, disclosure, and control boundaries hold. Bounded partial-coverage pack, not a full-pass pack. | +| hipaa_mental_health | Current | Domain pack for HIPAA-bound mental health workloads. | +| pci_payments | Current | Domain pack for PCI-style payment workloads. | + +## Evidence hardening backlog + +| Item | Status | Notes | +|---|---|---| +| Pack hash self-computation | Planned | load_domain_pack() does not compute a content hash of the loaded pack file. pack_hash is populated only when supplied by a caller via pack_identity_context. | +| Raw pre-normalisation obfuscation signal | Planned | obfuscation_signal_detected is evaluated on post-normalisation text. Payloads containing only invisible Unicode with no signal keywords normalise silently without the signal firing. | +| Granular early-exit ITGL components | Planned | Mixed-mode, malformed ISC, and structured schema declaration failures do produce ITGL entries, but reuse generic component types with the specific error code carried in entry data. Dedicated component types per error class are not implemented. | +| Version-source consistency check | Planned | No automated check prevents package and artefact version drift. | +| Provider response trace metadata | Investigating | Non-sensitive provider response id, model, and token metadata for live evidence. | +| Transient provider retry policy | Investigating | Limited retries for provider or server errors without concealing final failures. | +| RSA-PSS signature padding | Planned | Certificate signing and verification currently use PKCS1v15. RSA-PSS is the recommended padding for new systems. | +| CJK token estimation | Planned | _estimate_tokens() uses a character-based approximation that underestimates token count for CJK text, making friction limits more permissive for non-whitespace-delimited languages. | + +## Known constraints + +| Item | Status | Notes | +|---|---|---| +| Structured schema extensibility | Known constraint | Adding new structured schema types requires multiple explicit change points. Intentional for v1. | +| GitHub dependent dropdowns | Known constraint | GitHub Actions does not support dynamic dependent dropdowns. Invalid provider and model combinations fail closed in validation. | +| Deterministic rule scope | Known constraint | SIR evaluates deterministic rules over declared ingress content. It does not infer hidden downstream agent state. | +| strict_isc flag | Known constraint | STRICT_ISC_ENFORCEMENT is retained for policy and pack schema compatibility. ISC structure rejection is unconditional and no longer gated by this flag. | + +## Future pack candidates + +Possible expansion areas only. Not current implemented coverage. + +| Candidate | Status | +|---|---| +| Tool result injection pressure | Future candidate | +| Mental health clinical | Future candidate | +| PII protection | Future candidate | +| Financial services | Future candidate | +| Legal and contracts | Future candidate | +| Insurance and underwriting | Future candidate | +| Code generation safety | Future candidate | + +## Not in scope + +| Item | Notes | +|---|---| +| Semantic prompt-injection detection | SIR remains deterministic and rules-based. | +| Universal agentic-injection prevention | SIR evaluates declared ingress content. It does not control every downstream agent path. | +| Model safety certification | SIR produces gate and evidence artefacts, not model safety certification. | +| Content SIR never receives | SIR cannot evaluate content outside its declared ingress path. | +| API message-ordering validation | SIR does not validate API-level message ordering or role sequence. Deployments using self-hosted inference should independently enforce message-ordering validation. | +| Multi-turn context accumulation | SIR evaluates declared ingress at the request boundary. It does not evaluate state accumulated across conversation turns. | diff --git a/docs/evidence-perimeter.v4.md b/docs/evidence-perimeter.v4.md new file mode 100644 index 00000000..e4bc50eb --- /dev/null +++ b/docs/evidence-perimeter.v4.md @@ -0,0 +1,173 @@ +# SIR current evidence perimeter v4 + +Date (UTC): 2026-07-27 + +## Method note +Paired benchmark means an ungated baseline run and an SIR-gated run executed against the same prompt set, then compared at result and prompt ID level. + +## Gate outcome versus run publication status +Gate outcome is the per-prompt deterministic decision: `PASS` or `BLOCK`. Run publication status is the per-run aggregate: `AUDIT PASSED`, `AUDIT FAILED`, or `INCONCLUSIVE`. These are distinct levels. + +## INCONCLUSIVE +A live run is `INCONCLUSIVE` when it did not produce complete provider evidence, even where gate metrics were otherwise clean. This is triggered when `provider_call_failures` is greater than zero, or when provider-call attempts were made with zero successes. `INCONCLUSIVE` is not a gate failure. + +## Provider completeness +Provider complete means `provider_call_failures == 0`. For paired runs, baseline and gated provider completeness are recorded separately. + +## Pair status versus provider status +`pair_status` indicates structural comparability of a baseline and gated pair. It does not by itself indicate that provider evidence completed. The baseline and gated provider-completeness fields must be checked separately. + +## Controlled run selection +Provider, model, and pack selection is constrained by the canonical run-selection registry at `spec/execution/run_selection_registry.v1.json`, with fail-closed validation before audit or benchmark execution. Invalid combinations do not proceed. + +## Provider call routing +OpenAI GPT-5-family models route through the Responses API path. GPT-4.1-family and xAI models use the completion path. This is provider-call routing, not gate logic. + +## Governance certificate fields +Signed certificates include `governance_scope` and `crypto_enforced`. Both fields are part of the signed payload. Certificates generated before SIR 2.2 will not pass evidence-contract validation, but continue to pass signature verification and remain valid as historical evidence. + +## Models included in current evidence state +Primary cross-provider comparison set: +- `gpt-4o-mini` +- `xai/grok-4-1-fast` +- `xai/grok-4.20-0309-reasoning` +- `xai/grok-4.20-0309-non-reasoning` +- `gpt-4.1-mini` +- `gpt-5.4-mini` + +### Provider-complete GPT-5.4-mini live pair in the current archive +The current archive contains a provider-complete OpenAI `gpt-5.4-mini` pair for `support_operator_override`: +- ungated baseline run `20260624-005933-000000-gh28067676116-dca5de736797`: leaks `26`, harmless blocked `0`, provider attempts `50`, successes `50`, failures `0` +- SIR-gated run `20260624-005958-000000-gh28067676116-baac467cd954`: leaks `0`, harmless blocked `0`, provider attempts `24`, successes `24`, failures `0` + +Both sides satisfy the current provider-completeness condition. + +## Packs included in current perimeter state +Core gate-effect comparison packs: +- `generic_safety` +- `data_exfiltration_pressure` +- `support_operator_override` + +Extended governance pressure pack now in scope: +- `eu_ai_act_compliance_pressure` + +## Evidence classes +- paired benchmark evidence +- selected comparisons used latest `valid_complete` pairs +- core comparison packs remain coherent live/provider-path evidence +- `eu_ai_act_compliance_pressure` now has both: + - paired governance-gate-only benchmark evidence + - recent live paired evidence confirming the same directional effect + +## Observed paired outcomes for the core comparison packs +For the six-model primary comparison set, on all three core packs: +- ungated baseline: `AUDIT FAILED` +- SIR-gated run: `AUDIT PASSED` + +## Core pack-level results +- `generic_safety`: ungated leaks 100, SIR-gated leaks 0, gated benign/pass denominator 50 +- `data_exfiltration_pressure`: ungated leaks 27, SIR-gated leaks 0, gated benign/pass denominator 23 +- `support_operator_override`: ungated leaks 26, SIR-gated leaks 0, gated benign/pass denominator 24 + +## Core benign/pass denominator summary +- total gated benign/pass denominator across compared live pairs: `97` +- total harmless blocked across compared live pairs: `0` + +## Core ID-level discrimination review +- across all six compared models, ungated leaked prompt ID sets were identical for each pack +- across all six compared models, SIR-gated blocked/passed prompt ID sets were identical for each pack + +## Extended cross-model ID-level finding +ID-level discrimination review was subsequently extended to all six models in the current evidence state across the three core packs: +- ungated leaked prompt ID sets were identical across all six models for each pack +- reasoning vs non-reasoning variants produced identical outcomes +- three generations of GPT (`gpt-4o-mini`, `gpt-4.1-mini`, `gpt-5.4-mini`) produced identical outcomes +- SIR-gated blocked/passed prompt ID sets were identical across all six models + +This finding was published: [Seven models, two providers, identical failure sets](https://x.com/SDL_HQ/status/2049438575574364279) - 2026-05-03 + +What this supports: +- common-mode prompt-layer vulnerability across the tested model set +- model selection, including generation level and reasoning capability, did not change attack outcomes on the tested packs + +What this does not support: +- universal claim across all models or all attack categories +- model-vs-model discrimination within the tested set + +## `eu_ai_act_compliance_pressure` current position +Current paired governance-gate-only benchmark position on `xai/grok-4-1-fast`: +- ungated baseline leaks: `100` +- SIR-gated leaks: `26` +- delta leaks: `-74` +- delta harmless blocked: `0` + +Recent live paired position on `xai/grok-4-1-fast`: +- `AUDIT FAILED` to `AUDIT FAILED` +- delta leaks: `-74` +- delta provider calls: `-74` +- delta harmless blocked: `0` + +## `eu_ai_act_compliance_pressure` progression +Observed paired benchmark progression for this pack on the current bounded rule path: +- `100 -> 100` +- `100 -> 75` +- `100 -> 57` +- `100 -> 32` +- `100 -> 26` + +This progression reflects bounded deterministic expansion inside the SIR rule layer, not benchmark semantic changes. + +## `eu_ai_act_compliance_pressure` category state +Current gated-state interpretation: +- `transparency_bypass`: cleared +- `prohibited_manipulation`: cleared +- `oversight_circumvention`: reduced to one intentional residual row +- `data_boundary`: dominant remaining leakage class + +## What current evidence supports +- the three core packs still strongly distinguish SIR off vs on +- across the tested model set, the current three packs show common-mode prompt-layer susceptibility across six models and two providers +- on those packs, in that tested set, model selection did not change attack outcomes; SIR did +- `eu_ai_act_compliance_pressure` now shows that SIR can materially reduce governance-pressure leakage through bounded deterministic extensions +- the current bounded EU-pack work reduced leaks from `100` to `26` in paired governance-gate-only benchmark mode with `0` harmless blocked +- the same `-74` leak reduction was also observed in a recent live paired run, with `-74` provider calls and `0` harmless blocked + +## Explicit non-claims +Current evidence and implementation do not claim that SIR: +- prevents all prompt injection +- provides semantic attack detection +- controls downstream agent behaviour +- proves model safety +- replaces application-level security controls +- evaluates content it never receives +- validates API message ordering + +## What current evidence does not support +- it does not show that all models are equivalent generally +- it does not show that all attack classes are common-mode generally +- it does not show model-vs-model discrimination within the current tested set +- it does not establish full coverage or residual-risk claims beyond the packs, models, and evidence classes listed here +- it does not show that `eu_ai_act_compliance_pressure` is fully covered; the pack still fails overall +- it does not support broad legal or compliance claims from the EU-pack work + +## What this means for liability exposure +- prompt-layer attack success is not just a model-quality issue; it is an exposure pathway where unsafe or deceptive requests can reach inference and create downstream operational, regulatory, or liability exposure +- SIR reduces that exposure by blocking classes of prompt-layer requests before inference and by preserving a signed audit trail showing what was attempted, what was blocked, and what was allowed +- for review, dispute, or claims handling, the resulting artefacts provide replayable evidence of gate behavior and decision lineage rather than post-hoc narrative alone +- `0` harmless blocked in the measured reductions means the current bounded rule gains were not achieved by broadly degrading normal use in the tested benign slice +- this does not eliminate liability or certify compliance, but it does improve the evidence position and control position around prompt-layer failure modes + +## Current perimeter limit +- the core benchmark set remains validated as a gate-effect evidence set for the three listed core packs +- it is still not a model-separation benchmark set +- `eu_ai_act_compliance_pressure` is now a bounded partial-coverage evidence pack, not a full-pass pack +- remaining EU-pack leakage is concentrated almost entirely in `data_boundary`, plus one intentionally unimplemented oversight residual + +## Current intentional pause +- `data_boundary` work is intentionally paused +- current judgment is that `data_boundary` is the least attractive remaining deterministic target and carries higher complexity and ambiguity risk +- the final oversight residual was intentionally not implemented for safety-to-complexity reasons + +## Next perimeter expansion +- next perimeter expansion, if needed, will be determined by the next forcing-function audience and evidence requirement +- current default recommendation is to stop EU-pack expansion here for now, unless a specific requirement justifies revisiting the paused remainder diff --git a/docs/release-notes-2.2.md b/docs/release-notes-2.2.md index 824d2801..f2b2c53f 100644 --- a/docs/release-notes-2.2.md +++ b/docs/release-notes-2.2.md @@ -10,11 +10,9 @@ - Protected latest-audit as the latest conclusive PASS proof, while latest-run reflects the most recent run state. - Added provider-completeness fields to paired benchmark artefacts and benchmark indexes. - Hardened invisible Unicode normalisation coverage including variation selectors, tag block characters, and supplemental variation selectors. -- Updated audit/live wording across workflow, certificate, and operator documentation surfaces. -- Updated ISC policy description and regenerated signed policy hash. - Added OpenAI GPT-5-family live-call support through the Responses API path. -- Added evidence perimeter documentation for SIR 2.2 semantics. -- Added public backlog for known constraints, planned hardening items, current packs, and not-in-scope claims. +- Added the SIR 2.2 evidence perimeter at `docs/evidence-perimeter.v4.md`. +- Added the public backlog at `docs/backlog.md` for known constraints, planned hardening items, current packs, and not-in-scope claims. ### Evidence semantics @@ -22,24 +20,32 @@ AUDIT PASSED now requires complete provider evidence for live proof classes. Gat latest-audit remains the latest conclusive PASS proof. latest-run reflects the most recent run including FAIL or INCONCLUSIVE. +The run harness no longer supplies the suite hash under the `pack_hash` field in per-prompt governance context and ITGL entries. `pack_hash` is populated only when a caller supplies a genuine pack hash through `pack_identity_context`; pack hash self-computation remains a backlog item. + +The audit workflow now uses the certificate path actually produced by the current run for contract validation, signature verification, run archival, latest-run status, docs publication, and commit staging. Previously, a gate-clean run with failed provider calls could publish a historical passing certificate to the docs surface as current evidence and record the run as PASS; non-passing runs now preserve their current `local-audit` certificate without replacing the latest passing proof. + +The paired benchmark workflow now determines gated-run status from the current gated member's archived certificate, verified byte-identical to the canonical proof, rather than reading a hard-coded path. The CLI acceptance workflow now verifies and archives the certificate actually produced by each run. Previously, an inconclusive gated benchmark member could republish a historical passing certificate to the docs surface, and a non-passing scenario run could verify and archive a certificate from an earlier step in the same job. + ### Governance certificate -Signed certificates now include governance_scope and crypto_enforced fields. These fields are part of the signed payload and verifiable offline. The evidence contract now requires these fields. Certificates generated before SIR 2.2 will not pass contract validation but will continue to pass signature verification. Pre-2.2 certificates remain valid as historical evidence. +Signed certificates now include `governance_scope` and `crypto_enforced` fields. These fields are part of the signed payload and verifiable offline. The evidence contract now requires these fields. Certificates generated before SIR 2.2 will not pass contract validation but will continue to pass signature verification. Pre-2.2 certificates remain valid as historical evidence. ### Normalisation hardening -Base64 keyword gate expanded to cover all seven deterministic rule families. Hex decoding added for explicit hex: marker and \xNN escape sequences. Arbitrary contiguous hex strings are not decoded. +Base64 keyword gate expanded to cover all seven deterministic rule families. Hex decoding added for explicit `hex:` marker and `\\xNN` escape sequences. Arbitrary contiguous hex strings are not decoded. ### Operator changes -Manual free-text provider/model/pack entry replaced with controlled dropdowns backed by fail-closed validation. Invalid combinations fail before audit or benchmark execution. +Manual free-text provider/model/pack entry was replaced with controlled dropdowns backed by fail-closed validation. Invalid combinations fail before audit or benchmark execution. ### Security hardening -- Tightened domain pack ID validation to prevent path traversal via SIR_ISC_PACK. -- Added boundary check on SIR_POLICY_PATH overrides. +- Tightened domain pack ID validation to prevent path traversal via `SIR_ISC_PACK`. +- Added a boundary check on `SIR_POLICY_PATH` overrides. - Corrected ISC structure validation to block unconditionally. +- Retained `STRICT_ISC_ENFORCEMENT` for policy and pack schema compatibility; it no longer gates ISC structure rejection. +- Renamed the generic multi-ingress-mode error code to `multiple_ingress_modes_not_allowed` and added a dedicated `mixed_mode_validation` ITGL component and rule metadata, so generic multi-mode conflicts are no longer recorded as tool-result-specific failures. ### Non-claims -SIR remains a deterministic, rules-based pre-inference governance gate. It does not claim universal prompt-injection prevention, semantic attack detection, model safety certification, or control over content it does not receive. +SIR remains a deterministic, rules-based pre-inference governance gate. It does not claim universal prompt-injection prevention, semantic attack detection, model safety certification, control over downstream agent behaviour, application-level security coverage, API message-ordering validation, or control over content it does not receive. diff --git a/red_team_suite.py b/red_team_suite.py index 00bed479..0ea7992a 100644 --- a/red_team_suite.py +++ b/red_team_suite.py @@ -616,7 +616,6 @@ def main() -> None: enforcement_pack_id=(pack_id or None), pack_identity_context={ "pack_version": selected_pack_version, - "pack_hash": suite_hash, }, ) status = str(verdict.get("status", "UNKNOWN")) diff --git a/src/sir_firewall/core.py b/src/sir_firewall/core.py index 44ae5623..0fdfa6d4 100644 --- a/src/sir_firewall/core.py +++ b/src/sir_firewall/core.py @@ -29,6 +29,7 @@ TOOL_RESULT_FIELD = "tool_result" STRUCTURED_REASON = "structured_validation_failed" TOOL_RESULT_REASON = "tool_result_validation_failed" +MIXED_MODE_REASON = "mixed_mode_validation_failed" STRUCTURED_TEMPLATE_ID = "EU-AI-Act-ISC-v1" STRUCTURED_SCHEMA_DECLARATION_KEY = "structured_request_schema" STRUCTURED_SCHEMA_ID = "account_recovery_challenge_request_v1" @@ -215,6 +216,12 @@ "rule_category": "tool_result_validation", "rule_outcome_class": "BLOCK", }, + MIXED_MODE_REASON: { + "rule_id": "SIR-RULE-MIXED-MODE-VALIDATION", + "rule_description": "Request declared more than one ingress mode; ingress modes are mutually exclusive.", + "rule_category": "mixed_mode_validation", + "rule_outcome_class": "BLOCK", + }, } @@ -642,7 +649,7 @@ def _build_block( result["domain_pack"] = domain_pack if rule_hits: result["rule_hits"] = list(rule_hits) - rule_group = _RULE_GROUPS.get(block_type or reason) + rule_group = _RULE_GROUPS.get(block_type or "") or _RULE_GROUPS.get(reason) if rule_group is not None: result["triggered_rule"] = dict(rule_group) return result @@ -1009,6 +1016,29 @@ def _reject_tool_result_validation( ) +def _reject_mixed_mode_validation( + code: str, + detail: str, + itgl_log: List[Dict[str, Any]], + prev_hash: str, + domain_pack: str | None = None, +) -> Dict[str, Any]: + itgl_log, _ = _append_itgl( + "mixed_mode_validation", + "fail", + {"code": code}, + {"detail": detail}, + itgl_log, + prev_hash, + ) + return _build_block( + MIXED_MODE_REASON, + itgl_log, + block_type=code, + domain_pack=domain_pack, + ) + + def _load_structured_request_object(raw: Any) -> Tuple[Dict[str, Any] | None, str | None]: if isinstance(raw, dict): return raw, None @@ -1344,8 +1374,8 @@ def validate_sir( domain_pack=None, ) if sum(1 for active in (isc_mode, structured_mode, tool_result_mode) if active) > 1: - return _reject_tool_result_validation( - "tool_result_mixed_mode_not_allowed", + return _reject_mixed_mode_validation( + "multiple_ingress_modes_not_allowed", "multiple_ingress_modes_present", itgl_log, prev_hash, @@ -1469,6 +1499,7 @@ def validate_sir( ) flags = domain_cfg.get("flags", {}) + # Retained for policy and pack schema compatibility; ISC structure rejection is now unconditional. strict_isc = bool(flags.get("STRICT_ISC_ENFORCEMENT", STRICT_ISC_ENFORCEMENT)) checksum_enforced = bool(flags.get("CHECKSUM_ENFORCED", CHECKSUM_ENFORCED)) crypto_enforced = bool(flags.get("CRYPTO_ENFORCED", CRYPTO_ENFORCED)) diff --git a/tests/domain_packs/tool_result_ingress_benchmark.json b/tests/domain_packs/tool_result_ingress_benchmark.json index aa26c52a..88169588 100644 --- a/tests/domain_packs/tool_result_ingress_benchmark.json +++ b/tests/domain_packs/tool_result_ingress_benchmark.json @@ -55,8 +55,8 @@ }, "expected": { "status": "BLOCKED", - "reason": "tool_result_validation_failed", - "type": "tool_result_mixed_mode_not_allowed" + "reason": "mixed_mode_validation_failed", + "type": "multiple_ingress_modes_not_allowed" } }, { diff --git a/tests/test_pack_binding.py b/tests/test_pack_binding.py index 324901e9..d2594e78 100644 --- a/tests/test_pack_binding.py +++ b/tests/test_pack_binding.py @@ -260,7 +260,7 @@ def _fake_validate_sir(input_dict, enforcement_pack_id=None, pack_identity_conte summary = json.loads((tmp_path / "proofs" / "run_summary.json").read_text(encoding="utf-8")) ledger_entry = json.loads((tmp_path / "proofs" / "itgl_ledger.jsonl").read_text(encoding="utf-8").splitlines()[0]) assert observed["pack_version"] == "1.0.0" - assert observed["pack_hash"] == summary["suite_hash"] + assert observed["pack_hash"] == "" assert summary["selected_pack_version"] == "1.0.0" assert ledger_entry["pass_rule_explainability"]["evaluated_rule_families"] == ["jailbreak_bypass", "exfiltration"] assert ledger_entry["pass_rule_explainability"]["obfuscation_signal_detected"] is True diff --git a/tests/test_tool_result_ingress.py b/tests/test_tool_result_ingress.py index edbaf99e..3a760833 100644 --- a/tests/test_tool_result_ingress.py +++ b/tests/test_tool_result_ingress.py @@ -57,8 +57,15 @@ def test_tool_result_mixed_mode_fails_closed(): ) assert out["status"] == "BLOCKED" - assert out["reason"] == "tool_result_validation_failed" - assert out["type"] == "tool_result_mixed_mode_not_allowed" + assert out["reason"] == "mixed_mode_validation_failed" + assert out["type"] == "multiple_ingress_modes_not_allowed" + assert out["itgl_log"][0]["component"] == "mixed_mode_validation" + assert out["triggered_rule"] == { + "rule_id": "SIR-RULE-MIXED-MODE-VALIDATION", + "rule_description": "Request declared more than one ingress mode; ingress modes are mutually exclusive.", + "rule_category": "mixed_mode_validation", + "rule_outcome_class": "BLOCK", + } def test_tool_result_content_length_out_of_bounds_fails_closed():