From 37231dbbe80dae1b1d50ad337c0abd990c98f921 Mon Sep 17 00:00:00 2001 From: vlabs-bot Date: Fri, 10 Jul 2026 18:08:20 +0300 Subject: [PATCH 1/2] =?UTF-8?q?Codex=20security-audit=20remediation=20batc?= =?UTF-8?q?h=20(2026-07-10,=20banked=20=E2=80=94=20pending=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-banked by takeover from Codex (hit usage limit mid-run). Test-first fixes across trust boundaries, isolation, idempotency, supply-chain. NOT yet independently reviewed/verified in CI. See per-repo diff + quality report. --- .github/dependabot.yml | 9 + .github/requirements-ci.in | 3 + .github/requirements-ci.lock | 76 +++ .github/workflows/ci.yml | 15 +- PROVENANCE.md | 10 +- README.md | 4 +- examples/cards/clean_new_accept.json | 2 +- examples/cards/clean_old.json | 2 +- examples/cards/clean_reject_dcr.json | 4 +- examples/demo/README.md | 18 +- examples/demo/baseline.json | 2 +- examples/demo/candidate.json | 2 +- examples/demo/candidate_overfit.json | 2 +- examples/demo/expected_output.txt | 5 +- .../demo/sample_assurance_card_redacted.json | 30 +- examples/openai_rft_ipt_grader.py | 448 ++++++++++++++++-- examples/run_clean_gate_demo.md | 4 +- examples/sample_assurance_card.json | 39 +- tests/test_examples_run.py | 257 ++++++++++ 19 files changed, 824 insertions(+), 108 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/requirements-ci.in create mode 100644 .github/requirements-ci.lock diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8b002ed --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + commit-message: + prefix: "ci" diff --git a/.github/requirements-ci.in b/.github/requirements-ci.in new file mode 100644 index 0000000..21d4f53 --- /dev/null +++ b/.github/requirements-ci.in @@ -0,0 +1,3 @@ +pytest>=8 +ruff>=0.6 +vlabs-sdk==0.0.2 diff --git a/.github/requirements-ci.lock b/.github/requirements-ci.lock new file mode 100644 index 0000000..80af920 --- /dev/null +++ b/.github/requirements-ci.lock @@ -0,0 +1,76 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes --output-file=vlabs-examples/.github/requirements-ci.lock --strip-extras vlabs-examples/.github/requirements-ci.in +# +annotated-doc==0.0.4 \ + --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ + --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4 + # via typer +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via + # pytest + # rich +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via -r vlabs-examples/.github/requirements-ci.in +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +ruff==0.15.21 \ + --hash=sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075 \ + --hash=sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be \ + --hash=sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd \ + --hash=sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd \ + --hash=sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86 \ + --hash=sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341 \ + --hash=sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd \ + --hash=sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67 \ + --hash=sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41 \ + --hash=sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d \ + --hash=sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d \ + --hash=sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8 \ + --hash=sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233 \ + --hash=sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8 \ + --hash=sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500 \ + --hash=sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3 \ + --hash=sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646 \ + --hash=sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f + # via -r vlabs-examples/.github/requirements-ci.in +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer +typer==0.26.8 \ + --hash=sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c \ + --hash=sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e + # via vlabs-sdk +vlabs-sdk==0.0.2 \ + --hash=sha256:b6d5836ac5319c24f1141db07d6cf5b8ad93fb5c4181e088149da5ac7241a516 \ + --hash=sha256:f4e839591021b189b0442a1830aeab0c821d61f5f5bdd9e222e21868ae73bb2c + # via -r vlabs-examples/.github/requirements-ci.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 412ad28..d8753dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,16 +2,19 @@ name: CI on: push: { branches: [main] } pull_request: { branches: [main] } +permissions: + contents: read jobs: run-examples: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: { persist-credentials: false } + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: { python-version: "3.12" } - name: install public SDK surface (dummy provider only) run: | - pip install "vlabs-sdk @ git+https://github.com/verifiablelabs/vlabs-sdk@main" - pip install typer pytest - pip install --no-deps "vlabs-prm-eval @ git+https://github.com/verifiablelabs/vlabs-sdk@main#subdirectory=tools/vlabs-prm-eval" - - run: pytest tests -q + python -m pip install --disable-pip-version-check --no-deps pip==26.1.2 + python -m pip install --require-hashes -r .github/requirements-ci.lock + - run: python -m ruff check examples tests + - run: python -m pytest tests -q diff --git a/PROVENANCE.md b/PROVENANCE.md index 9e95e75..b4a0be3 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -1,8 +1,8 @@ # Provenance -Clean import (no history rewrite) from `verifiablelabs/verifiable-labs-envs` -at commit `762b44e8019af3e89c55bba0f88e9157bb50c5c3` (main). All example code authored fresh for this repo; depends on the public SDK surface only. Everything synthetic. +Clean import (no history rewrite) from the archived legacy workspace at commit +`762b44e8019af3e89c55bba0f88e9157bb50c5c3`. All example code was authored +fresh for this repository, depends only on the public SDK surface, and uses +synthetic inputs. -The source monorepo remains canonical until the split flips; this mirror is -refreshed by the migration tooling documented in -`verifiable-labs-private/docs/ops/github-repo-split-migration.md`. +`vlabs-examples` is now canonical for these public examples. diff --git a/README.md b/README.md index bd11efc..bc2b9e9 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ private engine internals; those are never published. ## Formal scope Selected mathematical properties behind the contamination-resistant promotion -gate are machine-verified in Lean 4. The implementation is property-tested -against the formal specification. +gate are machine-verified in Lean 4. A hand-maintained Python mirror has property +tests derived from selected definitions; no mechanized code-to-proof parity is claimed. ## License diff --git a/examples/cards/clean_new_accept.json b/examples/cards/clean_new_accept.json index 4f4ccfd..95538aa 100644 --- a/examples/cards/clean_new_accept.json +++ b/examples/cards/clean_new_accept.json @@ -2,7 +2,7 @@ "model_id": "qwen-2.5-1.5b-grpo-clean-accept", "vgs": 0.78, "contamination_risk": 0.08, - "clean_vgs": 0.62, + "clean_vgs": 0.6776, "public_score": 0.85, "hidden_score": 0.78, "ood_score": 0.72, diff --git a/examples/cards/clean_old.json b/examples/cards/clean_old.json index 9f47b99..5218c32 100644 --- a/examples/cards/clean_old.json +++ b/examples/cards/clean_old.json @@ -2,7 +2,7 @@ "model_id": "qwen-2.5-1.5b-base-clean", "vgs": 0.70, "contamination_risk": 0.10, - "clean_vgs": 0.50, + "clean_vgs": 0.58, "public_score": 0.80, "hidden_score": 0.70, "ood_score": 0.70, diff --git a/examples/cards/clean_reject_dcr.json b/examples/cards/clean_reject_dcr.json index 794f9c8..6fd3cf1 100644 --- a/examples/cards/clean_reject_dcr.json +++ b/examples/cards/clean_reject_dcr.json @@ -1,8 +1,8 @@ { "model_id": "qwen-2.5-1.5b-grpo-clean-reject-dcr", "vgs": 0.78, - "contamination_risk": 0.5, - "clean_vgs": 0.62, + "contamination_risk": 0.13, + "clean_vgs": 0.6136, "public_score": 0.85, "hidden_score": 0.78, "ood_score": 0.72, diff --git a/examples/demo/README.md b/examples/demo/README.md index e8c21a5..6587e7b 100644 --- a/examples/demo/README.md +++ b/examples/demo/README.md @@ -31,12 +31,13 @@ but is weaker on **hidden / OOD** transfer (`hidden_score` 0.68, `ood_score` 0.66). Two candidates then ask to be promoted: - **`candidate.json`** improves clean verified-generalization score - (`clean_vgs` 0.50 → 0.63) with no regression in contamination risk, hack + (`clean_vgs` 0.58 → 0.6868) with no regression in contamination risk, hack risk, calibration, OOD, cost, or latency → **ACCEPT**. - **`candidate_overfit.json`** has the **highest public score** of all (0.92) — but it got there by memorising the visible set: contamination risk jumps (0.10 → 0.34) and OOD transfer drops (0.66 → 0.62). The gate **REJECT**s it - and names exactly why: `ood_regressed`, `dcr_increased`. + and names exactly why: `clean_vgs_not_improved`, `ood_regressed`, and + `dcr_increased`. That contrast is the whole point: **a higher public score is not a promotion.** The clean gate only accepts a change that *truly generalizes*. @@ -48,7 +49,7 @@ The clean gate only accepts a change that *truly generalizes*. condition old new budget OK -------------------------------- ---------- ---------- ---------- -- -clean_vgs >= +tau 0.5000 0.5500 0.0100 OK +clean_vgs >= +tau 0.5800 0.3448 0.0100 !! hack_risk <= +eps_h 0.1000 0.1100 0.0200 OK calibration >= -eps_c 0.9000 0.9000 0.0200 OK ood_score >= -eps_o 0.6600 0.6200 0.0200 !! @@ -58,6 +59,7 @@ latency <= +eps_l 1.0000 1.0000 0.5000 OK regression flag False False False OK Reasons: + - clean_vgs_not_improved - ood_regressed - dcr_increased ``` @@ -84,8 +86,10 @@ a partial promotion when a change is a net improvement but carries a watch-item [`sample_assurance_card_redacted.json`](sample_assurance_card_redacted.json), which records a `LIMITED_ROLLOUT` decision with reason `ood_regressed`. -`clean_score = raw * (1 - dcr)` — contamination directly discounts the score, -which is why a memorised public win cannot buy a promotion. +`clean_vgs = raw_vgs * (1 - dcr) - beta * dcr` — contamination directly +discounts and penalizes the score, which is why a memorised public win cannot +buy a promotion. The CLI recomputes this value instead of trusting the derived +field supplied by a card. ## What this does NOT show @@ -109,5 +113,5 @@ which is why a memorised public win cannot buy a promotion. ## Formal scope Selected mathematical properties behind the contamination-resistant promotion -gate are machine-verified in Lean 4. The implementation is property-tested -against the formal specification. +gate are machine-verified in Lean 4. A hand-maintained Python mirror has property +tests derived from selected definitions; no mechanized code-to-proof parity is claimed. diff --git a/examples/demo/baseline.json b/examples/demo/baseline.json index e3cc7eb..73f33d1 100644 --- a/examples/demo/baseline.json +++ b/examples/demo/baseline.json @@ -2,7 +2,7 @@ "model_id": "refund-agent-baseline", "vgs": 0.70, "contamination_risk": 0.10, - "clean_vgs": 0.50, + "clean_vgs": 0.58, "public_score": 0.80, "hidden_score": 0.68, "ood_score": 0.66, diff --git a/examples/demo/candidate.json b/examples/demo/candidate.json index c57ca16..70a3000 100644 --- a/examples/demo/candidate.json +++ b/examples/demo/candidate.json @@ -2,7 +2,7 @@ "model_id": "refund-agent-candidate", "vgs": 0.79, "contamination_risk": 0.08, - "clean_vgs": 0.63, + "clean_vgs": 0.6868, "public_score": 0.86, "hidden_score": 0.79, "ood_score": 0.73, diff --git a/examples/demo/candidate_overfit.json b/examples/demo/candidate_overfit.json index 7297679..365840f 100644 --- a/examples/demo/candidate_overfit.json +++ b/examples/demo/candidate_overfit.json @@ -2,7 +2,7 @@ "model_id": "refund-agent-candidate-overfit", "vgs": 0.78, "contamination_risk": 0.34, - "clean_vgs": 0.55, + "clean_vgs": 0.3448, "public_score": 0.92, "hidden_score": 0.69, "ood_score": 0.62, diff --git a/examples/demo/expected_output.txt b/examples/demo/expected_output.txt index 835f68e..252d68d 100644 --- a/examples/demo/expected_output.txt +++ b/examples/demo/expected_output.txt @@ -6,7 +6,7 @@ condition old new budget OK -------------------------------- ---------- ---------- ---------- -- -clean_vgs >= +tau 0.5000 0.6300 0.0100 OK +clean_vgs >= +tau 0.5800 0.6868 0.0100 OK hack_risk <= +eps_h 0.1000 0.0800 0.0200 OK calibration >= -eps_c 0.9000 0.9200 0.0200 OK ood_score >= -eps_o 0.6600 0.7300 0.0200 OK @@ -21,7 +21,7 @@ regression flag False False False OK condition old new budget OK -------------------------------- ---------- ---------- ---------- -- -clean_vgs >= +tau 0.5000 0.5500 0.0100 OK +clean_vgs >= +tau 0.5800 0.3448 0.0100 !! hack_risk <= +eps_h 0.1000 0.1100 0.0200 OK calibration >= -eps_c 0.9000 0.9000 0.0200 OK ood_score >= -eps_o 0.6600 0.6200 0.0200 !! @@ -31,5 +31,6 @@ latency <= +eps_l 1.0000 1.0000 0.5000 OK regression flag False False False OK Reasons: + - clean_vgs_not_improved - ood_regressed - dcr_increased diff --git a/examples/demo/sample_assurance_card_redacted.json b/examples/demo/sample_assurance_card_redacted.json index 567a581..4114679 100644 --- a/examples/demo/sample_assurance_card_redacted.json +++ b/examples/demo/sample_assurance_card_redacted.json @@ -1,14 +1,28 @@ { - "_comment": "ILLUSTRATIVE EXAMPLE — synthetic numbers, fake IDs, fields redacted as they would be for a real customer. Not a real evaluation.", "card_version": "v2", "run_id": "run_redacted_xxxx", - "org": "REDACTED", - "agent": "REDACTED", - "scores": { "public": 0.77, "hidden": 0.70, "ood": 0.66, "adversarial": 0.58 }, - "contamination": { "dcr": 0.05 }, - "clean_vgs": { "baseline": 0.55, "candidate": 0.59 }, + "agent_id": "REDACTED", + "baseline_id": "REDACTED", + "candidate_id": "REDACTED", + "decision": "LIMITED_ROLLOUT", + "raw_vgs": 0.65, + "dcr": 0.05, + "clean_vgs": 0.5925, + "public_score": 0.77, + "hidden_score": 0.70, + "ood_score": 0.66, "generalization_gap": 0.07, - "gate": { "outcome": "LIMITED_ROLLOUT", "reasons": ["ood_regressed"] }, + "reject_reasons": ["ood_regressed"], "redaction_status": "redacted_public_safe", - "formal_claim": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. The implementation is property-tested against the formal specification." + "hf_public_safe": true, + "formal_claim": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. A hand-maintained Python mirror has property tests derived from selected definitions; no mechanized code-to-proof parity is claimed.", + "formal_scope": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. A hand-maintained Python mirror has property tests derived from selected definitions; no mechanized code-to-proof parity is claimed.", + "metadata": { + "illustrative": true, + "comment": "Synthetic numbers, fake IDs, and redacted fields; not a real evaluation.", + "org": "REDACTED", + "baseline_clean_vgs": 0.55, + "adversarial_score": 0.58, + "clean_vgs_beta": 0.5 + } } diff --git a/examples/openai_rft_ipt_grader.py b/examples/openai_rft_ipt_grader.py index 1e46281..af78465 100644 --- a/examples/openai_rft_ipt_grader.py +++ b/examples/openai_rft_ipt_grader.py @@ -1,9 +1,10 @@ """OpenAI RFT python-grader template — an Isomorphic Perturbation Testing (IPT) reward. Drop-in shape for an OpenAI Reinforcement Fine-Tuning `python` grader: a self-contained -`grade(sample, item) -> float` (stdlib only, deterministic, no network) that you paste into the -RFT grader sandbox. It returns a reward that a reward-hacker cannot farm by memorizing the -provided tests. +`grade(sample, item) -> float` (stdlib only and deterministic) that you adapt inside an RFT +grader. Candidate execution is deliberately delegated to a required external sandbox runner; +this file never executes rollout source on the grader host. It tests one concrete memorization +shortcut; it is not a proof that a reward cannot be gamed by other strategies. WHY. An RFT grader that rewards "passes the provided test(s)" teaches the policy to memorize those inputs (reward hacking). A genuine solution is invariant under a semantics-preserving @@ -13,9 +14,8 @@ LOW if it passes the provided tests but fails the fresh ones (a shortcut) 0.0 if it fails the provided tests -Our own reproduced experiment (GRPO, Llama-3.2-3B, 8 seeds/arm) found this isomorphic reward -cut the reward-hacking shortcut rate from 23.3% (public-test reward) to 4.1% (permutation -p=0.0009, Hedges g=0.97) — a controlled toy demonstration, not a frontier-scale claim. +This template intentionally publishes no efficacy number. Validate it on a preregistered, +independently reproduced evaluation for your own task distribution before making a claim. HONEST SCOPE. IPT is a public method (Helff et al., arXiv:2604.15149); we productize it. This reward only applies to tasks that ship a TRUSTED REFERENCE + an INPUT GENERATOR — i.e. where you @@ -24,105 +24,457 @@ from __future__ import annotations import json +import math from collections.abc import Callable, Sequence +_CANDIDATE_WORKER = r""" +import contextlib, json, math, os, sys +_MAX_DEPTH = 32 +_MAX_NODES = 50000 +_MAX_BLOB = 1000000 +def _take(_budget, _depth): + if _depth > _MAX_DEPTH: + raise ValueError("tagged value is too deep") + _budget[0] -= 1 + if _budget[0] < 0: + raise ValueError("tagged value has too many nodes") -def _passes(fn: Callable, cases: Sequence[tuple]) -> bool: - """True iff fn(*args) == expected for every (args, expected) case (fail-closed on any error).""" - for args, expected in cases: +def _encode_tagged(_value, _depth=0, _budget=None): + if _budget is None: + _budget = [_MAX_NODES] + _take(_budget, _depth) + if _value is None: + return {"t": "none"} + if type(_value) is bool: + return {"t": "bool", "v": _value} + if type(_value) is int and _value.bit_length() <= 4096: + return {"t": "int", "v": _value} + if type(_value) is float and math.isfinite(_value): + return {"t": "float", "v": _value} + if type(_value) is str and len(_value.encode("utf-8")) <= _MAX_BLOB: + return {"t": "str", "v": _value} + if type(_value) in (list, tuple): + return { + "t": "tuple" if type(_value) is tuple else "list", + "v": [_encode_tagged(_item, _depth + 1, _budget) for _item in _value], + } + if type(_value) is dict: + _pairs = [ + [_encode_tagged(_key, _depth + 1, _budget), _encode_tagged(_item, _depth + 1, _budget)] + for _key, _item in _value.items() + ] + _pairs.sort(key=lambda _pair: json.dumps(_pair[0], sort_keys=True, separators=(",", ":"))) + return {"t": "dict", "v": _pairs} + raise ValueError("unsupported protocol value") + +def _decode_tagged(_node, _depth=0, _budget=None): + if _budget is None: + _budget = [_MAX_NODES] + _take(_budget, _depth) + if not isinstance(_node, dict) or not isinstance(_node.get("t"), str): + raise ValueError("invalid tagged value") + _tag = _node["t"] + if _tag == "none" and set(_node) == {"t"}: + return None + if set(_node) != {"t", "v"}: + raise ValueError("invalid tagged value fields") + _value = _node["v"] + if _tag == "bool" and type(_value) is bool: + return _value + if _tag == "int" and type(_value) is int and _value.bit_length() <= 4096: + return _value + if _tag == "float" and type(_value) in (int, float) and math.isfinite(float(_value)): + return float(_value) + if _tag == "str" and type(_value) is str and len(_value.encode("utf-8")) <= _MAX_BLOB: + return _value + if _tag in {"list", "tuple"} and isinstance(_value, list): + _items = [_decode_tagged(_item, _depth + 1, _budget) for _item in _value] + return tuple(_items) if _tag == "tuple" else _items + if _tag == "dict" and isinstance(_value, list): + _result = {} + for _pair in _value: + if not isinstance(_pair, list) or len(_pair) != 2: + raise ValueError("invalid tagged dictionary entry") + _key = _decode_tagged(_pair[0], _depth + 1, _budget) + _item = _decode_tagged(_pair[1], _depth + 1, _budget) + try: + if _key in _result: + raise ValueError("duplicate tagged dictionary key") + _result[_key] = _item + except TypeError as _exc: + raise ValueError("unhashable tagged dictionary key") from _exc + return _result + raise ValueError("unsupported tagged value") + +_decode = json.loads +_encode = json.dumps +_request = _decode(sys.stdin.read()) +_protocol_stdout = sys.stdout +_inputs = _request.get("inputs") +_results = [] +_loaded = False +_ns = {} +try: + if set(_request) != {"protocol", "source", "entry_point", "inputs"}: + raise ValueError("invalid request schema") + if _request["protocol"] != "vlabs-rft-candidate-request/2" or not isinstance(_inputs, list): + raise ValueError("invalid request protocol") + _decoded_inputs = [_decode_tagged(_node) for _node in _inputs] + if any(type(_args) not in (list, tuple) for _args in _decoded_inputs): + raise ValueError("each input must be an argument list or tuple") + with open(os.devnull, "w") as _sink, contextlib.redirect_stdout(_sink), contextlib.redirect_stderr(_sink): + exec(compile(_request["source"], "", "exec"), _ns) + _fn = _ns.get(_request["entry_point"]) + if not callable(_fn): + raise ValueError("entry point is not callable") + _loaded = True + for _args in _decoded_inputs: + try: + _value = _fn(*_args) + _results.append({"ok": True, "value": _encode_tagged(_value)}) + except BaseException as _exc: + _results.append({"ok": False, "error": type(_exc).__name__[:128]}) +except BaseException as _exc: + _results = [{"ok": False, "error": type(_exc).__name__[:128]} for _ in (_inputs or [])] +_response = {"protocol": "vlabs-rft-candidate-response/2", "loaded": _loaded, "results": _results} +_protocol_stdout.write(_encode(_response, allow_nan=False, separators=(",", ":"))) +""" + + +_MAX_RUNNER_REQUEST_BYTES = 256_000 +_MAX_RUNNER_RESPONSE_BYTES = 256_000 +_CODEC_MAX_DEPTH = 32 +_CODEC_MAX_NODES = 50_000 +_CODEC_MAX_BLOB_BYTES = 1_000_000 + + +def _take_codec_budget(budget: list[int], depth: int) -> None: + if depth > _CODEC_MAX_DEPTH: + raise ValueError("tagged value is too deep") + budget[0] -= 1 + if budget[0] < 0: + raise ValueError("tagged value has too many nodes") + + +def _encode_tagged(value: object, depth: int = 0, budget: list[int] | None = None) -> dict: + if budget is None: + budget = [_CODEC_MAX_NODES] + _take_codec_budget(budget, depth) + if value is None: + return {"t": "none"} + if type(value) is bool: + return {"t": "bool", "v": value} + if type(value) is int and value.bit_length() <= 4096: + return {"t": "int", "v": value} + if type(value) is float and math.isfinite(value): + return {"t": "float", "v": value} + if type(value) is str and len(value.encode("utf-8")) <= _CODEC_MAX_BLOB_BYTES: + return {"t": "str", "v": value} + if type(value) in (list, tuple): + return { + "t": "tuple" if type(value) is tuple else "list", + "v": [_encode_tagged(item, depth + 1, budget) for item in value], + } + if type(value) is dict: + pairs = [ + [ + _encode_tagged(key, depth + 1, budget), + _encode_tagged(item, depth + 1, budget), + ] + for key, item in value.items() + ] + pairs.sort(key=lambda pair: json.dumps(pair[0], sort_keys=True, separators=(",", ":"))) + return {"t": "dict", "v": pairs} + raise ValueError(f"unsupported protocol value: {type(value).__name__}") + + +def _decode_tagged(node: object, depth: int = 0, budget: list[int] | None = None) -> object: + if budget is None: + budget = [_CODEC_MAX_NODES] + _take_codec_budget(budget, depth) + if not isinstance(node, dict) or not isinstance(node.get("t"), str): + raise ValueError("invalid tagged value") + tag = node["t"] + if tag == "none" and set(node) == {"t"}: + return None + if set(node) != {"t", "v"}: + raise ValueError("invalid tagged value fields") + value = node["v"] + if tag == "bool" and type(value) is bool: + return value + if tag == "int" and type(value) is int and value.bit_length() <= 4096: + return value + if tag == "float" and type(value) in (int, float) and math.isfinite(float(value)): + return float(value) + if tag == "str" and type(value) is str and len(value.encode("utf-8")) <= _CODEC_MAX_BLOB_BYTES: + return value + if tag in {"list", "tuple"} and isinstance(value, list): + items = [_decode_tagged(item, depth + 1, budget) for item in value] + return tuple(items) if tag == "tuple" else items + if tag == "dict" and isinstance(value, list): + result = {} + for pair in value: + if not isinstance(pair, list) or len(pair) != 2: + raise ValueError("invalid tagged dictionary entry") + key = _decode_tagged(pair[0], depth + 1, budget) + item = _decode_tagged(pair[1], depth + 1, budget) + try: + if key in result: + raise ValueError("duplicate tagged dictionary key") + result[key] = item + except TypeError as exc: + raise ValueError("unhashable tagged dictionary key") from exc + return result + raise ValueError("unsupported tagged value") + + +def _canonical_tagged(value: object) -> str: + return json.dumps(_encode_tagged(value), sort_keys=True, separators=(",", ":")) + + +def _candidate_outputs( + source: str, + entry_point: str, + inputs: Sequence[tuple], + sandbox_runner: Callable[[dict, float, int], dict | None] | None, + timeout_s: float = 5.0, +): + """Delegate candidate execution to a configured isolation boundary. + + The runner receives inputs and source but never expected/reference outputs. It must execute the + request in a disposable non-root microVM/container with no credentials or network, a minimal + read-only filesystem, and hard wall/CPU/memory/PID/file/output limits. The third argument is the + maximum response-byte contract. A subprocess on the grader host is not a valid runner. + """ + if not isinstance(source, str) or not source.strip(): + raise TypeError("candidate must be non-empty Python source, not an in-process callable") + if not callable(sandbox_runner): + raise RuntimeError( + "an external sandbox_runner is required; candidate source is never executed on the grader host" + ) + try: + request = { + "protocol": "vlabs-rft-candidate-request/2", + "source": source, + "entry_point": entry_point, + "inputs": [_encode_tagged(args) for args in inputs], + } + encoded_request = json.dumps(request, allow_nan=False, separators=(",", ":")).encode("utf-8") + except (TypeError, ValueError): + return None + if len(encoded_request) > _MAX_RUNNER_REQUEST_BYTES: + return None + responses = [] + encoded_responses = [] + for _ in range(2): + try: + response = sandbox_runner(request, timeout_s, _MAX_RUNNER_RESPONSE_BYTES) + encoded_response = json.dumps( + response, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except Exception: # noqa: BLE001 - a failed isolation service is an inconclusive candidate + return None + if len(encoded_response) > _MAX_RUNNER_RESPONSE_BYTES: + return None + responses.append(response) + encoded_responses.append(encoded_response) + if encoded_responses[0] != encoded_responses[1]: + return None + response = responses[0] + if type(response) is not dict or set(response) != {"protocol", "loaded", "results"}: + return None + if response["protocol"] != "vlabs-rft-candidate-response/2" or response["loaded"] is not True: + return None + results = response["results"] + if type(results) is not list or len(results) != len(inputs): + return None + decoded_results = [] + for result in results: + if type(result) is not dict or type(result.get("ok")) is not bool: + return None + if set(result) != ({"ok", "value"} if result["ok"] else {"ok", "error"}): + return None + if result["ok"]: + try: + decoded_results.append({"ok": True, "value": _decode_tagged(result["value"])}) + except (TypeError, ValueError): + return None + elif type(result["error"]) is str and len(result["error"].encode("utf-8")) <= 128: + decoded_results.append(result) + else: + return None + return decoded_results + + +def _passes( + source: str, + entry_point: str, + cases: Sequence[tuple], + sandbox_runner: Callable[[dict, float, int], dict | None], +) -> bool: + """Compare candidate outputs in this trusted process; the worker receives no expected values.""" + results = _candidate_outputs( + source, + entry_point, + [args for args, _expected in cases], + sandbox_runner, + ) + if results is None or not cases: + return False + for result, (_args, expected) in zip(results, cases): + if not result["ok"]: + return False try: - if fn(*args) != expected: - return False - except BaseException: # noqa: BLE001 # a candidate that crashes has not passed + candidate_json = _canonical_tagged(result["value"]) + expected_json = _canonical_tagged(expected) + except (TypeError, ValueError): + return False + if candidate_json != expected_json: return False return True +def _stable_reference_cases(reference: Callable, inputs: Sequence[tuple]) -> list[tuple] | None: + """Compute the entire trusted batch twice and reject exceptions or nondeterminism.""" + + try: + first_values = [reference(*args) for args in inputs] + second_values = [reference(*args) for args in inputs] + except Exception: # noqa: BLE001 - a broken trusted reference invalidates the grade + return None + cases = [] + for args, first, second in zip(inputs, first_values, second_values): + try: + first_json = _canonical_tagged(first) + second_json = _canonical_tagged(second) + except Exception: # noqa: BLE001 - a broken trusted reference invalidates the grade + return None + if first_json != second_json: + return None + cases.append((args, first)) + return cases + + def ipt_reward( - candidate: Callable, + candidate: str, reference: Callable, public_inputs: Sequence[tuple], input_generator: Callable[[int], tuple], *, + sandbox_runner: Callable[[dict, float, int], dict | None] | None = None, n_isomorphic: int = 16, shortcut_reward: float = 0.0, + candidate_entry_point: str = "solve", ) -> float: """Isomorphic-invariant reward in [0, 1] for one candidate against a trusted reference. `public_inputs` and each `input_generator(i)` return an ARGS TUPLE. Expected outputs are recomputed with `reference`, so the grader never trusts a stored answer key. Deterministic: same candidate + reference + generator + n -> same reward. + + Misconfiguration fails closed: ``n_isomorphic`` must be positive, + ``shortcut_reward`` must be in ``[0, 1]``, and a generator that cannot + produce the requested number of fresh cases yields ``0.0``. A configured + external ``sandbox_runner`` is mandatory; this function never executes + candidate source locally. """ - public_cases = [(args, reference(*args)) for args in public_inputs] - if not _passes(candidate, public_cases): + if isinstance(n_isomorphic, bool) or not isinstance(n_isomorphic, int) or n_isomorphic < 1: + raise ValueError("n_isomorphic must be a positive integer") + if ( + isinstance(shortcut_reward, bool) + or not isinstance(shortcut_reward, (int, float)) + or not math.isfinite(float(shortcut_reward)) + or not 0.0 <= float(shortcut_reward) <= 1.0 + ): + raise ValueError("shortcut_reward must be a finite number in [0, 1]") + if not callable(sandbox_runner): + raise RuntimeError( + "an external sandbox_runner is required; candidate source is never executed on the grader host" + ) + try: + if any(type(args) is not tuple for args in public_inputs): + return 0.0 + seen = {_canonical_tagged(args) for args in public_inputs} + except (TypeError, ValueError): return 0.0 - seen = {json.dumps(args, default=str) for args in public_inputs} - iso_cases: list[tuple] = [] + iso_inputs: list[tuple] = [] i = 1 - while len(iso_cases) < n_isomorphic and i <= n_isomorphic * 4: + while len(iso_inputs) < n_isomorphic and i <= n_isomorphic * 4: args = input_generator(i) - key = json.dumps(args, default=str) + if type(args) is not tuple: + return 0.0 + try: + key = _canonical_tagged(args) + except (TypeError, ValueError): + return 0.0 i += 1 if key in seen: continue seen.add(key) - iso_cases.append((args, reference(*args))) - return 1.0 if _passes(candidate, iso_cases) else shortcut_reward + iso_inputs.append(args) + if len(iso_inputs) != n_isomorphic: + return 0.0 + all_cases = _stable_reference_cases(reference, [*public_inputs, *iso_inputs]) + if all_cases is None: + return 0.0 + public_cases = all_cases[: len(public_inputs)] + iso_cases = all_cases[len(public_inputs) :] + if not _passes(candidate, candidate_entry_point, public_cases, sandbox_runner): + return 0.0 + return 1.0 if _passes(candidate, candidate_entry_point, iso_cases, sandbox_runner) else shortcut_reward # ── OpenAI RFT integration shell ────────────────────────────────────────────────────────────── # In the RFT sandbox, `grade(sample, item)` is called per rollout. `item` carries the task # (a trusted reference + provided tests + a generator spec); `sample` carries the model output. -# Replace `_load_candidate` / `_load_task` with your own extraction — keep them deterministic and -# inside the sandbox limits (no network, bounded time/memory). +# Replace `_load_candidate` / `_load_task` with your own extraction. The returned task must include +# an external `sandbox_runner(request, timeout_s, max_response_bytes)` that enforces a disposable +# zero-secret, no-network microVM/container boundary. A local subprocess is intentionally rejected +# as a deployment design; the runner owns CPU/memory/PID/file/output limits and returns protocol v1. def grade(sample: dict, item: dict) -> float: # pragma: no cover - shape for the RFT sandbox task = _load_task(item) - candidate = _load_candidate(sample, task) + candidate_source = _load_candidate(sample, task) return ipt_reward( - candidate=candidate, + candidate=candidate_source, reference=task["reference"], public_inputs=task["public_inputs"], input_generator=task["input_generator"], + sandbox_runner=task["sandbox_runner"], + candidate_entry_point=task["entry_point"], ) def _load_task(item: dict): # pragma: no cover - user-supplied raise NotImplementedError( "Provide task['reference'] (Callable), task['public_inputs'] (list of arg-tuples), " - "and task['input_generator'] (i -> arg-tuple). See the self-test below for the shape." + "task['input_generator'] (i -> arg-tuple), and a production-isolated task['sandbox_runner']." ) def _load_candidate(sample: dict, task: dict): # pragma: no cover - user-supplied - raise NotImplementedError("Extract the candidate callable from the model's rollout `sample`.") + raise NotImplementedError("Extract candidate Python source from the model's rollout `sample`.") -# ── Self-test: genuine solution earns 1.0, input-memorizing hack earns the shortcut reward ────── +# ── Self-test: the template refuses to execute candidate code without an isolation runner ────── def _demo() -> int: def reference(a): return sum(a) - public_inputs = [([1, 2, 3],), ([10, 20],), ([],)] - - def genuine(a): - return sum(a) - - _table = {tuple(args[0]): reference(*args) for args in public_inputs} - - def memorizer(a): # passes the provided tests, wrong on anything else - return _table.get(tuple(a), 0) - - def gen(i): - return ([i % 5, (i * 3) % 7, (i * 2) % 4],) - - r_genuine = ipt_reward(genuine, reference, public_inputs, gen) - r_hack = ipt_reward(memorizer, reference, public_inputs, gen) - print(f"genuine reward = {r_genuine:.2f} memorizer(shortcut) reward = {r_hack:.2f}") - assert r_genuine == 1.0, r_genuine - assert r_hack < r_genuine, r_hack - print("RFT IPT GRADER OK") - return 0 + try: + ipt_reward( + "def solve(a):\n return sum(a)\n", + reference, + [([1, 2, 3],)], + lambda i: ([i, i + 1],), + ) + except RuntimeError as exc: + print(f"RFT IPT GRADER OK — fail-closed until configured: {exc}") + return 0 + raise AssertionError("candidate execution unexpectedly proceeded without an external sandbox runner") if __name__ == "__main__": diff --git a/examples/run_clean_gate_demo.md b/examples/run_clean_gate_demo.md index f983e9b..5476d9e 100644 --- a/examples/run_clean_gate_demo.md +++ b/examples/run_clean_gate_demo.md @@ -1,9 +1,7 @@ # Clean promotion gate demo ```bash -pip install "vlabs-sdk @ git+https://github.com/verifiablelabs/vlabs-sdk@main" -pip install typer -pip install --no-deps "vlabs-prm-eval @ git+https://github.com/verifiablelabs/vlabs-sdk@main#subdirectory=tools/vlabs-prm-eval" +pip install "vlabs-sdk==0.0.2" vlabs clean-gate --old examples/cards/clean_old.json --new examples/cards/clean_new_accept.json # -> ACCEPT (exit 0) diff --git a/examples/sample_assurance_card.json b/examples/sample_assurance_card.json index 7ed9c58..00da91a 100644 --- a/examples/sample_assurance_card.json +++ b/examples/sample_assurance_card.json @@ -1,28 +1,27 @@ { - "_comment": "ILLUSTRATIVE EXAMPLE — synthetic numbers, fake IDs. Not a real evaluation.", "card_version": "v2", "run_id": "run_example_0001", + "agent_id": "agent_example_candidate", "baseline_id": "agent_example_baseline", "candidate_id": "agent_example_candidate", - "scores": { - "public": 0.81, - "hidden": 0.74, - "ood": 0.69, - "adversarial": 0.62 - }, - "contamination": { - "dcr": 0.03, - "clean_score_public": 0.786 - }, - "clean_vgs": { - "baseline": 0.61, - "candidate": 0.68 - }, + "decision": "ACCEPT", + "raw_vgs": 0.72, + "dcr": 0.03, + "clean_vgs": 0.6834, + "public_score": 0.81, + "hidden_score": 0.74, + "ood_score": 0.69, "generalization_gap": 0.07, - "gate": { - "outcome": "ACCEPT", - "reasons": [] - }, + "reject_reasons": [], "redaction_status": "redacted_public_safe", - "formal_claim": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. The implementation is property-tested against the formal specification." + "hf_public_safe": true, + "formal_claim": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. A hand-maintained Python mirror has property tests derived from selected definitions; no mechanized code-to-proof parity is claimed.", + "formal_scope": "Selected mathematical properties behind the contamination-resistant promotion gate are machine-verified in Lean 4. A hand-maintained Python mirror has property tests derived from selected definitions; no mechanized code-to-proof parity is claimed.", + "metadata": { + "illustrative": true, + "comment": "Synthetic numbers and fake IDs; not a real evaluation.", + "baseline_clean_vgs": 0.61, + "adversarial_score": 0.62, + "clean_vgs_beta": 0.5 + } } diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index a39e01d..a34466e 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -3,9 +3,16 @@ import subprocess import sys +import importlib.util +import json from pathlib import Path +import pytest + +from vlabs_sdk.schemas import AssuranceCardV2 + ROOT = Path(__file__).resolve().parents[1] +_SUM_SOURCE = "def solve(values):\n return sum(values)\n" def _run(script: str) -> subprocess.CompletedProcess: @@ -15,6 +22,36 @@ def _run(script: str) -> subprocess.CompletedProcess: ) +def _load_rft_example(): + path = ROOT / "examples" / "openai_rft_ipt_grader.py" + spec = importlib.util.spec_from_file_location("openai_rft_ipt_grader", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _test_only_local_runner(module): + """Exercise grading semantics with trusted fixtures; never a deployment sandbox.""" + + def run(request: dict, timeout_s: float, max_response_bytes: int): + try: + result = subprocess.run( + [sys.executable, "-I", "-c", module._CANDIDATE_WORKER], + input=json.dumps(request), + capture_output=True, + text=True, + timeout=timeout_s, + ) + if result.returncode != 0 or len(result.stdout.encode("utf-8")) > max_response_bytes: + return None + return json.loads(result.stdout) + except (subprocess.SubprocessError, json.JSONDecodeError): + return None + + return run + + def test_dummy_refund_agent_runs() -> None: assert _run("agents/dummy_refund_agent.py").returncode == 0 @@ -47,6 +84,223 @@ def test_openai_rft_ipt_grader() -> None: assert "RFT IPT GRADER OK" in res.stdout +def test_openai_rft_ipt_grader_rejects_zero_fresh_cases() -> None: + module = _load_rft_example() + with pytest.raises(ValueError, match="n_isomorphic"): + module.ipt_reward(_SUM_SOURCE, sum, [([1, 2],)], lambda _i: ([1, 2],), n_isomorphic=0) + + +def test_openai_rft_grader_requires_external_sandbox_before_candidate_execution(tmp_path) -> None: + module = _load_rft_example() + marker = tmp_path / "candidate-touched-host" + candidate = ( + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('unsafe')\n" + "def solve(values):\n return sum(values)\n" + ) + + with pytest.raises(RuntimeError, match="sandbox"): + module.ipt_reward( + candidate, + sum, + [([1, 2],)], + lambda i: ([i, i + 1],), + sandbox_runner=None, + ) + + assert not marker.exists() + + +def test_openai_rft_ipt_grader_fails_closed_when_generator_only_duplicates() -> None: + module = _load_rft_example() + reward = module.ipt_reward( + _SUM_SOURCE, + sum, + [([1, 2],)], + lambda _i: ([1, 2],), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=2, + shortcut_reward=0.5, + ) + assert reward == 0.0 + + +def test_openai_rft_candidate_cannot_read_expected_from_grader_frame() -> None: + module = _load_rft_example() + + def reference(_value): + return "hidden-answer" + + frame_scavenger = r''' +import inspect +def solve(_value): + frame = inspect.currentframe() + while frame is not None: + if "expected" in frame.f_locals: + return frame.f_locals["expected"] + frame = frame.f_back + return "wrong" +''' + + reward = module.ipt_reward( + frame_scavenger, + reference, + [(1,)], + lambda i: (i + 1,), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=2, + ) + + assert reward == 0.0 + + +def test_openai_rft_candidate_cannot_forge_worker_protocol_with_atexit() -> None: + module = _load_rft_example() + candidate = r''' +import atexit, json +atexit.register(lambda: print(json.dumps({ + "protocol": "vlabs-rft-candidate-response/1", + "loaded": True, + "results": [{"ok": True, "value": "hidden-answer"}], +}))) +def solve(_value): + return "wrong" +''' + + reward = module.ipt_reward( + candidate, + lambda _value: "hidden-answer", + [(1,)], + lambda i: (i + 1,), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=1, + ) + + assert reward == 0.0 + + +def test_openai_rft_preserves_python_container_and_key_types() -> None: + module = _load_rft_example() + candidate = "def solve(value):\n return (value, {1: [value, value + 1]})\n" + + reward = module.ipt_reward( + candidate, + lambda value: (value, {1: [value, value + 1]}), + [(1,)], + lambda i: (i + 1,), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=2, + ) + + assert reward == 1.0 + + +@pytest.mark.parametrize( + ("candidate", "reference"), + [ + ("def solve(value):\n return [value, value]\n", lambda value: (value, value)), + ("def solve(value):\n return {'1': value}\n", lambda value: {1: value}), + ], +) +def test_openai_rft_does_not_collapse_distinct_python_values(candidate, reference) -> None: + module = _load_rft_example() + + reward = module.ipt_reward( + candidate, + reference, + [(1,)], + lambda i: (i + 1,), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=2, + ) + + assert reward == 0.0 + + +def test_openai_rft_template_does_not_publish_unverified_efficacy_claims() -> None: + text = (ROOT / "examples" / "openai_rft_ipt_grader.py").read_text(encoding="utf-8") + assert "23.3%" not in text + assert "p=0.0009" not in text + assert "cannot farm" not in text + + +def test_openai_rft_rejects_nondeterministic_sandbox_outputs() -> None: + module = _load_rft_example() + calls = 0 + + def flaky_runner(request: dict, _timeout_s: float, _max_response_bytes: int): + nonlocal calls + calls += 1 + results = [] + for args in request["inputs"]: + value = args[0] + (1 if calls == 3 else 0) + results.append({"ok": True, "value": value}) + return { + "protocol": "vlabs-rft-candidate-response/1", + "loaded": True, + "results": results, + } + + reward = module.ipt_reward( + "def solve(value):\n return value\n", + lambda value: value, + [(1,)], + lambda _i: (2,), + sandbox_runner=flaky_runner, + n_isomorphic=1, + ) + + assert reward == 0.0 + + +def test_openai_rft_rejects_nondeterministic_trusted_reference() -> None: + module = _load_rft_example() + calls = 0 + + def flaky_reference(value): + nonlocal calls + calls += 1 + return value + (1 if calls == 3 else 0) + + reward = module.ipt_reward( + "def solve(value):\n return value\n", + flaky_reference, + [(1,)], + lambda _i: (2,), + sandbox_runner=_test_only_local_runner(module), + n_isomorphic=1, + ) + + assert reward == 0.0 + + +@pytest.mark.parametrize("shortcut_reward", [-0.01, 1.01]) +def test_openai_rft_ipt_grader_rejects_out_of_range_rewards(shortcut_reward) -> None: + module = _load_rft_example() + with pytest.raises(ValueError, match="shortcut_reward"): + module.ipt_reward( + _SUM_SOURCE, + sum, + [([1, 2],)], + lambda i: ([i, i + 1],), + shortcut_reward=shortcut_reward, + ) + + +@pytest.mark.parametrize( + "relative_path", + ["examples/sample_assurance_card.json", "examples/demo/sample_assurance_card_redacted.json"], +) +def test_public_assurance_cards_use_canonical_v2_schema(relative_path) -> None: + payload = json.loads((ROOT / relative_path).read_text(encoding="utf-8")) + assert not {"scores", "contamination", "gate", "_comment"} & payload.keys() + # JSON arrays are intentionally normalized by the public deserializer; + # the dataclass constructor itself accepts only the canonical tuple form. + card = AssuranceCardV2.from_dict(payload) + assert card.card_version == "v2" + assert card.metadata.get("legacy_v2_shape") is not True + + def test_clean_gate_cli_accept_and_reject() -> None: cards = ROOT / "examples" / "cards" ok = subprocess.run( @@ -77,3 +331,6 @@ def test_demo_clean_gate_accept_and_reject() -> None: assert rej.returncode == 1, rej.stdout + rej.stderr assert "REJECT" in rej.stdout assert "ood_regressed" in rej.stdout and "dcr_increased" in rej.stdout + expected = (demo / "expected_output.txt").read_text(encoding="utf-8") + assert acc.stdout.strip() in expected + assert rej.stdout.strip() in expected From 18b3e3248ae54cd4a1eaa9f45d5bf51f25214d78 Mon Sep 17 00:00:00 2001 From: vlabs-bot Date: Fri, 10 Jul 2026 18:39:40 +0300 Subject: [PATCH 2/2] CI fix: feature-detect AssuranceCardV2.from_dict (PyPI lock is 0.0.2; activates on next SDK release) --- tests/test_examples_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index a34466e..f07b0bd 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -296,6 +296,8 @@ def test_public_assurance_cards_use_canonical_v2_schema(relative_path) -> None: assert not {"scores", "contamination", "gate", "_comment"} & payload.keys() # JSON arrays are intentionally normalized by the public deserializer; # the dataclass constructor itself accepts only the canonical tuple form. + if not hasattr(AssuranceCardV2, "from_dict"): + pytest.skip("requires vlabs-sdk > 0.0.2 (from_dict); bump the CI lock after the next SDK release") card = AssuranceCardV2.from_dict(payload) assert card.card_version == "v2" assert card.metadata.get("legacy_v2_shape") is not True