From 50de297eef6ab70a37cb6483e35ee79ec9e2493f Mon Sep 17 00:00:00 2001 From: srinivasdrona Date: Mon, 3 Aug 2026 18:31:15 +0530 Subject: [PATCH] fix(ci): restore offline cross-platform contract Fetch full Git history for preservation tests, formalize the approved CRLF byte contract for pinned artifacts, skip reference-backed tests only when external data is absent, retain explicit opt-in markers, and replace collection-order-dependent conftest imports with stable eval factories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7c146921-f3dd-4a1e-8cf0-8f574de49204 --- .gitattributes | 15 +++++ .github/workflows/ci.yml | 4 ++ tests/eval/factories.py | 78 ++++++++++++++++++++++ tests/eval/test_export_dev_vcf.py | 5 +- tests/eval/test_path_resolver.py | 15 +++++ tests/eval/test_pp3bp4_disabled_policy.py | 8 ++- tests/eval/test_pp3bp4_matrix_expansion.py | 1 - tests/eval/test_pp3bp4_transportability.py | 8 ++- 8 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 .gitattributes create mode 100644 tests/eval/factories.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7c9858c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# These text artifacts were approved and hash-pinned from CRLF bytes. +# Preserve that byte contract on every platform; changing it requires a +# separately approved policy/provenance repin. +configs/acmg/tsc.yaml text eol=crlf +configs/eval/tsc2.yaml text eol=crlf +configs/eval/bias_lineage.yaml text eol=crlf +configs/packet/candidate_direction.yaml text eol=crlf +configs/eval/predictor_aggregation.yaml text eol=crlf +configs/eval/predictor_training_manifests.yaml text eol=crlf +configs/eval/pp3bp4_source_register.yaml text eol=crlf +configs/eval/pp3bp4_candidate_policy.json text eol=crlf +src/raptor/eval/predictor_aggregation.py text eol=crlf +src/raptor/eval/predictor_policy.py text eol=crlf +src/raptor/eval/terminal_source.py text eol=crlf +scripts/run_masked_holdout_eval.py text eol=crlf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97a917a..1d24f1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # Preservation tests compare immutable v1/v2 surfaces with their + # approved base commit, so the base must exist on the runner. + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/tests/eval/factories.py b/tests/eval/factories.py new file mode 100644 index 0000000..f121d58 --- /dev/null +++ b/tests/eval/factories.py @@ -0,0 +1,78 @@ +"""Stable shared constructors for eval tests. + +Unlike pytest's specially loaded ``conftest`` modules, this module has one +package-qualified identity regardless of collection order. +""" + +from __future__ import annotations + +from raptor.eval.config import EvalConfig +from raptor.eval.model import Metrics + + +def make_eval_config(**overrides) -> EvalConfig: + base = dict( + automatable_criteria=["PVS1", "PS3", "PM1", "PM2", "PP3", "BA1", "BS1", "BS2", "BP4", "BP7"], + tavtigian_points={ + "supporting": 1, "moderate": 2, "strong": 4, "very_strong": 8, "stand_alone": 8, + }, + tavtigian_cutoffs={ + "pathogenic_min": 10, "likely_pathogenic_min": 6, + "vus_min": 0, "vus_max": 5, + "likely_benign_max": -1, "benign_max": -7, + }, + min_count_per_class=10, + split={"seed": 42, "holdout_fraction": 0.3}, + oracle_thresholds={}, + labels_snapshot="clinvar_2026-07-01", + ) + base.update(overrides) + return EvalConfig(**base) + + +def make_v2_auth_config() -> dict: + return { + "schema_version": 2, + "research_use_disclaimer": "Research-evidence validation only; this authorizes no clinical classification, VUS worklist, or ClinVar submission.", + "full_spectrum": { + "requires": ["missense:pathogenic", "missense:benign", "truncating:pathogenic"] + }, + "research_scopes": { + "truncating_pathogenic_research_scope_validated": { + "requires": ["truncating:pathogenic"] + } + }, + "governance_statements": { + "FULL_SPECTRUM": "All pre-registered research scopes are validated for research-evidence use only; this authorizes no clinical classification, VUS worklist, or ClinVar submission.", + "TRUNCATING_PATHOGENIC_ONLY": "Full-spectrum VUS automation is not authorized. Evidence supports only the validated truncating-pathogenic scope; missense remains unvalidated.", + "NONE_VALIDATED": "Full-spectrum VUS automation is not authorized; no pre-registered research scope is currently validated." + }, + } + + +def make_oracle_thresholds() -> dict: + return { + "confidence": 0.95, + "strata": { + "missense": { + "precision": 0.90, + "recall": 0.85, + "gating": True, + "directions": ["pathogenic", "benign"], + }, + "truncating": { + "precision": 0.95, + "recall": 0.95, + "gating": True, + "directions": ["pathogenic"], + }, + }, + } + + +__all__ = [ + "Metrics", + "make_eval_config", + "make_v2_auth_config", + "make_oracle_thresholds", +] diff --git a/tests/eval/test_export_dev_vcf.py b/tests/eval/test_export_dev_vcf.py index 371279d..c3912ea 100644 --- a/tests/eval/test_export_dev_vcf.py +++ b/tests/eval/test_export_dev_vcf.py @@ -6,6 +6,7 @@ from pathlib import Path +@pytest.mark.requires_reference def test_ta1_dev_export_blocked_and_deterministic(tmp_path): """T-A1 dev export blocked. @@ -24,9 +25,9 @@ def test_ta1_dev_export_blocked_and_deterministic(tmp_path): if not script_path.exists(): pytest.fail(f"implementation missing: {script_path}") - from tests.eval.test_path_resolver import resolve_raptor_data_root + from tests.eval.test_path_resolver import require_raptor_data_root repo_root = Path(__file__).resolve().parents[2] - data_root = resolve_raptor_data_root(repo_root) + data_root = require_raptor_data_root(repo_root) real_benchmark = data_root / "clinvar" / "benchmark" / "benchmark.jsonl" if not real_benchmark.exists(): pytest.fail(f"Missing real frozen benchmark: {real_benchmark}") diff --git a/tests/eval/test_path_resolver.py b/tests/eval/test_path_resolver.py index 92f8cf1..8f1b21f 100644 --- a/tests/eval/test_path_resolver.py +++ b/tests/eval/test_path_resolver.py @@ -26,6 +26,21 @@ def resolve_raptor_data_root(repo_root: Path) -> Path: ) +def require_raptor_data_root(repo_root: Path) -> Path: + """Resolve reference data or skip explicitly in offline CI. + + An explicitly configured but missing root is a broken environment and + fails instead of silently skipping. + """ + env_root = os.environ.get("RAPTOR_DATA_ROOT") + if env_root and not Path(env_root).exists(): + pytest.fail(f"RAPTOR_DATA_ROOT does not exist: {env_root}") + try: + return resolve_raptor_data_root(repo_root) + except FileNotFoundError as exc: + pytest.skip(f"requires_reference: {exc}") + + def test_resolver_with_synthetic_paths(tmp_path): # Proves the resolver works for both main and worktree layouts diff --git a/tests/eval/test_pp3bp4_disabled_policy.py b/tests/eval/test_pp3bp4_disabled_policy.py index 54e8f86..e9ef628 100644 --- a/tests/eval/test_pp3bp4_disabled_policy.py +++ b/tests/eval/test_pp3bp4_disabled_policy.py @@ -774,8 +774,12 @@ def test_g_dm13_authorization_neutrality_unit(): assert "policy_mode" not in sig.parameters # Run them with passing and failing metrics and verify they behave normally without any policy override - from conftest import make_eval_config, Metrics - from test_scope_gate import make_v2_auth_config, make_oracle_thresholds + from tests.eval.factories import ( + Metrics, + make_eval_config, + make_oracle_thresholds, + make_v2_auth_config, + ) # PASSING CASE (distinct Metrics objects for each stratum with passing pathogenic and benign bounds) passing_config = make_eval_config( diff --git a/tests/eval/test_pp3bp4_matrix_expansion.py b/tests/eval/test_pp3bp4_matrix_expansion.py index 046859a..742314c 100644 --- a/tests/eval/test_pp3bp4_matrix_expansion.py +++ b/tests/eval/test_pp3bp4_matrix_expansion.py @@ -847,4 +847,3 @@ def test_t20_source_id_rule(): if avail.get("status") == "confirm_pending": assert len(avail.get("source_ids", [])) == 0, f"'{cid}' structured_score_availability has non-empty source_ids" - diff --git a/tests/eval/test_pp3bp4_transportability.py b/tests/eval/test_pp3bp4_transportability.py index 5db5239..755de9f 100644 --- a/tests/eval/test_pp3bp4_transportability.py +++ b/tests/eval/test_pp3bp4_transportability.py @@ -10,9 +10,9 @@ # Helper to load the real benchmark rows def _load_real_benchmark_rows(): from raptor.eval.model import BenchmarkRow - from tests.eval.test_path_resolver import resolve_raptor_data_root + from tests.eval.test_path_resolver import require_raptor_data_root repo_root = Path(__file__).resolve().parents[2] - data_root = resolve_raptor_data_root(repo_root) + data_root = require_raptor_data_root(repo_root) real_benchmark_path = data_root / "clinvar" / "benchmark" / "benchmark.jsonl" if not real_benchmark_path.exists(): pytest.fail(f"Missing real frozen benchmark: {real_benchmark_path}") @@ -51,6 +51,7 @@ def _compute_ids_hash(ids): return hasher.hexdigest() +@pytest.mark.requires_reference def test_te1_stage_b_attestation_and_rejection(tmp_path): """T-E1 Stage B transportability boundary. @@ -168,6 +169,7 @@ class DummyPolicy: ) +@pytest.mark.requires_reference def test_te2_dev_split_composition_and_power(): """T-E2 counts/power. @@ -203,6 +205,7 @@ def test_te2_dev_split_composition_and_power(): assert power == "UNDERPOWERED" +@pytest.mark.requires_reference def test_te1_no_writeback(): """T-E1 no label writeback. @@ -282,4 +285,3 @@ def test_cli_help_bootstrap(): res = subprocess.run(cmd, capture_output=True, text=True, env=env) assert res.returncode == 0, f"CLI help failed under clean PYTHONPATH: {res.stderr}" assert "usage" in res.stdout.lower() or "help" in res.stdout.lower() -