From faf66b0198682d9bc638ffd17571af482eeacd42 Mon Sep 17 00:00:00 2001 From: Risa Wechsler Date: Thu, 2 Jul 2026 08:19:35 -0700 Subject: [PATCH 1/2] Add conclave TS1 submission (committee ensemble: PZFlow+GPz+FlexZBoost, QP weights, global-PIT) tests/test_conclave.py delegates to the pip-installable conclave package (github.com/rhw/conclave, pinned ts1-v1); requirements_conclave.txt installs it + RAIL deps; submit_conclave.yaml runs the CI with PZDC_CI_MAX_TRAIN so the train path subsamples (a full 3-estimator retrain on 100k x4 exceeds the CI budget). Pre-made full-scale estimates + models ship via the release tgz for the estimation-only path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/submit_conclave.yaml | 43 ++++++++ requirements_conclave.txt | 6 ++ tests/test_conclave.py | 140 +++++++++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 .github/workflows/submit_conclave.yaml create mode 100644 requirements_conclave.txt create mode 100644 tests/test_conclave.py diff --git a/.github/workflows/submit_conclave.yaml b/.github/workflows/submit_conclave.yaml new file mode 100644 index 0000000..a8a2010 --- /dev/null +++ b/.github/workflows/submit_conclave.yaml @@ -0,0 +1,43 @@ +--- +# This workflow will install Python dependencies and run tests + +name: Unit test and code coverage + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + submission: ['conclave'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt install libbz2-dev + python -m pip install --upgrade pip + pip install wheel + pip install . + pip install .[dev] + if [ -f requirements_${{ matrix.submission }}.txt ]; then pip install -r requirements_${{ matrix.submission }}.txt; fi + - name: Run unit tests with pytest + # conclave trains 3 estimators; a full-scale retrain (100k x4) exceeds the CI budget. + # Subsample the train path for CI (validity smoke); estimation runs on the full test set, + # and the shipped full-scale models/estimates (release tgz) carry the real performance. + env: + PZDC_CI_MAX_TRAIN: "4000" + run: | + python -m pytest tests/test_${{ matrix.submission }}.py diff --git a/requirements_conclave.txt b/requirements_conclave.txt new file mode 100644 index 0000000..bf9df1c --- /dev/null +++ b/requirements_conclave.txt @@ -0,0 +1,6 @@ +# conclave TS1 submission requirements. +# The method lives in the pip-installable `conclave` package; installing it pulls the +# RAIL estimator stack (pz-rail-{base,flexzboost,pzflow,gpz-v1}), qp-prob, tables_io, +# numpy, scipy transitively via conclave's own pyproject dependencies. +# Pin to the tagged release that matches the shipped pretrained models / estimates. +conclave @ git+https://github.com/rhw/conclave@ts1-v1 diff --git a/tests/test_conclave.py b/tests/test_conclave.py new file mode 100644 index 0000000..954223d --- /dev/null +++ b/tests/test_conclave.py @@ -0,0 +1,140 @@ +"""CI test for the `conclave` TS1 submission (LSST-DESC PZ Data Challenge). + +conclave = a committee ensemble of PZFlow + GPz + FlexZBoost combined with convex-QP +optimal weights and a global-PIT recalibration. The method lives in the pip-installable +`conclave` package (github.com/rhw/conclave, pinned in requirements_conclave.txt); this +test file is the thin submission entry point that the upstream harness discovers and runs. + +Structure mirrors the accepted reference submissions (e.g. test_rail_knn_test.py): it defines +the four run_taskset_* entry points and calls run_taskset_1/2 over SIMS x SCENARIOS. The two +functions delegate to conclave.submission, which internally applies the estimators' catalog tag +and freezes the QP weights + recalibrator (the blind test has no redshift, so recal is fit on a +held-out slice of the training file). Pre-made estimates + pretrained models are hosted at +SUBMISSION_URL and unpacked into submissions/conclave/. +""" +import os +from pathlib import Path + +import numpy as np +import pytest +from rail.core.data import TableHandle +from rail.utils import catalog_utils + +from pz_data_challenge import submit_utils +from pz_data_challenge.taskset_1 import run_taskset_1 +from pz_data_challenge.taskset_2 import run_taskset_2 + +from conclave.submission import ( + run_taskset_1_training_and_estimation as _conclave_train_and_estimate, + run_taskset_1_estimation_only as _conclave_estimate_only, +) + +# CI concession: the full method trains 3 estimators (PZFlow/GPz/FlexZBoost) on 100k x 4 +# sim/scenario — hours, over the GitHub-runner budget. For the train+estimate CI path we train +# on a subsample so CI proves the pipeline runs and emits valid p(z); the REAL full-scale-trained +# models + estimates ship in the release tarball (SUBMISSION_URL) and feed the estimation-only +# path. PZDC_CI_MAX_TRAIN=0 (default) trains on the full data (the real submission behaviour). +CI_MAX_TRAIN: int = int(os.environ.get("PZDC_CI_MAX_TRAIN", "0")) + +SUBMISSION_NAME: str = "conclave" +# GitHub release .tgz of the pre-made estimates + pretrained models (set before opening the PR). +SUBMISSION_URL: str = "https://github.com/rhw/conclave/releases/download/ts1-v1/conclave_submission.tgz" + +SUBMIT_DIR: str = f"submissions/{SUBMISSION_NAME}" +PUBLIC_AREA: str = os.environ.get("PZDC_PUBLIC_AREA", "tests/public") + +# Official catalog tag for reading the challenge catalogs. conclave's estimators additionally +# apply their own (equivalent LSST+Roman) tag internally via conclave.bands.apply_band_set. +CATALOG_TAG = "cardinal_roman_rubin" + + +@pytest.fixture(name="setup_submit_area", scope="module") +def setup_submit_area(request: pytest.FixtureRequest) -> int: + if not os.path.exists(SUBMIT_DIR): + if not SUBMISSION_URL: + raise ValueError( + f"SUBMISSION_URL in tests/test_{SUBMISSION_NAME}.py has not been set" + ) + submit_utils.download_and_extract_tar(SUBMISSION_URL, SUBMIT_DIR) + + def teardown_submit_area() -> None: + if not os.environ.get("NO_TEARDOWN"): + os.system(f"\\rm -rf {SUBMIT_DIR}/outputs_2 {SUBMIT_DIR}/outputs_3") + + for sub in ("outputs_2", "outputs_3"): + try: + os.makedirs(os.path.join(SUBMIT_DIR, sub)) + except Exception: + pass + request.addfinalizer(teardown_submit_area) + + catalog_utils.clear() + catalog_utils.load_yaml("tests/catalogs.yaml") + catalog_utils.apply(CATALOG_TAG) + return 0 + + +def _maybe_subsample_train(train_file: str) -> str: + """Return train_file unchanged unless PZDC_CI_MAX_TRAIN>0, in which case write a + stratified-by-nothing random subsample to a temp hdf5 (keeps CI train fast).""" + if CI_MAX_TRAIN <= 0: + return train_file + import tempfile + import tables_io + d = tables_io.read(train_file) + keys = list(d.keys()) + n = len(d[keys[0]]) + if n <= CI_MAX_TRAIN: + return train_file + idx = np.sort(np.random.default_rng(0).choice(n, CI_MAX_TRAIN, replace=False)) + sub = {k: np.asarray(d[k])[idx] for k in keys} + stem = os.path.join(tempfile.mkdtemp(), "ci_train") + tables_io.write(sub, stem, "hdf5") + return stem + ".hdf5" + + +def run_taskset_1_estimation_only( + model_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + _conclave_estimate_only(str(model_file), str(test_file), str(output_file)) + + +def run_taskset_1_training_and_estimation( + train_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + train_file = _maybe_subsample_train(str(train_file)) + _conclave_train_and_estimate(train_file, str(test_file), str(output_file)) + + +def run_taskset_2_estimation_only( + model_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + run_taskset_1_estimation_only(model_file, test_file, output_file) + + +def run_taskset_2_training_and_estimation( + train_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + run_taskset_1_training_and_estimation(train_file, test_file, output_file) + + +def test_example_taskset_1(setup_public_area: int, setup_submit_area: int) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + run_taskset_1( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_1_estimation_only, + run_taskset_1_training_and_estimation, + ) + + +def test_example_taskset_2(setup_public_area: int, setup_submit_area: int) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + run_taskset_2( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_2_estimation_only, + run_taskset_2_training_and_estimation, + ) From 421f86e7366c77a910f7997e8c5c456756719d3c Mon Sep 17 00:00:00 2001 From: Risa Wechsler Date: Thu, 2 Jul 2026 15:26:18 -0700 Subject: [PATCH 2/2] conclave: TS1-only test (remove taskset_2 test; ship TS1 deliverables only) Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_conclave.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/tests/test_conclave.py b/tests/test_conclave.py index 954223d..292689f 100644 --- a/tests/test_conclave.py +++ b/tests/test_conclave.py @@ -22,7 +22,6 @@ from pz_data_challenge import submit_utils from pz_data_challenge.taskset_1 import run_taskset_1 -from pz_data_challenge.taskset_2 import run_taskset_2 from conclave.submission import ( run_taskset_1_training_and_estimation as _conclave_train_and_estimate, @@ -106,19 +105,8 @@ def run_taskset_1_training_and_estimation( _conclave_train_and_estimate(train_file, str(test_file), str(output_file)) -def run_taskset_2_estimation_only( - model_file: str | Path, test_file: str | Path, output_file: str | Path, -) -> None: - run_taskset_1_estimation_only(model_file, test_file, output_file) - - -def run_taskset_2_training_and_estimation( - train_file: str | Path, test_file: str | Path, output_file: str | Path, -) -> None: - run_taskset_1_training_and_estimation(train_file, test_file, output_file) - - def test_example_taskset_1(setup_public_area: int, setup_submit_area: int) -> None: + # TS1 submission: only Task Set 1 deliverables are shipped (Task Set 2 closes later). assert setup_public_area == 0 assert setup_submit_area == 0 run_taskset_1( @@ -127,14 +115,3 @@ def test_example_taskset_1(setup_public_area: int, setup_submit_area: int) -> No run_taskset_1_estimation_only, run_taskset_1_training_and_estimation, ) - - -def test_example_taskset_2(setup_public_area: int, setup_submit_area: int) -> None: - assert setup_public_area == 0 - assert setup_submit_area == 0 - run_taskset_2( - PUBLIC_AREA, - SUBMISSION_NAME, - run_taskset_2_estimation_only, - run_taskset_2_training_and_estimation, - )