diff --git a/.github/workflows/submit_conclave.yaml b/.github/workflows/submit_conclave.yaml new file mode 100644 index 0000000..a8a2010 --- /dev/null +++ b/.github/workflows/submit_conclave.yaml @@ -0,0 +1,43 @@ +--- +# This workflow will install Python dependencies and run tests + +name: Unit test and code coverage + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + submission: ['conclave'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt install libbz2-dev + python -m pip install --upgrade pip + pip install wheel + pip install . + pip install .[dev] + if [ -f requirements_${{ matrix.submission }}.txt ]; then pip install -r requirements_${{ matrix.submission }}.txt; fi + - name: Run unit tests with pytest + # conclave trains 3 estimators; a full-scale retrain (100k x4) exceeds the CI budget. + # Subsample the train path for CI (validity smoke); estimation runs on the full test set, + # and the shipped full-scale models/estimates (release tgz) carry the real performance. + env: + PZDC_CI_MAX_TRAIN: "4000" + run: | + python -m pytest tests/test_${{ matrix.submission }}.py diff --git a/requirements_conclave.txt b/requirements_conclave.txt new file mode 100644 index 0000000..bf9df1c --- /dev/null +++ b/requirements_conclave.txt @@ -0,0 +1,6 @@ +# conclave TS1 submission requirements. +# The method lives in the pip-installable `conclave` package; installing it pulls the +# RAIL estimator stack (pz-rail-{base,flexzboost,pzflow,gpz-v1}), qp-prob, tables_io, +# numpy, scipy transitively via conclave's own pyproject dependencies. +# Pin to the tagged release that matches the shipped pretrained models / estimates. +conclave @ git+https://github.com/rhw/conclave@ts1-v1 diff --git a/tests/test_conclave.py b/tests/test_conclave.py new file mode 100644 index 0000000..292689f --- /dev/null +++ b/tests/test_conclave.py @@ -0,0 +1,117 @@ +"""CI test for the `conclave` TS1 submission (LSST-DESC PZ Data Challenge). + +conclave = a committee ensemble of PZFlow + GPz + FlexZBoost combined with convex-QP +optimal weights and a global-PIT recalibration. The method lives in the pip-installable +`conclave` package (github.com/rhw/conclave, pinned in requirements_conclave.txt); this +test file is the thin submission entry point that the upstream harness discovers and runs. + +Structure mirrors the accepted reference submissions (e.g. test_rail_knn_test.py): it defines +the four run_taskset_* entry points and calls run_taskset_1/2 over SIMS x SCENARIOS. The two +functions delegate to conclave.submission, which internally applies the estimators' catalog tag +and freezes the QP weights + recalibrator (the blind test has no redshift, so recal is fit on a +held-out slice of the training file). Pre-made estimates + pretrained models are hosted at +SUBMISSION_URL and unpacked into submissions/conclave/. +""" +import os +from pathlib import Path + +import numpy as np +import pytest +from rail.core.data import TableHandle +from rail.utils import catalog_utils + +from pz_data_challenge import submit_utils +from pz_data_challenge.taskset_1 import run_taskset_1 + +from conclave.submission import ( + run_taskset_1_training_and_estimation as _conclave_train_and_estimate, + run_taskset_1_estimation_only as _conclave_estimate_only, +) + +# CI concession: the full method trains 3 estimators (PZFlow/GPz/FlexZBoost) on 100k x 4 +# sim/scenario — hours, over the GitHub-runner budget. For the train+estimate CI path we train +# on a subsample so CI proves the pipeline runs and emits valid p(z); the REAL full-scale-trained +# models + estimates ship in the release tarball (SUBMISSION_URL) and feed the estimation-only +# path. PZDC_CI_MAX_TRAIN=0 (default) trains on the full data (the real submission behaviour). +CI_MAX_TRAIN: int = int(os.environ.get("PZDC_CI_MAX_TRAIN", "0")) + +SUBMISSION_NAME: str = "conclave" +# GitHub release .tgz of the pre-made estimates + pretrained models (set before opening the PR). +SUBMISSION_URL: str = "https://github.com/rhw/conclave/releases/download/ts1-v1/conclave_submission.tgz" + +SUBMIT_DIR: str = f"submissions/{SUBMISSION_NAME}" +PUBLIC_AREA: str = os.environ.get("PZDC_PUBLIC_AREA", "tests/public") + +# Official catalog tag for reading the challenge catalogs. conclave's estimators additionally +# apply their own (equivalent LSST+Roman) tag internally via conclave.bands.apply_band_set. +CATALOG_TAG = "cardinal_roman_rubin" + + +@pytest.fixture(name="setup_submit_area", scope="module") +def setup_submit_area(request: pytest.FixtureRequest) -> int: + if not os.path.exists(SUBMIT_DIR): + if not SUBMISSION_URL: + raise ValueError( + f"SUBMISSION_URL in tests/test_{SUBMISSION_NAME}.py has not been set" + ) + submit_utils.download_and_extract_tar(SUBMISSION_URL, SUBMIT_DIR) + + def teardown_submit_area() -> None: + if not os.environ.get("NO_TEARDOWN"): + os.system(f"\\rm -rf {SUBMIT_DIR}/outputs_2 {SUBMIT_DIR}/outputs_3") + + for sub in ("outputs_2", "outputs_3"): + try: + os.makedirs(os.path.join(SUBMIT_DIR, sub)) + except Exception: + pass + request.addfinalizer(teardown_submit_area) + + catalog_utils.clear() + catalog_utils.load_yaml("tests/catalogs.yaml") + catalog_utils.apply(CATALOG_TAG) + return 0 + + +def _maybe_subsample_train(train_file: str) -> str: + """Return train_file unchanged unless PZDC_CI_MAX_TRAIN>0, in which case write a + stratified-by-nothing random subsample to a temp hdf5 (keeps CI train fast).""" + if CI_MAX_TRAIN <= 0: + return train_file + import tempfile + import tables_io + d = tables_io.read(train_file) + keys = list(d.keys()) + n = len(d[keys[0]]) + if n <= CI_MAX_TRAIN: + return train_file + idx = np.sort(np.random.default_rng(0).choice(n, CI_MAX_TRAIN, replace=False)) + sub = {k: np.asarray(d[k])[idx] for k in keys} + stem = os.path.join(tempfile.mkdtemp(), "ci_train") + tables_io.write(sub, stem, "hdf5") + return stem + ".hdf5" + + +def run_taskset_1_estimation_only( + model_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + _conclave_estimate_only(str(model_file), str(test_file), str(output_file)) + + +def run_taskset_1_training_and_estimation( + train_file: str | Path, test_file: str | Path, output_file: str | Path, +) -> None: + train_file = _maybe_subsample_train(str(train_file)) + _conclave_train_and_estimate(train_file, str(test_file), str(output_file)) + + +def test_example_taskset_1(setup_public_area: int, setup_submit_area: int) -> None: + # TS1 submission: only Task Set 1 deliverables are shipped (Task Set 2 closes later). + assert setup_public_area == 0 + assert setup_submit_area == 0 + run_taskset_1( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_1_estimation_only, + run_taskset_1_training_and_estimation, + )