diff --git a/.github/workflows/submit_rail_knn_4tasks.yaml b/.github/workflows/submit_rail_knn_4tasks.yaml new file mode 100644 index 0000000..b99a399 --- /dev/null +++ b/.github/workflows/submit_rail_knn_4tasks.yaml @@ -0,0 +1,38 @@ +--- +# This workflow will install Python dependencies and run tests + +name: Unit test and code coverage + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + submission: ['rail_knn_4tasks'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt install libbz2-dev + python -m pip install --upgrade pip + pip install wheel + pip install . + pip install .[dev] + if [ -f requirements_${{ matrix.submission }}.txt ]; then pip install -r requirements_${{ matrix.submission }}.txt; fi + - name: Run unit tests with pytest + run: | + python -m pytest tests/test_${{ matrix.submission }}.py diff --git a/requirements_rail_knn_4tasks.txt b/requirements_rail_knn_4tasks.txt new file mode 100644 index 0000000..f649f3b --- /dev/null +++ b/requirements_rail_knn_4tasks.txt @@ -0,0 +1,4 @@ +pz-rail-base>=2.0.0 +pz-rail-sklearn>=2.0.0 +qp-prob>=1.0.0 +tables_io>=1.0.0 diff --git a/scripts/build_rail_knn_4tasks_submission.py b/scripts/build_rail_knn_4tasks_submission.py new file mode 100644 index 0000000..3ad0cf7 --- /dev/null +++ b/scripts/build_rail_knn_4tasks_submission.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Build precomputed rail_knn_4tasks submission tarball.""" +import math +import os +import tarfile + +import numpy as np +import tables_io +from rail.core.data import TableHandle +from rail.estimation.algos.k_nearneigh import KNearNeighEstimator, KNearNeighInformer +from rail.utils import catalog_utils + +PUBLIC_AREA = os.environ.get("PUBLIC_AREA", "tests/public") +OUT_DIR = os.environ.get("OUT_DIR", "build_submission/rail_knn_4tasks") +TASKSETS = [1, 2, 3, 4] +SIMS = ["cardinal", "flagship"] +SCENARIOS = ["1yr", "10yr"] +_CHUNK = 150_000 +_ZMAX = 3.0 +_NZ = 151 + + +def clean_training_file(train_file: str) -> str: + data = tables_io.read(train_file) + bad_mask = np.isnan(data["redshift"]) + if not bad_mask.any(): + return train_file + cleaned_path = train_file.replace(".hdf5", "_cleaned.hdf5") + cleaned_data = {key: val[~bad_mask] for key, val in data.items()} + tables_io.write(cleaned_data, cleaned_path) + return cleaned_path + + +def make_informer() -> KNearNeighInformer: + return KNearNeighInformer.make_stage( + name="inform", + hdf5_groupname="", + zmax=_ZMAX, + nzbins=_NZ, + chunk_size=_CHUNK, + nondetect_val=math.nan, + trainfrac=0.2, + nneigh_min=3, + nneigh_max=5, + ngrid_sigma=6, + ) + + +def make_estimator(model) -> KNearNeighEstimator: + return KNearNeighEstimator.make_stage( + name="estimate", + model=model, + hdf5_groupname="", + output_mode="return", + nzbins=_NZ, + zmax=_ZMAX, + chunk_size=_CHUNK, + nondetect_val=math.nan, + ) + + +def main() -> None: + os.makedirs(OUT_DIR, exist_ok=True) + catalog_utils.clear() + catalog_utils.load_yaml("tests/catalogs.yaml") + catalog_utils.apply("cardinal_roman_rubin") + + for taskset in TASKSETS: + for sim in SIMS: + for scenario in SCENARIOS: + train_file = ( + f"{PUBLIC_AREA}/pz_challenge_taskset_{taskset}_{sim}_" + f"training_{scenario}.hdf5" + ) + test_file = ( + f"{PUBLIC_AREA}/pz_challenge_taskset_{taskset}_{sim}_" + f"test_{scenario}.hdf5" + ) + model_path = ( + f"{OUT_DIR}/pz_challenge_taskset_{taskset}_{sim}_" + f"pz_model_{scenario}.pkl" + ) + estimate_path = ( + f"{OUT_DIR}/pz_challenge_taskset_{taskset}_{sim}_" + f"pz_estimate_{scenario}.hdf5" + ) + + cleaned_train_file = clean_training_file(train_file) + train_data = TableHandle("train", path=cleaned_train_file) + test_data = TableHandle("test", path=test_file) + + model = make_informer().inform(train_data) + model.path = model_path + model.write() + + pz_out = make_estimator(model).estimate(test_data) + pz_out.data.ancil["object_id"] = np.asarray(test_data()["object_id"]) + pz_out.path = estimate_path + pz_out.write() + print(f"Wrote {model_path} and {estimate_path}") + + tarball = "rail_knn_4tasks_submission.tgz" + with tarfile.open(tarball, "w:gz") as tar: + tar.add(OUT_DIR, arcname="rail_knn_4tasks") + print(f"Created {tarball}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_rail_knn_4tasks.py b/tests/test_rail_knn_4tasks.py new file mode 100644 index 0000000..6aa23a3 --- /dev/null +++ b/tests/test_rail_knn_4tasks.py @@ -0,0 +1,260 @@ +import math +import os +from pathlib import Path + +import numpy as np +import pytest +import tables_io +from rail.core.data import TableHandle +from rail.estimation.algos.k_nearneigh import KNearNeighEstimator, KNearNeighInformer +from rail.utils import catalog_utils + +from pz_data_challenge import submit_utils +from pz_data_challenge.taskset_1 import run_taskset_1 +from pz_data_challenge.taskset_2 import run_taskset_2 +from pz_data_challenge.taskset_3 import run_taskset_3 +from pz_data_challenge.taskset_4 import run_taskset_4 + +SUBMISSION_NAME: str = "rail_knn_4tasks" +SUBMISSION_URL: str = ( + "https://github.com/Lhior/pz_data_challenge/releases/download/" + "pzdc-rail-knn-4tasks-v1/rail_knn_4tasks_submission.tgz" +) + +SUBMIT_DIR: str = f"submissions/{SUBMISSION_NAME}" +PUBLIC_AREA: str = "tests/public" + +CATALOG_TAG = "cardinal_roman_rubin" +_CHUNK = 150_000 +_ZMAX = 3.0 +_NZ = 151 + + +def _attach_object_ids(pz_out, test_data: TableHandle) -> None: + pz_out.data.ancil["object_id"] = np.asarray(test_data()["object_id"]) + + +def _clean_training_file(train_file: str | Path) -> str: + path = str(train_file) + data = tables_io.read(path) + bad_mask = np.isnan(data["redshift"]) + if not bad_mask.any(): + return path + cleaned_path = path.replace(".hdf5", "_cleaned.hdf5") + cleaned_data = {key: val[~bad_mask] for key, val in data.items()} + tables_io.write(cleaned_data, cleaned_path) + return cleaned_path + + +def _make_knn_informer() -> KNearNeighInformer: + return KNearNeighInformer.make_stage( + name="inform", + hdf5_groupname="", + zmax=_ZMAX, + nzbins=_NZ, + chunk_size=_CHUNK, + nondetect_val=math.nan, + trainfrac=0.2, + nneigh_min=3, + nneigh_max=5, + ngrid_sigma=6, + ) + + +def _make_knn_estimator(model) -> KNearNeighEstimator: + return KNearNeighEstimator.make_stage( + name="estimate", + model=model, + hdf5_groupname="", + output_mode="return", + nzbins=_NZ, + zmax=_ZMAX, + chunk_size=_CHUNK, + nondetect_val=math.nan, + ) + + +@pytest.fixture(name="setup_submit_area", scope="module") +def setup_submit_area(request: pytest.FixtureRequest) -> int: + if not os.path.exists(SUBMIT_DIR): + if not SUBMISSION_URL: + raise ValueError( + f"SUBMISSION_URL in tests/test_{SUBMISSION_NAME}.py has not been set" + ) + submit_utils.download_and_extract_tar(SUBMISSION_URL, "submissions") + + def teardown_submit_area() -> None: + if not os.environ.get("NO_TEARDOWN"): + os.system(f"\\rm -rf {SUBMIT_DIR}") + + try: + os.makedirs(os.path.join(SUBMIT_DIR, "outputs_2")) + except Exception: + pass + + try: + os.makedirs(os.path.join(SUBMIT_DIR, "outputs_3")) + except Exception: + pass + + request.addfinalizer(teardown_submit_area) + + catalog_utils.clear() + catalog_utils.load_yaml("tests/catalogs.yaml") + catalog_utils.apply(CATALOG_TAG) + + return 0 + + +def run_taskset_x_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + test_data = TableHandle("test", path=str(test_file)) + estimator = _make_knn_estimator(str(model_file)) + pz_out = estimator.estimate(test_data) + _attach_object_ids(pz_out, test_data) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_x_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + cleaned_train_file = _clean_training_file(train_file) + train_data = TableHandle("train", path=cleaned_train_file) + test_data = TableHandle("test", path=str(test_file)) + informer = _make_knn_informer() + model = informer.inform(train_data) + estimator = _make_knn_estimator(model) + pz_out = estimator.estimate(test_data) + _attach_object_ids(pz_out, test_data) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_1_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_estimation_only(model_file, test_file, output_file) + + +def run_taskset_1_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_training_and_estimation(train_file, test_file, output_file) + + +def run_taskset_2_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_estimation_only(model_file, test_file, output_file) + + +def run_taskset_2_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_training_and_estimation(train_file, test_file, output_file) + + +def run_taskset_3_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_estimation_only(model_file, test_file, output_file) + + +def run_taskset_3_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_training_and_estimation(train_file, test_file, output_file) + + +def run_taskset_4_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_estimation_only(model_file, test_file, output_file) + + +def run_taskset_4_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + run_taskset_x_training_and_estimation(train_file, test_file, output_file) + + +def test_example_taskset_1( + setup_public_area: int, + setup_submit_area: int, +) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_1( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_1_estimation_only, + run_taskset_1_training_and_estimation, + ) + + +def test_example_taskset_2( + setup_public_area: int, + setup_submit_area: int, +) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_2( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_2_estimation_only, + run_taskset_2_training_and_estimation, + ) + + +def test_example_taskset_3( + setup_public_area: int, + setup_submit_area: int, +) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_3( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_3_estimation_only, + run_taskset_3_training_and_estimation, + ) + + +def test_example_taskset_4( + setup_public_area: int, + setup_submit_area: int, +) -> None: + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_4( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_4_estimation_only, + run_taskset_4_training_and_estimation, + )