From c256cf5f08fb7027ced3d6a4187827398c207065 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 9 Apr 2026 22:36:48 -0700 Subject: [PATCH 1/6] Submitting example --- .github/workflows/submit_example.yaml | 38 ++++ requirements_example.txt | 2 + tests/test_example.py | 251 ++++++++++++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 .github/workflows/submit_example.yaml create mode 100644 requirements_example.txt create mode 100644 tests/test_example.py diff --git a/.github/workflows/submit_example.yaml b/.github/workflows/submit_example.yaml new file mode 100644 index 0000000..fd2bcb2 --- /dev/null +++ b/.github/workflows/submit_example.yaml @@ -0,0 +1,38 @@ +--- +# This workflow will install Python dependencies and run tests + +name: Unit test and code coverage + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.13'] + submission: ['example'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt install libbz2-dev + python -m pip install --upgrade pip + pip install wheel + pip install . + pip install .[dev] + if [ -f requirements_${{ matrix.submission }}.txt ]; then pip install -r requirements_${{ matrix.submission }}.txt; fi + - name: Run unit tests with pytest + run: | + python -m pytest tests/test_${{ matrix.submission }}.py diff --git a/requirements_example.txt b/requirements_example.txt new file mode 100644 index 0000000..2681716 --- /dev/null +++ b/requirements_example.txt @@ -0,0 +1,2 @@ +pz-rail-base +pz-rail-sklearn diff --git a/tests/test_example.py b/tests/test_example.py new file mode 100644 index 0000000..82c46cc --- /dev/null +++ b/tests/test_example.py @@ -0,0 +1,251 @@ +import os +from pathlib import Path +import pytest + +from rail.core.data import TableHandle +from rail.estimation.algos import sklearn_neurnet +from rail.utils import catalog_utils + +from pz_data_challenge.taskset_1 import run_taskset_1 +from pz_data_challenge.taskset_2 import run_taskset_2 + +from pz_data_challenge import submit_utils + +SUBMISSION_NAME: str = "example" +SUBMISSION_URL: str = "https://s3df.slac.stanford.edu/people/echarles/submit_example.tgz" + +# don't change these +SUBMIT_DIR: str = f"submissions/{SUBMISSION_NAME}" +PUBLIC_AREA: str = "tests/public" + + +@pytest.fixture(name="setup_submit_area", scope="module") +def setup_submit_area(request: pytest.FixtureRequest) -> int: + + if not os.path.exists(SUBMIT_DIR): + submit_utils.download_and_extract_tar(SUBMISSION_URL, SUBMIT_DIR) + + def teardown_submit_area() -> None: + if not os.environ.get("NO_TEARDOWN"): + os.system(f"\\rm -rf {SUBMIT_DIR}") + + try: + os.makedirs(os.path.join(SUBMIT_DIR, "outputs_2")) + except Exception: + pass + + try: + os.makedirs(os.path.join(SUBMIT_DIR, "outputs_3")) + except Exception: + pass + + request.addfinalizer(teardown_submit_area) + + catalog_utils.load_yaml("tests/catalogs.yaml") + catalog_utils.apply("cardinal_roman_rubin") + + return 0 + + +def run_taskset_1_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run estimation for task set 1 + + This function should use a model stored in model_file, which + is downloaded as part of the submission tar file. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + model_file: + Path to the model. This should be part of the submission + tar file. + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + test_data = TableHandle("test", path=test_file) + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model_file, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_1_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run training and estimation for task set 1 + + This function should train a model and use it. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + train_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be trained + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + train_data = TableHandle("train", path=train_file) + test_data = TableHandle("test", path=test_file) + + informer = sklearn_neurnet.SklNeurNetInformer.make_stage( + name="inform", + ) + model = informer.inform(train_data) + + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_2_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run estimation for task set 1 + + This function should use a model stored in model_file, which + is downloaded as part of the submission tar file. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + model_file: + Path to the model. This should be part of the submission + tar file. + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + test_data = TableHandle("test", path=test_file) + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model_file, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_2_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run training and estimation for task set 1 + + This function should train a model and use it. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + train_data = TableHandle("train", path=train_file) + test_data = TableHandle("test", path=test_file) + + informer = sklearn_neurnet.SklNeurNetInformer.make_stage( + name="inform", + ) + model = informer.inform(train_data) + + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def test_example_taskset_1( + setup_public_area: int, + setup_submit_area: int, +) -> None: + """ + Test fuction to validate a submisson for Taskset 1 + + You should not need to change this function + """ + + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_1( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_1_estimation_only, + run_taskset_1_training_and_estimation, + ) + + +def test_example_taskset_2( + setup_public_area: int, + setup_submit_area: int, +) -> None: + """ + Test fuction to validate a submisson for Taskset 1 + + You should not need to change this function + """ + + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_2( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_2_estimation_only, + run_taskset_2_training_and_estimation, + ) + From 5b6897e6a1dbb68c169de70f967f89ee4219c1b3 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 4 Jun 2026 18:07:55 -0700 Subject: [PATCH 2/6] updates for tasksets 3 and 4 --- examples/do_example.py | 17 ++- scripts/download_public.py | 2 +- tests/conftest.py | 2 +- tests/test_example.py | 222 ++++++++++++++++++++++++++++++++++++ tests/test_taskset_paths.py | 4 +- 5 files changed, 240 insertions(+), 7 deletions(-) diff --git a/examples/do_example.py b/examples/do_example.py index 9c7e071..dce7488 100644 --- a/examples/do_example.py +++ b/examples/do_example.py @@ -1,3 +1,5 @@ +import tables_io +import numpy as np from rail.core.data import TableHandle from rail.estimation.algos import sklearn_neurnet from rail.utils import catalog_utils @@ -17,8 +19,17 @@ def train_and_estimate( scenario: str, ) -> None: + # clean up the training data + train_data_path = f"public/pz_challenge_{taskset}_{sim}_training_{scenario}.hdf5" + uncleaned_training_data = tables_io.read(train_data_path) + bad_mask = np.isnan(uncleaned_training_data['redshift']) + if bad_mask.any(): + cleaned_training_data = {key: val[~bad_mask] for key, val in uncleaned_training_data.items()} + train_data_path = train_data_path.replace('.hdf5', '_cleaned.hdf5') + tables_io.write(cleaned_training_data, train_data_path) + train_data = TableHandle( - "train", path=f"public/pz_challenge_{taskset}_{sim}_training_{scenario}.hdf5" + "train", path=train_data_path, ) test_data = TableHandle( "test", path=f"public/pz_challenge_{taskset}_{sim}_test_{scenario}.hdf5" @@ -33,7 +44,7 @@ def train_and_estimate( os.makedirs("evaluation") except: pass - + model_path = f"submission/pz_challenge_{taskset}_{sim}_pz_model_{scenario}.pkl" output_path = f"submission/pz_challenge_{taskset}_{sim}_pz_estimate_{scenario}.hdf5" evaluate_path = f"evaluation/pz_challenge_{taskset}_{sim}_pz_evaluation_{scenario}.hdf5" @@ -66,7 +77,7 @@ def train_and_estimate( pz_evaluate.data.ancil["object_id"] = train_data()["object_id"].astype(int) pz_evaluate.path = evaluate_path pz_evaluate.write() - + if __name__ == "__main__": diff --git a/scripts/download_public.py b/scripts/download_public.py index 0a81aa2..7ef79cd 100644 --- a/scripts/download_public.py +++ b/scripts/download_public.py @@ -2,7 +2,7 @@ from pz_data_challenge import submit_utils # don't change these -PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public.tgz" +PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" def setup_public_area() -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 82b72e0..87ca168 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ from pz_data_challenge import submit_utils # don't change these -PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public.tgz" +PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" @pytest.fixture(name="setup_public_area", scope="package") diff --git a/tests/test_example.py b/tests/test_example.py index 82c46cc..b75733c 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -2,12 +2,17 @@ from pathlib import Path import pytest +import numpy as np +import tables_io + from rail.core.data import TableHandle from rail.estimation.algos import sklearn_neurnet from rail.utils import catalog_utils from pz_data_challenge.taskset_1 import run_taskset_1 from pz_data_challenge.taskset_2 import run_taskset_2 +from pz_data_challenge.taskset_3 import run_taskset_3 +from pz_data_challenge.taskset_4 import run_taskset_4 from pz_data_challenge import submit_utils @@ -208,6 +213,182 @@ def run_taskset_2_training_and_estimation( pz_out.write() +def run_taskset_3_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run estimation for task set 3 + + This function should use a model stored in model_file, which + is downloaded as part of the submission tar file. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + model_file: + Path to the model. This should be part of the submission + tar file. + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + test_data = TableHandle("test", path=test_file) + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model_file, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_3_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run training and estimation for task set 3 + + This function should train a model and use it. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + train_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be trained + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + uncleaned_training_data = tables_io.read(train_file) + bad_mask = np.isnan(uncleaned_training_data['redshift']) + if bad_mask.any(): + cleaned_training_data = {key: val[~bad_mask] for key, val in uncleaned_training_data.items()} + train_file = train_file.replace('.hdf5', '_cleaned.hdf5') + tables_io.write(cleaned_training_data, train_file) + + train_data = TableHandle("train", path=train_file) + test_data = TableHandle("test", path=test_file) + + informer = sklearn_neurnet.SklNeurNetInformer.make_stage( + name="inform", + ) + model = informer.inform(train_data) + + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_4_estimation_only( + model_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run estimation for task set 4 + + This function should use a model stored in model_file, which + is downloaded as part of the submission tar file. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + model_file: + Path to the model. This should be part of the submission + tar file. + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + test_data = TableHandle("test", path=test_file) + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model_file, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + +def run_taskset_4_training_and_estimation( + train_file: str | Path, + test_file: str | Path, + output_file: str | Path, +) -> None: + """ + User supplied function to run training and estimation for task set 4 + + This function should train a model and use it. + + This function should write output data to output_file in qp + format. + + Parameters + ---------- + test_file: + Path to the test file contains the photometric test data on + which the PZ estimation will be run + output_file: + Path to write the output data to. The output data should + be written in qp format. + """ + uncleaned_training_data = tables_io.read(train_file) + bad_mask = np.isnan(uncleaned_training_data['redshift']) + if bad_mask.any(): + cleaned_training_data = {key: val[~bad_mask] for key, val in uncleaned_training_data.items()} + train_file = train_file.replace('.hdf5', '_cleaned.hdf5') + tables_io.write(cleaned_training_data, train_file) + + train_data = TableHandle("train", path=train_file) + test_data = TableHandle("test", path=test_file) + + informer = sklearn_neurnet.SklNeurNetInformer.make_stage( + name="inform", + ) + model = informer.inform(train_data) + + estimator = sklearn_neurnet.SklNeurNetEstimator.make_stage( + name="estimate", + model=model, + output_mode="return", + ) + pz_out = estimator.estimate(test_data) + pz_out.data.ancil["object_id"] = test_data()["object_id"].astype(int) + pz_out.path = output_file + pz_out.write() + + + def test_example_taskset_1( setup_public_area: int, setup_submit_area: int, @@ -249,3 +430,44 @@ def test_example_taskset_2( run_taskset_2_training_and_estimation, ) + +def test_example_taskset_3( + setup_public_area: int, + setup_submit_area: int, +) -> None: + """ + Test fuction to validate a submisson for Taskset 3 + + You should not need to change this function + """ + + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_3( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_3_estimation_only, + run_taskset_3_training_and_estimation, + ) + + +def test_example_taskset_4( + setup_public_area: int, + setup_submit_area: int, +) -> None: + """ + Test fuction to validate a submisson for Taskset 4 + + You should not need to change this function + """ + + assert setup_public_area == 0 + assert setup_submit_area == 0 + + run_taskset_4( + PUBLIC_AREA, + SUBMISSION_NAME, + run_taskset_4_estimation_only, + run_taskset_4_training_and_estimation, + ) diff --git a/tests/test_taskset_paths.py b/tests/test_taskset_paths.py index 00e7456..b5bfcd7 100644 --- a/tests/test_taskset_paths.py +++ b/tests/test_taskset_paths.py @@ -50,7 +50,7 @@ def training_and_estimation(training_file, test_file, output_file) -> None: runner( "public", - "submission", + ".", estimation_only, training_and_estimation, ) @@ -75,4 +75,4 @@ def test_taskset_2_builds_only_taskset_2_paths( assert filenames assert all(name.startswith("pz_challenge_taskset_2_") for name in filenames) - assert all("taskset_1" not in name for name in filenames) \ No newline at end of file + assert all("taskset_1" not in name for name in filenames) From 1ed6e12e5e9ed108154c43a034579f3405c0f786 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 4 Jun 2026 18:39:02 -0700 Subject: [PATCH 3/6] move download area --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87ca168..f1e85d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,8 @@ from pz_data_challenge import submit_utils # don't change these -PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" +#PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" +PUBLIC_URL: str = "https://s3df.slac.stanford.edu/people/echarles/public_test.tgz" @pytest.fixture(name="setup_public_area", scope="package") From bd5b2ce0128d08e08235532368d285c487293470 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 11 Jun 2026 14:26:17 -0700 Subject: [PATCH 4/6] fix url --- tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f1e85d5..87ca168 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,7 @@ from pz_data_challenge import submit_utils # don't change these -#PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" -PUBLIC_URL: str = "https://s3df.slac.stanford.edu/people/echarles/public_test.tgz" +PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" @pytest.fixture(name="setup_public_area", scope="package") From fa28cada5095f82c34a3f0e16db9ed3a7049f728 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 30 Jun 2026 17:15:27 -0700 Subject: [PATCH 5/6] fix public url --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87ca168..82b72e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ from pz_data_challenge import submit_utils # don't change these -PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public_test.tgz" +PUBLIC_URL: str = "https://portal.nersc.gov/cfs/lsst/PZ/data_challenge/public.tgz" @pytest.fixture(name="setup_public_area", scope="package") From e0bce927cdb11126ac116334efc652a2ba658098 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 7 Jul 2026 14:27:07 -0700 Subject: [PATCH 6/6] removed extract workflow --- .github/workflows/submit_pz_resnet_flow.yaml | 38 -------------------- 1 file changed, 38 deletions(-) delete mode 100644 .github/workflows/submit_pz_resnet_flow.yaml diff --git a/.github/workflows/submit_pz_resnet_flow.yaml b/.github/workflows/submit_pz_resnet_flow.yaml deleted file mode 100644 index 6f06808..0000000 --- a/.github/workflows/submit_pz_resnet_flow.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# This workflow will install Python dependencies and run tests - -name: Unit test and code coverage - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.13'] - submission: ['pz_resnet_flow'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt install libbz2-dev - python -m pip install --upgrade pip - pip install wheel - pip install . - pip install .[dev] - if [ -f requirements_${{ matrix.submission }}.txt ]; then pip install -r requirements_${{ matrix.submission }}.txt; fi - - name: Run unit tests with pytest - run: | - python -m pytest tests/test_${{ matrix.submission }}.py