From f842e0b4f5f4dd9ee494d54ec6850805b739d274 Mon Sep 17 00:00:00 2001 From: Adrien La Posta Date: Thu, 19 Feb 2026 14:39:32 +0000 Subject: [PATCH 01/11] adapt to new sotodlib --- pipeline/filtering/filter_sims_sotodlib.py | 32 ++++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index 9b07edb..edd35e3 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -257,17 +257,25 @@ def main(args): # Process data here to have t2p leakage template # Only need to run it once for all simulations # and only the pre-demodulation part. - if args.t2p_template: - data_aman = pp_util.multilayer_load_and_preprocess( - obs_id, - configs_init, - configs_proc, - meta=meta, - logger=logger, - init_only=True, - ) - else: - data_aman = None + # if args.t2p_template: + # data_aman = pp_util.multilayer_load_and_preprocess( + # obs_id, + # configs_init, + # configs_proc, + # meta=meta, + # logger=logger, + # init_only=True, + # ) + # else: + # data_aman = None + data_aman = pp_util.multilayer_load_and_preprocess( + obs_id, + configs_init, + configs_proc, + meta=meta, + logger=logger, + stop_for_sims=True + ) for sim_id, sim_type in product(sim_ids, args.sim_types): @@ -301,7 +309,7 @@ def main(args): sim_map=sim, meta=meta, logger=logger, - t2ptemplate_aman=data_aman + data_amans=data_aman ) except loader.LoaderError: logger.warning( From 63d2abfaa73531fb7523b31b4672b0528c1007af Mon Sep 17 00:00:00 2001 From: Adrien La Posta Date: Fri, 20 Feb 2026 15:50:31 +0000 Subject: [PATCH 02/11] fix cfg check --- pipeline/filtering/filter_sims_sotodlib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index edd35e3..f86c47c 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -274,7 +274,8 @@ def main(args): configs_proc, meta=meta, logger=logger, - stop_for_sims=True + stop_for_sims=True, + ignore_cfg_check=True ) for sim_id, sim_type in product(sim_ids, args.sim_types): @@ -309,6 +310,7 @@ def main(args): sim_map=sim, meta=meta, logger=logger, + ignore_cfg_check=True, data_amans=data_aman ) except loader.LoaderError: From 318db59d5417c87065aba7d92d8d2a0da7ec23f2 Mon Sep 17 00:00:00 2001 From: kwolz Date: Wed, 4 Mar 2026 06:20:34 -0500 Subject: [PATCH 03/11] catch zero det exception in data_aman --- pipeline/filtering/filter_sims_sotodlib.py | 39 +++++++++------------- pipeline/filtering/filtering_utils.py | 9 ++--- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index f86c47c..6325704 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -39,7 +39,7 @@ def main(args): rank, size, comm = mpi.init(True) # Initialize the logger - logger = pp_util.init_logger("benchmark", verbosity=1) + logger = pp_util.init_logger("benchmark", verbosity=2) if rank == 0: start = time.time() @@ -254,29 +254,20 @@ def main(args): ] meta.restrict("dets", thinned) - # Process data here to have t2p leakage template - # Only need to run it once for all simulations - # and only the pre-demodulation part. - # if args.t2p_template: - # data_aman = pp_util.multilayer_load_and_preprocess( - # obs_id, - # configs_init, - # configs_proc, - # meta=meta, - # logger=logger, - # init_only=True, - # ) - # else: - # data_aman = None - data_aman = pp_util.multilayer_load_and_preprocess( - obs_id, - configs_init, - configs_proc, - meta=meta, - logger=logger, - stop_for_sims=True, - ignore_cfg_check=True - ) + try: + data_aman = pp_util.multilayer_load_and_preprocess( + obs_id, + configs_init, + configs_proc, + meta=meta, + logger=logger, + stop_for_sims=True, + ignore_cfg_check=True, + ) + except loader.LoaderError: + logger.warning(f"NO METADATA IN DATA_AMAN: " + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + continue for sim_id, sim_type in product(sim_ids, args.sim_types): diff --git a/pipeline/filtering/filtering_utils.py b/pipeline/filtering/filtering_utils.py index 49e6a13..00a5020 100644 --- a/pipeline/filtering/filtering_utils.py +++ b/pipeline/filtering/filtering_utils.py @@ -211,7 +211,8 @@ def make_map_wrapper(obs, split_labels, pix_type="hp", shape=None, wcs=None, elif hasattr(obs.preprocess.noiseQ_mapmaking, "white_noise"): inv_var = 1 / obs.preprocess.noiseQ_mapmaking.white_noise ** 2 else: - raise ValueError("obs.preprocess.noiseQ_mapmaking does not have either a std or white_noise") + raise ValueError("obs.preprocess.noiseQ_mapmaking does not have" + " either a std or white_noise") elif hasattr(obs.preprocess, "noiseQ_nofit"): # ISO v1 inv_var = 1 / obs.preprocess.noiseQ_nofit.white_noise ** 2 else: @@ -307,15 +308,15 @@ class Cfg: sim_dir: str atomic_sim_dir: str output_dir: str - sim_ids: list sim_string_format: str - sim_types: list freq_channels: list patches: list - intra_obs_splits: list query_restrict: Optional[str] = "" pix_type: Optional[str] = "car" bundle_id: Optional[int] = 0 + intra_obs_splits: Optional[list] = None + sim_ids: Optional[list] = None + sim_types: Optional[list] = None intra_obs_pair: Optional[list] = None inter_obs_splits: Optional[list] = None car_map_template: Optional[str] = None From f9027962d10782c9a4477a24c8c64631bc746894 Mon Sep 17 00:00:00 2001 From: Arianna Rizzieri Date: Wed, 4 Mar 2026 10:16:07 -0500 Subject: [PATCH 04/11] updated ext data filtering to do t2p subtraction --- pipeline/filtering/filter_ext_sotodlib.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pipeline/filtering/filter_ext_sotodlib.py b/pipeline/filtering/filter_ext_sotodlib.py index 62b70ec..f775a5a 100755 --- a/pipeline/filtering/filter_ext_sotodlib.py +++ b/pipeline/filtering/filter_ext_sotodlib.py @@ -11,7 +11,6 @@ import time from itertools import product -sys.path.insert(0, "/home/ar3186/.local/soconda_v0.6.15/lib/python3.12/site-packages") import sotodlib.preprocess.preprocess_util as pp_util from sotodlib.core.metadata import loader @@ -45,7 +44,7 @@ def main(args): rank, size, comm = mpi.init(True) # Initialize the logger - logger = pp_util.init_logger("benchmark", verbosity=3) + logger = pp_util.init_logger("benchmark", verbosity=2) if rank == 0: start = time.time() @@ -249,20 +248,21 @@ def main(args): ] meta.restrict("dets", thinned) - # Process data here to have t2p leakage template - # Only need to run it once for all simulations - # and only the pre-demodulation part. - if args.t2p_template: + try: + print(pp_util.__file__) data_aman = pp_util.multilayer_load_and_preprocess( obs_id, configs_init, configs_proc, meta=meta, logger=logger, - init_only=True, + stop_for_sims=True, + ignore_cfg_check=True, ) - else: - data_aman = None + except loader.LoaderError: + logger.warning(f"NO METADATA IN DATA_AMAN: " + f"({patch}, {freq_channel}, {obs_id}, {wafer})") + continue for isim_type, sim_type in enumerate(args.sim_types): @@ -294,7 +294,8 @@ def main(args): sim_map=sim, meta=meta, logger=logger, - t2ptemplate_aman=data_aman + ignore_cfg_check=True, + data_amans=data_aman ) except loader.LoaderError: logger.warning( From 5a5ef786d074947305210344ef6d8bac77b36b47 Mon Sep 17 00:00:00 2001 From: kwolz Date: Fri, 6 Mar 2026 03:13:02 -0500 Subject: [PATCH 05/11] bug fix in coadder; better logging --- pipeline/filtering/coadd_filtered_sims.py | 4 ++-- pipeline/filtering/filter_sims_sotodlib.py | 23 ++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pipeline/filtering/coadd_filtered_sims.py b/pipeline/filtering/coadd_filtered_sims.py index 738326c..97a8b70 100644 --- a/pipeline/filtering/coadd_filtered_sims.py +++ b/pipeline/filtering/coadd_filtered_sims.py @@ -305,6 +305,7 @@ def main(args): sim_id=sim_id ) assert os.path.isdir(map_dir), map_dir + tracemalloc.start() if not ib: logger.info(f"Loading atomics for ({patch}, {freq_channel}, " @@ -314,7 +315,6 @@ def main(args): w_list, wmap_list = ([], []) if split_label == "science": - tracemalloc.start() for coadd in intra_obs_pair: wmap_l, w_l = fu.get_atomics_maps_list( sim_id, sim_type, @@ -329,7 +329,6 @@ def main(args): for c in tracemalloc.get_traced_memory()] logger.info("Traced Memory for 'science' (Current, Peak): " f"{current_gb:.2f} GB, {peak_gb:.2f} GB") - tracemalloc.stop() elif split_label in inter_obs_splits: for coadd in intra_obs_pair: wmap_l, w_l = fu.get_atomics_maps_list( @@ -381,6 +380,7 @@ def main(args): plot_dirs[patch, freq_channel], pix_type=pix_type, do_plot=False ) + tracemalloc.stop() comm.Barrier() if rank == 0: end = time.time() diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index 6325704..352c034 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -52,7 +52,8 @@ def main(args): preprocess_config_init = args.preprocess_config_init preprocess_config_proc = args.preprocess_config_proc - logger.debug(f"Using atomic DB from {atom_db}") + if rank == 0: + logger.debug(f"Using atomic DB from {atom_db}") # Sim related arguments sim_dir = args.sim_dir @@ -66,7 +67,8 @@ def main(args): sim_ids = np.array([int(sim_ids)]) else: raise ValueError("Argument 'sim_ids' has the wrong format") - logger.debug(f"Processing sim_ids {sim_ids} in parallel.") + if rank == 0: + logger.debug(f"Processing sim_ids {sim_ids} in parallel.") # Ensure that freq_channels for the metadata follow the "f090" convention. # We keep the original labels in a dict called freq_labels. @@ -120,7 +122,8 @@ def main(args): ctimes = {patch: None for patch in args.patches} for patch in args.patches: if os.path.isfile(bundle_dbs[patch]): - logger.info(f"Loading from {bundle_dbs[patch]}.") + if rank == 0: + logger.info(f"Loading from {bundle_dbs[patch]}.") bundle_coordinator = coord.BundleCoordinator.from_dbfile( bundle_dbs[patch], bundle_id=bundle_id ) @@ -130,7 +133,6 @@ def main(args): # Extract all ctimes for the given bundle_id ctimes[patch] = bundle_coordinator.get_ctimes(bundle_id=bundle_id) - # TODO: check if query_restrict is channel- or patch-specific query_restrict = args.query_restrict queries = { (patch, freq_channel): fu.get_query_atomics( @@ -164,10 +166,11 @@ def main(args): atomic_metadata[split_label] += [ (patch, freq_channel, obs_id, wafer) ] - logger.info( - f"{patch}, {freq_channel}, 'science': " - f"{len(res_science)} atomic maps to filter." - ) + if rank == 0: + logger.info( + f"{patch}, {freq_channel}, 'science': " + f"{len(res_science)} atomic maps to filter." + ) # Load preprocessing pipeline and extract from it list of preprocessing # metadata (detectors, samples, etc.) corresponding to each atomic map @@ -320,13 +323,13 @@ def main(args): continue if aman is None: - logger.warrning( + logger.warning( "No detectors left in this atomic." f"({patch}, {freq_channel}, {obs_id}, {wafer}) " ) continue if aman.dets.count <= 1: - logger.warrning( + logger.warning( "No detectors left in this atomic." f"({patch}, {freq_channel}, {obs_id}, {wafer}) " ) From 6412524f17ed06e0d764f897b066cecbaf0c4e94 Mon Sep 17 00:00:00 2001 From: Arianna Rizzieri Date: Tue, 10 Mar 2026 14:33:22 -0400 Subject: [PATCH 06/11] import config files from main --- .../external-data/filtering_configs/README.md | 9 + ...ics_planck_bundle0_w_t2p_subtraction.slurm | 32 ++ ...ics_planck_bundle1_w_t2p_subtraction.slurm | 32 ++ ...omics_wmap_bundle0_w_t2p_subtraction.slurm | 32 ++ ...omics_wmap_bundle1_w_t2p_subtraction.slurm | 30 ++ ...nfig_planck_bundle0_w_t2p_subtraction.yaml | 34 ++ ...nfig_planck_bundle1_w_t2p_subtraction.yaml | 34 ++ ...config_wmap_bundle0_w_t2p_subtraction.yaml | 34 ++ ...config_wmap_bundle1_w_t2p_subtraction.yaml | 34 ++ ...ing_planck_bundle0_w_t2p_subtraction.slurm | 33 ++ ...ing_planck_bundle1_w_t2p_subtraction.slurm | 33 ++ ...ering_wmap_bundle0_w_t2p_subtraction.slurm | 33 ++ ...ering_wmap_bundle1_w_t2p_subtraction.slurm | 33 ++ ...onfig_20251216_init_w_t2p_subtraction.yaml | 423 ++++++++++++++++++ ...onfig_20251216_proc_w_t2p_subtraction.yaml | 398 ++++++++++++++++ ...ics_planck_bundle0_w_t2p_subtraction.slurm | 32 ++ ...ics_planck_bundle1_w_t2p_subtraction.slurm | 32 ++ ...omics_wmap_bundle0_w_t2p_subtraction.slurm | 32 ++ ...omics_wmap_bundle1_w_t2p_subtraction.slurm | 32 ++ ...nfig_planck_bundle0_w_t2p_subtraction.yaml | 34 ++ ...nfig_planck_bundle1_w_t2p_subtraction.yaml | 34 ++ ...config_wmap_bundle0_w_t2p_subtraction.yaml | 34 ++ ...config_wmap_bundle1_w_t2p_subtraction.yaml | 34 ++ ...ing_planck_bundle0_w_t2p_subtraction.slurm | 33 ++ ...ing_planck_bundle1_w_t2p_subtraction.slurm | 33 ++ ...ering_wmap_bundle0_w_t2p_subtraction.slurm | 33 ++ ...ering_wmap_bundle1_w_t2p_subtraction.slurm | 33 ++ ...onfig_20251216_init_w_t2p_subtraction.yaml | 417 +++++++++++++++++ ...onfig_20251216_proc_w_t2p_subtraction.yaml | 398 ++++++++++++++++ ..._science_filtered_satp1_f090_20260304.yaml | 171 +++++++ ..._science_filtered_satp1_f150_20260304.yaml | 171 +++++++ .../satp1/spectra_computation_steps.sh | 68 ++- ...ile_south_science_filtered_satp3_f090.yaml | 44 +- ..._science_filtered_satp3_f090_20260304.yaml | 171 +++++++ ...ile_south_science_filtered_satp3_f150.yaml | 44 +- .../satp3/spectra_computation_steps.sh | 69 ++- 36 files changed, 3130 insertions(+), 43 deletions(-) create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm create mode 100755 towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle0_w_t2p_subtraction.yaml create mode 100755 towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle1_w_t2p_subtraction.yaml create mode 100755 towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml create mode 100755 towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_init_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle0_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle1_w_t2p_subtraction.slurm create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_init_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml create mode 100644 towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f090_20260304.yaml create mode 100644 towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f150_20260304.yaml create mode 100644 towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090_20260304.yaml diff --git a/towards_r/v4/external-data/filtering_configs/README.md b/towards_r/v4/external-data/filtering_configs/README.md index b16d133..5a87277 100644 --- a/towards_r/v4/external-data/filtering_configs/README.md +++ b/towards_r/v4/external-data/filtering_configs/README.md @@ -1,5 +1,14 @@ Run with: + +1st run (20260220): no t2p template subtraction and used bundle_id = 1 for the filtering when using ext data bundle (hm) 1 - loaded soconda evironments (soconda/3.12/v0.6.15): module use --append /scratch/gpfs/SIMONSOBS/modules module load soconda/3.12/v0.6.15 - on top of that installed: [sotodlib](https://github.com/simonsobs/sotodlib/tree/0a061ce9804f2b80da9e52137323f79ae34f7b1c) +Resulting maps: /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260220 + +2nd run (20260304): with t2p template subtraction and always used bundle_id = 0 for the filtering +- loaded latest soconda evironment (soconda/3.12/v0.6.16): +- on top of that installed: [sotodlib](https://github.com/adrien-laposta/sotodlib/tree/e23958880a15ad305afd7b3d4d8f3033a3350173) +- used bb-awg-scripts branch: https://github.com/simonsobs/bb-awg-scripts/tree/f9027962d10782c9a4477a24c8c64631bc746894 +Resulting maps: /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304 diff --git a/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..67bc062 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=2 +#SBATCH --ntasks=22 +#SBATCH --cpus-per-task=10 +#SBATCH --time=8:00:00 +#SBATCH --job-name=planck-0-satp1-ext-coadd-w-t2p-subtraction + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp1" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=10 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 22 -c 10 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..bffa09b --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=planck-1-satp1-ext-coadd-w-t2p-subtraction + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp1" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..842d6e9 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=2 +#SBATCH --ntasks=22 +#SBATCH --cpus-per-task=10 +#SBATCH --time=8:00:00 +#SBATCH --job-name=wmap-0-satp1-ext-coadd-w-t2p-subtraction + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp1" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=10 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 22 -c 10 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..6bacd83 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,30 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=2 +#SBATCH --ntasks=22 +#SBATCH --cpus-per-task=10 +#SBATCH --time=8:00:00 +#SBATCH --job-name=wmap-1-satp1-ext-coadd-w-t2p-subtraction + +set -e + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp1" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=10 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 22 -c 10 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle0_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle0_w_t2p_subtraction.yaml new file mode 100755 index 0000000..e5bc18f --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle0_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp1_20251216_full-dataset/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp1_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp1/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle1_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle1_w_t2p_subtraction.yaml new file mode 100755 index 0000000..dd09fe5 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_planck_bundle1_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp1_20251216_full-dataset/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp1_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp1/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml new file mode 100755 index 0000000..6fa4831 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp1_20251216_full-dataset/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp1_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/wmap/f{sim_type}/satp1/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml new file mode 100755 index 0000000..20f5c66 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp1_20251216_full-dataset/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp1_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/wmap/f{sim_type}/satp1/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..19c01b2 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=15:00:00 +#SBATCH --job-name=planck-satp1-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp1" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..6ce1323 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_planck_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=15:00:00 +#SBATCH --job-name=planck-satp1-ext-filtering-bundle1 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp1" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..9912c25 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=15:00:00 +#SBATCH --job-name=wmap-satp1-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp1" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..0029c8b --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/filtering_wmap_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=15:00:00 +#SBATCH --job-name=wmap-satp1-ext-filtering-bundle1 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp1" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_init_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_init_w_t2p_subtraction.yaml new file mode 100644 index 0000000..ce4c5c8 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_init_w_t2p_subtraction.yaml @@ -0,0 +1,423 @@ +context_file: 'use_this_local_260115.yaml' +plot_dir: '/scratch/gpfs/SIMONSOBS/external_data_e2e/v4/preprocessing/satp1_20251216_init/preprocess_plots' +jobdb: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_jobs.db' + +subobs: + use: ['wafer_slot', 'wafer.bandpass'] + label: wafer_slot + +archive: + index: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_init/process_archive.sqlite' + policy: + type: 'simple' + filename: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_init/preprocess_archive.h5' + +process_pipe: + - name: "move" + skip_on_sim: True + process: + name: "det_info.wiregrid_cal.gamma" + new_name: "focal_plane.gamma" + + - name: "pointing_model" + skip_on_sim: True + process: True + + - name: "hwp_angle_model" + skip_on_sim: True + process: True + calc: + on_sign_ambiguous: 'fail' + save: True + + - name: "correct_iir_params" + skip_on_sim: False + process: True + + - name: "smurfgaps_flags" + skip_on_sim: True + calc: + buffer: 200 + name: "smurfgaps" + merge: True + save: True + + - name: "dark_dets" + skip_on_sim: False + signal: "signal" + calc: True + save: True + select: True + + - name : "fp_flags" + skip_on_sim: False + calc: + merge: False + save: True + select: True + + - name: "detcal_nan_cuts" + skip_on_sim: False + select: + fields: ['tau_eff', 'phase_to_pW', 'r_n', 'r_frac', 'p_sat'] + + - name: "flag_turnarounds" + skip_on_sim: False + process: + method: "scanspeed" + t_buffer: 4. + calc: + method: "scanspeed" + t_buffer: 4. + save: True + + - name: "det_bias_flags" + skip_on_sim: False + calc: + rfrac_range: [0.05, 0.95] + psat_range: [1.0e-6, 20.0] + rn_range: [0.0, 0.1] + si_range: [-1.0e+9, -1.0e+1] + save: True + select: True + + - name: "trends" + skip_on_sim: False + calc: + max_trend: 2.5 + t_piece: 10 + save: True + select: + kind: "any" + + - name: "jumps" + skip_on_sim: False + calc: + function: "slow_jumps" + jump_configs: + win_size: 800 + thresh: 20.0 + save: + jumps_name: "jumps_slow" + select: + max_n_jumps: 5 + + - name: "fix_jumps" + skip_on_sim: True + process: + jumps_aman: "jumps_slow" + + - name: "glitchfill" + skip_on_sim: True + flags: "jumps_slow.jump_flag" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "jumps" + skip_on_sim: False + calc: + function: "twopi_jumps" + jump_configs: + win_size: 30 + nsigma: 10.0 + save: + jumps_name: "jumps_2pi" + select: + max_n_jumps: 20 + + - name: "fix_jumps" + skip_on_sim: True + process: + jumps_aman: "jumps_2pi" + nbuf: 10 + use_pca: False + modes: 1 + + - name: "glitchfill" + skip_on_sim: True + flags: "jumps_2pi.jump_flag" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "detrend" + skip_on_sim: False + process: + method: "median" + + - name: "glitches" + skip_on_sim: False + glitch_name: "glitches_pre_hwpss" + calc: + t_glitch: 0.007 + buffer: 100 + hp_fc: 6.0 + n_sig: 30 + subscan: True + save: True + select: + max_n_glitch: 1000 + sig_glitch: 30 + + - name : "source_flags" + skip_on_sim: False + calc: + mask: {'shape': 'circle', + 'xyr': [0, 0, 20.]} + center_on: ['moon'] # list of str + res: 20 # arcmin + max_pix: 4000000 # max number of allowed pixels in map + distance: 20 # max distance of footprint from source in degrees + save: True + select: True # optional + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', + 'source_flags.moon', 'turnaround_flags.turnarounds','smurfgaps.smurfgaps'] + total_flags_label: 'pre_hwpss_flags' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['glitches_pre_hwpss.glitch_flags', 'source_flags.moon'] + total_flags_label: 'pre_glitchfill' + method: 'union' + + - name: "calibrate" + skip_on_sim: True + process: + kind: "array" + cal_array: "det_cal.phase_to_pW" + + - name: "estimate_hwpss" + skip_on_sim: True + calc: + hwpss_stats_name: "pre_hwpss_stats" + flags: "flags.pre_hwpss_flags" + save: True + + - name: "subtract_hwpss" + skip_on_sim: False + hwpss_stats: "pre_hwpss_stats" + process: + subtract_name: "signal" + + - name: "glitchfill" + skip_on_sim: True + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + glitch_flags: "pre_glitchfill" + + - name: "glitches" + skip_on_sim: False + glitch_name: "glitches_post_hwpss" + calc: + t_glitch: 0.007 + buffer: 100 + hp_fc: 6.0 + n_sig: 30 + subscan: True + save: True + select: + max_n_glitch: 100 + sig_glitch: 30 + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', + 'glitches_post_hwpss.glitch_flags', 'source_flags.moon', 'turnaround_flags.turnarounds','smurfgaps.smurfgaps'] + total_flags_label: 'glitch_flags' + method: 'union' + + - name: "estimate_hwpss" + skip_on_sim: True + calc: + hwpss_stats_name: "post_hwpss_stats" + flags: "flags.glitch_flags" + save: True + + - name: "subtract_hwpss" + skip_on_sim: False + hwpss_stats: "post_hwpss_stats" + process: + subtract_name: "signal" + + - name: "glitchfill" + skip_on_sim: True + flags: "glitches_post_hwpss.glitch_flags" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "detrend" + skip_on_sim: False + process: + method: "median" + + - name : "ptp_flags" + skip_on_sim: False + calc: + signal_name: "signal" + kurtosis_threshold: 1 + save: True + select: True + + - name: "psd" + skip_on_sim: True + signal: "signal" + wrap: "Pxx_raw" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: False # Don't skip for cuts. + fit: True + subscan: False + psd: "Pxx_raw" + calc: + fwhite: [5, 20] + lowf: 0.3 + f_max: 50 + mask: True + fixed_param: 'wn' + binning: True + fknee_est: 1.5 + alpha_est: 3.0 + merge_name: 'noise_fit_statsT' + bounds: + - [0.3, 20.0] # lower bound fknee, alpha + - [1.0, 6.0] # upper bound fknee, alpha + save: + wrap_name: "noiseT" + select: + min_noise: 18.0e-6 + max_noise: 125.0e-6 + max_fknee: 7.0 + + - name: "cut_bad_dist" + skip_on_sim: False + select: + param_name: "noiseT.white_noise" + + - name: "calibrate" + skip_on_sim: True + process: + kind: "array" + cal_array: "relcal.rel_factor" + select: + cut_array: "relcal.missing_factor" + + - name: "fourier_filter" + skip_on_sim: True + process: + filt_function: "iir_filter" + filter_params: + invert: True + + - name: "fourier_filter" + skip_on_sim: True + process: + filt_function: "timeconst_filter" + filter_params: + timeconst: "det_cal.tau_eff" + invert: True + + - name: "calibrate" + skip_on_sim: True + signal: "signal" + process: + kind: "array" + cal_array: "abscal.abscal_cmb" + + - name: "apodize" + skip_on_sim: False + process: + apodize_samps: 2000 + + - name: "demodulate" + skip_on_sim: False + process: + trim_samps: 6000 + demod_cfgs: + bpf_cfg: {'type': 'sine2', 'center': '4*f_HWP', 'width': '3.8*f_HWP', 'trans_width': 0.1} + lpf_cfg: {'type': 'sine2', 'cutoff': '1.9*f_HWP', 'trans_width': 0.1} + + - name : "tod_stats" + skip_on_sim: True + signal: "dsT" + wrap: "tod_stats_T" + calc: + stat_names: ["std", "ptp"] + split_subscans: True + save: True + + - name : "tod_stats" + skip_on_sim: True + signal: "demodQ" + wrap: "tod_stats_Q" + calc: + stat_names: ["median", "std", "skew", "kurtosis", "ptp"] + split_subscans: True + save: True + + - name : "tod_stats" + skip_on_sim: True + signal: "demodU" + wrap: "tod_stats_U" + calc: + stat_names: ["median", "std", "skew", "kurtosis", "ptp"] + split_subscans: True + save: True + + - name : "noisy_subscan_flags" + skip_on_sim: False # Keeping flags for sims + stats_name: tod_stats + calc: + subscan_stats: ["demodQ", "demodU", "dsT"] + nstd_lim: 3.0 + ptp_lim: 0.8 + kurt_lim: 0.5 + skew_lim: 0.5 + noisy_detector_lim: 0.5 + merge: False + save: True + select: True + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.left_scan', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags','smurfgaps.smurfgaps', 'turnaround_flags.turnarounds', 'source_flags.moon', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags_right' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.right_scan', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags','smurfgaps.smurfgaps', 'turnaround_flags.turnarounds', 'source_flags.moon', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags_left' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.turnarounds', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags', 'source_flags.moon','smurfgaps.smurfgaps', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags' + method: 'union' \ No newline at end of file diff --git a/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml new file mode 100644 index 0000000..8b33ae1 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp1/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml @@ -0,0 +1,398 @@ +context_file: 'use_this_local_260115.yaml' +plot_dir: '/scratch/gpfs/SIMONSOBS/external_data_e2e/v4/preprocessing/satp1_20251216_proc/preprocess_plots' +jobdb: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_jobs.db' + +subobs: + use: ['wafer_slot', 'wafer.bandpass'] + label: wafer_slot + +archive: + index: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_proc/process_archive.sqlite' + policy: + type: 'simple' + filename: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp1_20251216_proc/preprocess_archive.h5' + +process_pipe: + - name: "detrend" + skip_on_sim: False + signal: "dsT" + process: + method: "linear" + count: 2500 + + - name: "detrend" + skip_on_sim: False + signal: "demodQ" + process: + method: "linear" + count: 2500 + + - name: "detrend" + skip_on_sim: False + signal: "demodU" + process: + method: "linear" + count: 2500 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "dsT" + apodize_samps: 2000 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "demodQ" + apodize_samps: 2000 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "demodU" + apodize_samps: 2000 + + - name : "estimate_t2p" + skip_on_sim: True + fit_in_freq : True + calc: + T_sig_name: 'dsT' + Q_sig_name: 'demodQ' + U_sig_name: 'demodU' + fit_freq_range: (0.01, 0.1) + calc_and_save: True + save: True + select: {} + + - name : "subtract_t2p" + skip_on_sim: False + use_data_aman: True + process: + fit_in_freq: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'dsT' + azss_stats_name: 'azss_statsT_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'dsT' + azss: 'azss_statsT_left' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodQ' + azss_stats_name: 'azss_statsQ_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_left' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodQ' + azss: 'azss_statsQ_left' + method: 'interpolate' + scan_flags: 'left_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodU' + azss_stats_name: 'azss_statsU_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_left' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodU' + azss: 'azss_statsU_left' + method: 'interpolate' + scan_flags: 'left_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'dsT' + azss_stats_name: 'azss_statsT_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'dsT' + azss: 'azss_statsT_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodQ' + azss_stats_name: 'azss_statsQ_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodQ' + azss: 'azss_statsQ_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodU' + azss_stats_name: 'azss_statsU_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodU' + azss: 'azss_statsU_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "psd" + skip_on_sim: True + signal: "demodQ" + wrap: "psdQ" + process: + # max_samples: 524288 #2**19 + merge: False + nperseg: 200000 + noverlap: 0 + full_output: True + + - name: "psd" + skip_on_sim: True + signal: "demodU" + wrap: "psdU" + process: + # max_samples: 524288 #2**19 + merge: False + nperseg: 200000 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True + fit: True + subscan: False + psd: "psdQ" + calc: + fwhite: [0.5, 1.75] + lowf: 1.0e-4 + f_max: 1.9 + fknee_est: 1.0e-3 + alpha_est: 2.0 + fixed_param: 'wn' + binning: True + merge_name: 'noise_fit_statsQ' + bounds: + - [1.0e-4, 1.0] # lower bound fknee, alpha + - [0.1, 3.0] # upper bound fknee, alpha + fit_method: 'log_curve_fit' + curve_fit_kwargs: + maxfev: 20000 + save: + wrap_name: "noiseQ" + select: + require_finite_fit: True + + - name: "noise" + skip_on_sim: True + fit: True + subscan: False + psd: "psdU" + calc: + fwhite: [0.5, 1.75] + lowf: 1.0e-4 + f_max: 1.9 + fknee_est: 1.0e-3 + alpha_est: 2.0 + fixed_param: 'wn' + binning: True + merge_name: 'noise_fit_statsU' + bounds: + - [1.0e-4, 1.0] # lower bound fknee, alpha + - [0.1, 3.0] # upper bound fknee, alpha + fit_method: 'log_curve_fit' + curve_fit_kwargs: + maxfev: 20000 + save: + wrap_name: "noiseU" + select: + require_finite_fit: True + + - name: "fourier_filter" + skip_on_sim: False + signal_name: "demodQ" + wrap_name: "demodQ" + process: + filt_function: "counter_1_over_f" + noise_fit_array: "noiseQ" + filter_params: + each_detector: True + + - name: "fourier_filter" + skip_on_sim: False + signal_name: "demodU" + wrap_name: "demodU" + process: + filt_function: "counter_1_over_f" + noise_fit_array: "noiseU" + filter_params: + each_detector: True + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "dsT" + degree: 9 + in_place: True + mask: "glitch_flags" + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "demodQ" + degree: 1 + in_place: True + mask: "glitch_flags" + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "demodU" + degree: 1 + in_place: True + mask: "glitch_flags" + + - name : 'scan_freq_cut' + skip_on_sim: False + process: + signal_name_T: 'dsT' + signal_name_Q: 'demodQ' + signal_name_U: 'demodU' + + - name: "inv_var_flags" + skip_on_sim: False + calc: + signal_name: 'demodQ' + nsigma: 2 + save: True + select: True + + - name: "split_flags" + skip_on_sim: True + calc: + right_focal_plane: 0 + top_focal_plane: 0 + central_pixels: 0.20023039 + high_gain: 0.115 + high_noise: 3.5e-5 + high_tau: 1.5e-3 + det_A: A + pol_angle: 35 + crossover: BL + high_leakage: 1.0e-3 + high_2f: 1.5e-3 + save: True + + - name: "psd" + skip_on_sim: True # We won't need this from sims + signal: "demodQ" + wrap: "psdQ_mapmaking" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True # We won't need this from sims + psd: "psdQ_mapmaking" + fit: False + calc: + low_f: 1 + high_f: 2 + save: + wrap_name: "noiseQ_mapmaking" + + - name: "psd" + skip_on_sim: True # We won't need this from sims + signal: "demodU" + wrap: "psdU_mapmaking" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True # We won't need this from sims + psd: "psdU_mapmaking" + fit: False + calc: + low_f: 1 + high_f: 2 + save: + wrap_name: "noiseU_mapmaking" \ No newline at end of file diff --git a/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..02b3a94 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=planck-0-satp3-ext-coadd + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..2469b56 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=planck-1-satp3-ext-coadd + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..29029e9 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=2 +#SBATCH --ntasks=22 +#SBATCH --cpus-per-task=10 +#SBATCH --time=8:00:00 +#SBATCH --job-name=wmap-0-satp3-ext-coadd + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=10 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 22 -c 10 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..83da74f --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/coadd_atomics_wmap_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=2 +#SBATCH --ntasks=22 +#SBATCH --cpus-per-task=10 +#SBATCH --time=8:00:00 +#SBATCH --job-name=wmap-1-satp3-ext-coadd + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_w_t2p_subtraction" + +export OMP_NUM_THREADS=10 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 22 -c 10 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_w_t2p_subtraction.yaml new file mode 100644 index 0000000..c33f196 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_w_t2p_subtraction.yaml new file mode 100644 index 0000000..5f1494c --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml new file mode 100644 index 0000000..9a3a863 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/wmap/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: " pwv < 2 AND number_dets > 10 " + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml new file mode 100644 index 0000000..65b5721 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_config_wmap_bundle1_w_t2p_subtraction.yaml @@ -0,0 +1,34 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/wmap/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: " pwv < 2 AND number_dets > 10 " + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init_w_t2p_subtraction.yaml +preprocess_config_proc: preprocessing_config_20251216_proc_w_t2p_subtraction.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..10d20df --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=planck-satp3-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..1808d75 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_planck_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=planck-satp3-ext-filtering-bundle1 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle0_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle0_w_t2p_subtraction.slurm new file mode 100644 index 0000000..42a9497 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle0_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=wmap-satp3-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle1_w_t2p_subtraction.slurm b/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle1_w_t2p_subtraction.slurm new file mode 100644 index 0000000..71d55b3 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/filtering_wmap_bundle1_w_t2p_subtraction.slurm @@ -0,0 +1,33 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=wmap-satp3-ext-filtering-bundle1 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +export PYTHONPATH="${PYTHONPATH}:/home/ar3186/software/sotodlib_forks/" + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_w_t2p_subtraction" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_w_t2p_subtraction.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_init_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_init_w_t2p_subtraction.yaml new file mode 100644 index 0000000..7ec1bd8 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_init_w_t2p_subtraction.yaml @@ -0,0 +1,417 @@ +context_file: 'use_this_local_260115.yaml' +plot_dir: '/scratch/gpfs/SIMONSOBS/external_data_e2e/v4/preprocessing/satp3_20251216_init/preprocess_plots' +jobdb: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_jobdb.db' + +subobs: + use: ['wafer_slot', 'wafer.bandpass'] + label: wafer_slot + +archive: + index: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_init/process_archive.sqlite' + policy: + type: 'simple' + filename: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_init/preprocess_archive.h5' + +process_pipe: + - name: "pointing_model" + skip_on_sim: True + process: True + + - name: "hwp_angle_model" + skip_on_sim: True + process: True + calc: + on_sign_ambiguous: 'fail' + save: True + + - name: "correct_iir_params" + skip_on_sim: False + process: True + + - name: "smurfgaps_flags" + skip_on_sim: True + calc: + buffer: 200 + name: "smurfgaps" + merge: True + save: True + + - name: "dark_dets" + skip_on_sim: False + signal: "signal" + calc: True + save: True + select: True + + - name : "fp_flags" + skip_on_sim: False + calc: + merge: False + save: True + select: True + + - name: "detcal_nan_cuts" + skip_on_sim: False + select: + fields: ['tau_eff', 'phase_to_pW', 'r_n', 'r_frac', 'p_sat'] + + - name: "flag_turnarounds" + skip_on_sim: False + process: + method: "scanspeed" + t_buffer: 4. + calc: + method: "scanspeed" + t_buffer: 4. + save: True + + - name: "det_bias_flags" + skip_on_sim: False + calc: + rfrac_range: [0.05, 0.95] + psat_range: [1.0e-6, 20.0] + rn_range: [0.0, 0.1] + si_range: [-1.0e+9, -1.0e+1] + save: True + select: True + + - name: "trends" + skip_on_sim: False + calc: + max_trend: 2.5 + t_piece: 10 + save: True + select: + kind: "any" + + - name: "jumps" + skip_on_sim: False + calc: + function: "slow_jumps" + jump_configs: + win_size: 800 + thresh: 20.0 + save: + jumps_name: "jumps_slow" + select: + max_n_jumps: 5 + + - name: "fix_jumps" + skip_on_sim: True + process: + jumps_aman: "jumps_slow" + + - name: "glitchfill" + skip_on_sim: True + flags: "jumps_slow.jump_flag" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "jumps" + skip_on_sim: False + calc: + function: "twopi_jumps" + jump_configs: + win_size: 30 + nsigma: 10.0 + save: + jumps_name: "jumps_2pi" + select: + max_n_jumps: 20 + + - name: "fix_jumps" + skip_on_sim: True + process: + jumps_aman: "jumps_2pi" + nbuf: 10 + use_pca: False + modes: 1 + + - name: "glitchfill" + skip_on_sim: True + flags: "jumps_2pi.jump_flag" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "detrend" + skip_on_sim: False + process: + method: "median" + + - name: "glitches" + skip_on_sim: False + glitch_name: "glitches_pre_hwpss" + calc: + t_glitch: 0.007 + buffer: 100 + hp_fc: 6.0 + n_sig: 30 + subscan: True + save: True + select: + max_n_glitch: 1000 + sig_glitch: 30 + + - name : "source_flags" + skip_on_sim: False + calc: + mask: {'shape': 'circle', + 'xyr': [0, 0, 20.]} + center_on: ['moon'] # list of str + res: 20 # arcmin + max_pix: 4000000 # max number of allowed pixels in map + distance: 20 # max distance of footprint from source in degrees + save: True + select: True # optional + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', + 'source_flags.moon', 'turnaround_flags.turnarounds','smurfgaps.smurfgaps'] + total_flags_label: 'pre_hwpss_flags' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['glitches_pre_hwpss.glitch_flags', 'source_flags.moon'] + total_flags_label: 'pre_glitchfill' + method: 'union' + + - name: "calibrate" + skip_on_sim: True + process: + kind: "array" + cal_array: "det_cal.phase_to_pW" + + - name: "estimate_hwpss" + skip_on_sim: True + calc: + hwpss_stats_name: "pre_hwpss_stats" + flags: "flags.pre_hwpss_flags" + save: True + + - name: "subtract_hwpss" + skip_on_sim: False + hwpss_stats: "pre_hwpss_stats" + process: + subtract_name: "signal" + + - name: "glitchfill" + skip_on_sim: True + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + glitch_flags: "pre_glitchfill" + + - name: "glitches" + skip_on_sim: False + glitch_name: "glitches_post_hwpss" + calc: + t_glitch: 0.007 + buffer: 100 + hp_fc: 6.0 + n_sig: 30 + subscan: True + save: True + select: + max_n_glitch: 100 + sig_glitch: 30 + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', + 'glitches_post_hwpss.glitch_flags', 'source_flags.moon', 'turnaround_flags.turnarounds','smurfgaps.smurfgaps'] + total_flags_label: 'glitch_flags' + method: 'union' + + - name: "estimate_hwpss" + skip_on_sim: True + calc: + hwpss_stats_name: "post_hwpss_stats" + flags: "flags.glitch_flags" + save: True + + - name: "subtract_hwpss" + skip_on_sim: False + hwpss_stats: "post_hwpss_stats" + process: + subtract_name: "signal" + + - name: "glitchfill" + skip_on_sim: True + flags: "glitches_post_hwpss.glitch_flags" + process: + nbuf: 10 + use_pca: False + modes: 1 + wrap: "signal" + + - name: "detrend" + skip_on_sim: False + process: + method: "median" + + - name : "ptp_flags" + skip_on_sim: False + calc: + signal_name: "signal" + kurtosis_threshold: 1 + save: True + select: True + + - name: "psd" + skip_on_sim: True + signal: "signal" + wrap: "Pxx_raw" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: False # Don't skip for cuts. + fit: True + subscan: False + psd: "Pxx_raw" + calc: + fwhite: [5, 20] + lowf: 0.3 + f_max: 50 + mask: True + fixed_param: 'wn' + binning: True + fknee_est: 1.5 + alpha_est: 3.0 + merge_name: 'noise_fit_statsT' + bounds: + - [0.3, 20.0] # lower bound fknee, alpha + - [1.0, 6.0] # upper bound fknee, alpha + save: + wrap_name: "noiseT" + select: + min_noise: 2.0e-6 + max_noise: 80.0e-6 + max_fknee: 7.0 + + - name: "cut_bad_dist" + skip_on_sim: False + select: + param_name: "noiseT.white_noise" + + - name: "calibrate" + skip_on_sim: True + process: + kind: "array" + cal_array: "relcal.rel_factor" + select: + cut_array: "relcal.missing_factor" + + - name: "fourier_filter" + skip_on_sim: True + process: + filt_function: "iir_filter" + filter_params: + invert: True + + - name: "fourier_filter" + skip_on_sim: True + process: + filt_function: "timeconst_filter" + filter_params: + timeconst: "det_cal.tau_eff" + invert: True + + - name: "calibrate" + skip_on_sim: True + signal: "signal" + process: + kind: "array" + cal_array: "abscal.abscal_cmb" + + - name: "apodize" + skip_on_sim: False + process: + apodize_samps: 2000 + + - name: "demodulate" + skip_on_sim: False + process: + trim_samps: 6000 + demod_cfgs: + bpf_cfg: {'type': 'sine2', 'center': '4*f_HWP', 'width': '3.8*f_HWP', 'trans_width': 0.1} + lpf_cfg: {'type': 'sine2', 'cutoff': '1.9*f_HWP', 'trans_width': 0.1} + + - name : "tod_stats" + skip_on_sim: True + signal: "dsT" + wrap: "tod_stats_T" + calc: + stat_names: ["std", "ptp"] + split_subscans: True + save: True + + - name : "tod_stats" + skip_on_sim: True + signal: "demodQ" + wrap: "tod_stats_Q" + calc: + stat_names: ["median", "std", "skew", "kurtosis", "ptp"] + split_subscans: True + save: True + + - name : "tod_stats" + skip_on_sim: True + signal: "demodU" + wrap: "tod_stats_U" + calc: + stat_names: ["median", "std", "skew", "kurtosis", "ptp"] + split_subscans: True + save: True + + - name : "noisy_subscan_flags" + skip_on_sim: False # Keeping flags for sims + stats_name: tod_stats + calc: + subscan_stats: ["demodQ", "demodU", "dsT"] + nstd_lim: 3.0 + ptp_lim: 0.8 + kurt_lim: 0.5 + skew_lim: 0.5 + noisy_detector_lim: 0.5 + merge: False + save: True + select: True + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.left_scan', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags','smurfgaps.smurfgaps', 'turnaround_flags.turnarounds', 'source_flags.moon', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags_right' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.right_scan', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags','smurfgaps.smurfgaps', 'turnaround_flags.turnarounds', 'source_flags.moon', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags_left' + method: 'union' + + - name: "combine_flags" + skip_on_sim: False + process: + flag_labels: ['turnaround_flags.turnarounds', 'jumps_2pi.jump_flag', 'glitches_pre_hwpss.glitch_flags', 'jumps_slow.jump_flag', 'glitches_post_hwpss.glitch_flags', 'source_flags.moon','smurfgaps.smurfgaps', 'noisy_subscan_flags.valid_subscans'] + total_flags_label: 'glitch_flags' + method: 'union' \ No newline at end of file diff --git a/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml b/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml new file mode 100644 index 0000000..0a611e5 --- /dev/null +++ b/towards_r/v4/external-data/filtering_configs/satp3/preprocessing_config_20251216_proc_w_t2p_subtraction.yaml @@ -0,0 +1,398 @@ +context_file: 'use_this_local_260115.yaml' +plot_dir: '/scratch/gpfs/SIMONSOBS/external_data_e2e/v4/preprocessing/satp3_20251216_proc/preprocess_plots' +jobdb: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_jobdb.db' + +subobs: + use: ['wafer_slot', 'wafer.bandpass'] + label: wafer_slot + +archive: + index: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_proc/process_archive.sqlite' + policy: + type: 'simple' + filename: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/preprocessing/satp3_20251216_proc/preprocess_archive.h5' + +process_pipe: + - name: "detrend" + skip_on_sim: False + signal: "dsT" + process: + method: "linear" + count: 2500 + + - name: "detrend" + skip_on_sim: False + signal: "demodQ" + process: + method: "linear" + count: 2500 + + - name: "detrend" + skip_on_sim: False + signal: "demodU" + process: + method: "linear" + count: 2500 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "dsT" + apodize_samps: 2000 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "demodQ" + apodize_samps: 2000 + + - name: "apodize" + skip_on_sim: False + process: + signal_name: "demodU" + apodize_samps: 2000 + + - name : "estimate_t2p" + skip_on_sim: True + fit_in_freq : True + calc: + T_sig_name: 'dsT' + Q_sig_name: 'demodQ' + U_sig_name: 'demodU' + fit_freq_range: (0.01, 0.1) + calc_and_save: True + save: True + select: {} + + - name : "subtract_t2p" + skip_on_sim: False + use_data_aman: True + process: + fit_in_freq: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'dsT' + azss_stats_name: 'azss_statsT_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'dsT' + azss: 'azss_statsT_left' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodQ' + azss_stats_name: 'azss_statsQ_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_left' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodQ' + azss: 'azss_statsQ_left' + method: 'interpolate' + scan_flags: 'left_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodU' + azss_stats_name: 'azss_statsU_left' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_left' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodU' + azss: 'azss_statsU_left' + method: 'interpolate' + scan_flags: 'left_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'dsT' + azss_stats_name: 'azss_statsT_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'dsT' + azss: 'azss_statsT_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodQ' + azss_stats_name: 'azss_statsQ_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodQ' + azss: 'azss_statsQ_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "azss" + skip_on_sim: True # No estimation on sims + calc: + signal: 'demodU' + azss_stats_name: 'azss_statsU_right' + azrange: [-1.57079, 7.85398] + bins: 1351 # This is 0.4 deg bins. + flags: 'glitch_flags_right' + merge_stats: True + merge_model: False + save: True + process: + subtract: False + + - name: "subtract_azss_template" + skip_on_sim: False + process: + signal: 'demodU' + azss: 'azss_statsU_right' + method: 'interpolate' + scan_flags: 'right_scan' + subtract: True + + - name: "psd" + skip_on_sim: True + signal: "demodQ" + wrap: "psdQ" + process: + # max_samples: 524288 #2**19 + merge: False + nperseg: 200000 + noverlap: 0 + full_output: True + + - name: "psd" + skip_on_sim: True + signal: "demodU" + wrap: "psdU" + process: + # max_samples: 524288 #2**19 + merge: False + nperseg: 200000 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True + fit: True + subscan: False + psd: "psdQ" + calc: + fwhite: [0.5, 1.75] + lowf: 1.0e-4 + f_max: 1.9 + fknee_est: 1.0e-3 + alpha_est: 2.0 + fixed_param: 'wn' + binning: True + merge_name: 'noise_fit_statsQ' + bounds: + - [1.0e-4, 1.0] # lower bound fknee, alpha + - [0.1, 3.0] # upper bound fknee, alpha + fit_method: 'log_curve_fit' + curve_fit_kwargs: + maxfev: 20000 + save: + wrap_name: "noiseQ" + select: + require_finite_fit: True + + - name: "noise" + skip_on_sim: True + fit: True + subscan: False + psd: "psdU" + calc: + fwhite: [0.5, 1.75] + lowf: 1.0e-4 + f_max: 1.9 + fknee_est: 1.0e-3 + alpha_est: 2.0 + fixed_param: 'wn' + binning: True + merge_name: 'noise_fit_statsU' + bounds: + - [1.0e-4, 1.0] # lower bound fknee, alpha + - [0.1, 3.0] # upper bound fknee, alpha + fit_method: 'log_curve_fit' + curve_fit_kwargs: + maxfev: 20000 + save: + wrap_name: "noiseU" + select: + require_finite_fit: True + + - name: "fourier_filter" + skip_on_sim: False + signal_name: "demodQ" + wrap_name: "demodQ" + process: + filt_function: "counter_1_over_f" + noise_fit_array: "noiseQ" + filter_params: + each_detector: True + + - name: "fourier_filter" + skip_on_sim: False + signal_name: "demodU" + wrap_name: "demodU" + process: + filt_function: "counter_1_over_f" + noise_fit_array: "noiseU" + filter_params: + each_detector: True + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "dsT" + degree: 9 + in_place: True + mask: "glitch_flags" + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "demodQ" + degree: 1 + in_place: True + mask: "glitch_flags" + + - name: "sub_polyf" + skip_on_sim: False + process: + signal_name: "demodU" + degree: 1 + in_place: True + mask: "glitch_flags" + + - name : 'scan_freq_cut' + skip_on_sim: False + process: + signal_name_T: 'dsT' + signal_name_Q: 'demodQ' + signal_name_U: 'demodU' + + - name: "inv_var_flags" + skip_on_sim: False + calc: + signal_name: 'demodQ' + nsigma: 2 + save: True + select: True + + - name: "split_flags" + skip_on_sim: True + calc: + right_focal_plane: 0 + top_focal_plane: 0 + central_pixels: 0.20023039 + high_gain: 0.115 + high_noise: 3.5e-5 + high_tau: 1.5e-3 + det_A: A + pol_angle: 35 + crossover: BL + high_leakage: 1.0e-3 + high_2f: 1.5e-3 + save: True + + - name: "psd" + skip_on_sim: True # We won't need this from sims + signal: "demodQ" + wrap: "psdQ_mapmaking" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True # We won't need this from sims + psd: "psdQ_mapmaking" + fit: False + calc: + low_f: 1 + high_f: 2 + save: + wrap_name: "noiseQ_mapmaking" + + - name: "psd" + skip_on_sim: True # We won't need this from sims + signal: "demodU" + wrap: "psdU_mapmaking" + process: + max_samples: 524288 #2**19 + merge: False + nperseg: 65536 #2**16 + noverlap: 0 + full_output: True + + - name: "noise" + skip_on_sim: True # We won't need this from sims + psd: "psdU_mapmaking" + fit: False + calc: + low_f: 1 + high_f: 2 + save: + wrap_name: "noiseU_mapmaking" \ No newline at end of file diff --git a/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f090_20260304.yaml b/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f090_20260304.yaml new file mode 100644 index 0000000..ee425ff --- /dev/null +++ b/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f090_20260304.yaml @@ -0,0 +1,171 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4 +output_directory: &out_dir !path [*ext_data_dir, /v4/spectra/filtered_satp1_f090_20260304/south/20251216_full-dataset/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp1_20251216_full-dataset/signflip/south/] + +map_sets: + satp1_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp1_20251216_full-dataset/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp1_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp1" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp1" + filtering_tag: "satp1_f090_south_science" + wmap_f023_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fK/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_K_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_K1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 23 # Freq. tag (e.g. useful to coadd freqs) Used the ext data one + exp_tag: "wmap" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + wmap_f033_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fKa/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_Ka_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_Ka1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 33 + exp_tag: "wmap" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + planck_f030_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f030/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_030_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_030_long.txt" + n_bundles: 2 + freq_tag: 030 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + planck_f100_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f100/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f143/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + planck_f217_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f217/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_217_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_217_long.txt" + n_bundles: 2 + freq_tag: 217 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp1_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f353/satp1/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f090_south_science" # tag only used for couplings and transfer function + satp1_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp1_20251216_full-dataset/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp1_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp1" + hits_tag: "satp1" + filtering_tag: "satp1_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp1/south/20251216_full-dataset/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4/spectra/satp1/south/20251216_full-dataset/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp1: !path [*iso_dir, spectra/satp1/south/full-dataset/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/al5147/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*out_dir, transfer_functions] + + external_couplings_dir: !path [*out_dir, couplings] + + # For estimation + ## Number of sims for tf estimation + tf_est_num_sims: 10 + sim_id_start: 0 + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp1_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp1_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp1_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp1_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp1_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp1/south/f090/20251216_full-dataset_KW20260223/coadded_sims + satp1_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp1/south/f150/20251216_full-dataset_KW20260223/coadded_sims + filtered_map_template: + satp1_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp1_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" diff --git a/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f150_20260304.yaml b/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f150_20260304.yaml new file mode 100644 index 0000000..d4bc53b --- /dev/null +++ b/towards_r/v4/external-data/soopercool_configs/satp1/paramfile_south_science_filtered_satp1_f150_20260304.yaml @@ -0,0 +1,171 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4 +output_directory: &out_dir !path [*ext_data_dir, /v4/spectra/filtered_satp1_f150_20260304/south/20251216_full-dataset/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp1_20251216_full-dataset/signflip/south/] + +map_sets: + satp1_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp1_20251216_full-dataset/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp1_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp1" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp1" + filtering_tag: "satp1_f090_south_science" + wmap_f023_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fK/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_K_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_K1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 23 # Freq. tag (e.g. useful to coadd freqs) Used the ext data one + exp_tag: "wmap" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + wmap_f033_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fKa/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_Ka_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_Ka1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 33 + exp_tag: "wmap" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + planck_f030_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f030/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_030_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_030_long.txt" + n_bundles: 2 + freq_tag: 030 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + planck_f100_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f100/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f143/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + planck_f217_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f217/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_217_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_217_long.txt" + n_bundles: 2 + freq_tag: 217 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp1_f150_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f353/satp1/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp1_f150_south_science" # tag only used for couplings and transfer function + satp1_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp1_20251216_full-dataset/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp1_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp1" + hits_tag: "satp1" + filtering_tag: "satp1_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp1/south/20251216_full-dataset/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4/spectra/satp1/south/20251216_full-dataset/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp1: !path [*iso_dir, spectra/satp1/south/full-dataset/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/al5147/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*out_dir, transfer_functions] + + external_couplings_dir: !path [*out_dir, couplings] + + # For estimation + ## Number of sims for tf estimation + tf_est_num_sims: 10 + sim_id_start: 0 + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp1_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp1_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp1_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp1_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp1_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp1/south/f090/20251216_full-dataset_KW20260223/coadded_sims + satp1_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp1/south/f150/20251216_full-dataset_KW20260223/coadded_sims + filtered_map_template: + satp1_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp1_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" diff --git a/towards_r/v4/external-data/soopercool_configs/satp1/spectra_computation_steps.sh b/towards_r/v4/external-data/soopercool_configs/satp1/spectra_computation_steps.sh index 3acdc2c..25fbe4f 100644 --- a/towards_r/v4/external-data/soopercool_configs/satp1/spectra_computation_steps.sh +++ b/towards_r/v4/external-data/soopercool_configs/satp1/spectra_computation_steps.sh @@ -4,12 +4,11 @@ set -e # General instruments=("satp1") # ("satp1" "satp3") -sat_freqs=("f150") #"f090") +sat_freqs=("f150","f090") # For now, we only consider south patch. patches=("south") - soopercool_dir=/home/ar3186/SOOPERCOOL # Insert path to your local SOOPERCOOL directory # You can add more/take away splits here @@ -17,23 +16,54 @@ splits=("science") # Run each step for each instrument and split for sat in "${instruments[@]}"; do - for patch in "${patches[@]}"; do - for split in "${splits[@]}"; do - echo "=== Running for sat: $sat | split: $split ===" - - paramfile="paramfile_${patch}_${split}_filtered_${sat}_${sat_freqs}.yaml" - - echo "Using paramfile: $paramfile" - - # Uncomment the steps you want to run - # # This following script was taken from soopercool's "sa_dev" branch - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} - echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freqs ===" - echo + for sat_freq in "${sat_freqs[@]}"; do + for patch in "${patches[@]}"; do + for split in "${splits[@]}"; do + echo "=== Running for sat: $sat | split: $split ===" + + paramfile="paramfile_${patch}_${split}_filtered_${sat}_${sat_freq}_20260304.yaml" + + echo "Using paramfile: $paramfile" + + # Uncomment the steps you want to run + # This following script was taken from soopercool's "sa_dev" branch + srun -n 1 -c 112 --cpu_bind=cores python ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} + echo "=== Done with get_analysis_mask.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} + echo "=== Done with get_mode_coupling.py ===" + srun -n 12 -c 8 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_pseudo_cells_tf_estimation.py --globals ${paramfile} + echo "=== Done with compute_pseudo_cells_tf_estimation.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_transfer_function.py --globals ${paramfile} + echo "=== Done with compute_transfer_function.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} + echo "=== Done with get_full_couplings.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose + echo "=== Done with compute_pseudo_cells.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} --no_plots + echo "=== Done with coadd_pseudo_cells.py ===" + + srun -n 16 -c 7 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/precompute_cov_couplings.py --globals ${paramfile} + echo "=== Done with precompute_cov_couplings.py ===" + + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/prepare_cov_inputs.py --globals ${paramfile} + echo "=== Done with prepare_cov_inputs.py ===" + srun -n 112 -c 4 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_covariance.py --globals ${paramfile} + echo "=== Done with compute_covariance.py ===" + srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/create_sacc_file_analytic.py --globals ${paramfile} --data + echo "=== Done with create_sacc_file_analytic.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freqs}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_cals_and_beams.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_EB_angles.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freqs}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated.fits \ + --lmin-fit 150 --lmax-fit 600 --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_EB_angles.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freqs}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated_rotated.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_cals_and_beams.py (round 2) ===" + + echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freq ===" + done done done done \ No newline at end of file diff --git a/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml index 65f134a..ac61311 100644 --- a/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml +++ b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml @@ -14,6 +14,7 @@ map_sets: n_bundles: 4 # Number of bundles freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" filtering_tag: "satp3_f090_south_science" wmap_f023_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/wmap/fK/satp3/south/f090/coadded_maps] @@ -23,6 +24,7 @@ map_sets: n_bundles: 2 freq_tag: 23 # Freq. tag (e.g. useful to coadd freqs) Used the ext data one exp_tag: "wmap" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function wmap_f033_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/wmap/fKa/satp3/south/f090/coadded_maps] @@ -32,6 +34,7 @@ map_sets: n_bundles: 2 freq_tag: 33 exp_tag: "wmap" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function planck_f030_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f030/satp3/south/f090/coadded_maps] @@ -41,6 +44,7 @@ map_sets: n_bundles: 2 freq_tag: 030 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function planck_f100_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f100/satp3/south/f090/coadded_maps] @@ -50,6 +54,7 @@ map_sets: n_bundles: 2 freq_tag: 100 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function planck_f143_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f143/satp3/south/f090/coadded_maps] @@ -59,6 +64,7 @@ map_sets: n_bundles: 2 freq_tag: 143 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function planck_f217_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f217/satp3/south/f090/coadded_maps] @@ -68,6 +74,7 @@ map_sets: n_bundles: 2 freq_tag: 217 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function planck_f353_filtered_satp3_f090_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f353/satp3/south/f090/coadded_maps] @@ -77,6 +84,7 @@ map_sets: n_bundles: 2 freq_tag: 353 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function satp3_f150_south_science: map_dir: !path [*iso_dir, bundling/satp3_20251216/south] @@ -86,6 +94,7 @@ map_sets: n_bundles: 4 freq_tag: 150 exp_tag: "satp3" + hits_tag: "satp3" filtering_tag: "satp3_f150_south_science" @@ -130,6 +139,9 @@ general_pars: compute_Dl: True covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/al5147/homogeneous_hits_for_planck.fits cov_num_sims: 200 fiducial_cmb: null fiducial_dust: null @@ -138,5 +150,33 @@ covariance: transfer_settings: transfer_directory: !path [*transfer_dir, transfer_functions] - external_couplings_dir: !path [*out_dir, couplings] - \ No newline at end of file + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 2 + sim_id_start: 1 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090_20260304.yaml b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090_20260304.yaml new file mode 100644 index 0000000..d0b0687 --- /dev/null +++ b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090_20260304.yaml @@ -0,0 +1,171 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4 +output_directory: &out_dir !path [*ext_data_dir, /v4/spectra/filtered_satp3_f090_20260304/south/full-dataset/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp3_20251216/signflip/south/] + +map_sets: + satp3_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20251216/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" + filtering_tag: "satp3_f090_south_science" + wmap_f023_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fK/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_K_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_K1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 23 # Freq. tag (e.g. useful to coadd freqs) Used the ext data one + exp_tag: "wmap" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + wmap_f033_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/wmap/fKa/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/wmap/beams/] + map_template: "wmap_car_Ka_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "wmap_ampl_bl_Ka1_9yr_v5p1_long.txt" + n_bundles: 2 + freq_tag: 33 + exp_tag: "wmap" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f030_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f030/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_030_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_030_long.txt" + n_bundles: 2 + freq_tag: 030 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f100_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f100/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f143/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f217_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f217/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_217_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_217_long.txt" + n_bundles: 2 + freq_tag: 217 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /v4/filtered_ext_data_20260304/planck/f353/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle0_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + satp3_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20251216/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp3" + hits_tag: "satp3" + filtering_tag: "satp3_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp3/south/full-dataset/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4/spectra/satp3/south/full-dataset/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/al5147/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*out_dir, transfer_functions] + + external_couplings_dir: !path [*out_dir, couplings] + + # For estimation + ## Number of sims for tf estimation + tf_est_num_sims: 10 + sim_id_start: 0 + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216_t2p/south/f090/full-dataset/coadded_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216_t2p/south/f150/full-dataset/coadded_sims/ + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml index 0e43f27..adefc42 100644 --- a/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml +++ b/towards_r/v4/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml @@ -14,6 +14,7 @@ map_sets: n_bundles: 4 # Number of bundles freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" filtering_tag: "satp3_f090_south_science" wmap_f023_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/wmap/fK/satp3/south/f150/coadded_maps] @@ -23,6 +24,7 @@ map_sets: n_bundles: 2 freq_tag: 23 # Freq. tag (e.g. useful to coadd freqs) Used the ext data one exp_tag: "wmap" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function wmap_f033_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/wmap/fKa/satp3/south/f150/coadded_maps] @@ -32,6 +34,7 @@ map_sets: n_bundles: 2 freq_tag: 33 exp_tag: "wmap" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function planck_f030_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f030/satp3/south/f150/coadded_maps] @@ -41,6 +44,7 @@ map_sets: n_bundles: 2 freq_tag: 030 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function planck_f100_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f100/satp3/south/f150/coadded_maps] @@ -50,6 +54,7 @@ map_sets: n_bundles: 2 freq_tag: 100 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function planck_f143_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f143/satp3/south/f150/coadded_maps] @@ -59,6 +64,7 @@ map_sets: n_bundles: 2 freq_tag: 143 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function planck_f217_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f217/satp3/south/f150/coadded_maps] @@ -68,6 +74,7 @@ map_sets: n_bundles: 2 freq_tag: 217 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function planck_f353_filtered_satp3_f150_south_science: map_dir: !path [*ext_data_dir, /v4/filtered_ext_data/planck/f353/satp3/south/f150/coadded_maps] @@ -77,6 +84,7 @@ map_sets: n_bundles: 2 freq_tag: 353 exp_tag: "planck" + hits_tag: "flat" filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function satp3_f150_south_science: map_dir: !path [*iso_dir, bundling/satp3_20251216/south] @@ -86,6 +94,7 @@ map_sets: n_bundles: 4 freq_tag: 150 exp_tag: "satp3" + hits_tag: "satp3" filtering_tag: "satp3_f150_south_science" @@ -130,6 +139,9 @@ general_pars: compute_Dl: True covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/al5147/homogeneous_hits_for_planck.fits cov_num_sims: 200 fiducial_cmb: null fiducial_dust: null @@ -138,5 +150,33 @@ covariance: transfer_settings: transfer_directory: !path [*transfer_dir, transfer_functions] - external_couplings_dir: !path [*out_dir, couplings] - \ No newline at end of file + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 20 + sim_id_start: 0 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/satp3_20251216/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4/external-data/soopercool_configs/satp3/spectra_computation_steps.sh b/towards_r/v4/external-data/soopercool_configs/satp3/spectra_computation_steps.sh index a93ddbb..81a8327 100644 --- a/towards_r/v4/external-data/soopercool_configs/satp3/spectra_computation_steps.sh +++ b/towards_r/v4/external-data/soopercool_configs/satp3/spectra_computation_steps.sh @@ -4,13 +4,11 @@ set -e # General instruments=("satp3") # ("satp1" "satp3") -sat_freqs=("f090" "f150") +sat_freqs=("f090", "f150") # For now, we only consider south patch. patches=("south") -export OMP_NUM_THREADS=1 - soopercool_dir=/home/ar3186/SOOPERCOOL # Insert path to your local SOOPERCOOL directory # You can add more/take away splits here @@ -18,23 +16,54 @@ splits=("science") # Run each step for each instrument and split for sat in "${instruments[@]}"; do - for patch in "${patches[@]}"; do - for split in "${splits[@]}"; do - echo "=== Running for sat: $sat | split: $split ===" - - paramfile="paramfile_${patch}_${split}_filtered_${sat}_${sat_freqs}.yaml" - - echo "Using paramfile: $paramfile" - - # Uncomment the steps you want to run - # # This following script was taken from soopercool's "sa_dev" branch - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose - srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} #--no_plots - echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freqs ===" - echo + for sat_freq in "${sat_freqs[@]}"; do + for patch in "${patches[@]}"; do + for split in "${splits[@]}"; do + echo "=== Running for sat: $sat | split: $split ===" + + paramfile="paramfile_${patch}_${split}_filtered_${sat}_${sat_freq}_20260304.yaml" + + echo "Using paramfile: $paramfile" + + # Uncomment the steps you want to run + # This following script was taken from soopercool's "sa_dev" branch + srun -n 1 -c 112 --cpu_bind=cores python ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} + echo "=== Done with get_analysis_mask.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} + echo "=== Done with get_mode_coupling.py ===" + srun -n 12 -c 8 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_pseudo_cells_tf_estimation.py --globals ${paramfile} + echo "=== Done with compute_pseudo_cells_tf_estimation.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_transfer_function.py --globals ${paramfile} + echo "=== Done with compute_transfer_function.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} + echo "=== Done with get_full_couplings.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose + echo "=== Done with compute_pseudo_cells.py ===" + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} --no_plots + echo "=== Done with coadd_pseudo_cells.py ===" + + srun -n 16 -c 7 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/precompute_cov_couplings.py --globals ${paramfile} + echo "=== Done with precompute_cov_couplings.py ===" + + srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/prepare_cov_inputs.py --globals ${paramfile} + echo "=== Done with prepare_cov_inputs.py ===" + srun -n 112 -c 4 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_covariance.py --globals ${paramfile} + echo "=== Done with compute_covariance.py ===" + srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/create_sacc_file_analytic.py --globals ${paramfile} --data + echo "=== Done with create_sacc_file_analytic.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_cals_and_beams.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_EB_angles.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated.fits \ + --lmin-fit 150 --lmax-fit 600 --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_EB_angles.py ===" + + srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated_rotated.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + echo "=== Done with fit_cals_and_beams.py (round 2) ===" + + echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freq ===" + done done done done \ No newline at end of file From 1ef9c2c4fc389d40374148e3600f539d1318d9b5 Mon Sep 17 00:00:00 2001 From: Adrien La Posta Date: Fri, 27 Mar 2026 15:32:33 +0000 Subject: [PATCH 07/11] adapt to latest T2P(v4) --- pipeline/filtering/filter_sims_sotodlib.py | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index 0a17f95..81f11c4 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -267,19 +267,19 @@ def main(args): meta.restrict("dets", thinned) # Process data here to have t2p leakage template - # Only need to run it once for all simulations - # and only the pre-demodulation part. - if args.t2p_template: - data_aman = pp_util.multilayer_load_and_preprocess( - obs_id, - configs_init, - configs_proc, - meta=meta, - logger=logger, - init_only=True, - ) - else: - data_aman = None + # It will stop before each step with the + # use_data_aman flag in the preprocess config + # files and store the AxisManager in the data_aman + # dict. + data_aman = pp_util.multilayer_load_and_preprocess( + obs_id, + configs_init, + configs_proc, + meta=meta, + logger=logger, + stop_for_sims=True, + ignore_cfg_check=True + ) for sim_id, sim_type in product(sim_ids, sim_types): @@ -313,7 +313,8 @@ def main(args): sim_map=sim, meta=meta, logger=logger, - t2ptemplate_aman=data_aman + ignore_cfg_check=True, + data_amans=data_aman ) except loader.LoaderError: logger.warning( From 890f21278d4618ad9033cfa8683c27ece31341d2 Mon Sep 17 00:00:00 2001 From: susannaaz Date: Mon, 11 May 2026 19:17:16 -0400 Subject: [PATCH 08/11] modified make_signflip_noise to reuse SignFlipper per bundle instead of per sim for efficiency. modified coadder to cache weights_noise instead of recomputing for each realisation. --- pipeline/bundling/coadder.py | 41 +++++-- pipeline/bundling/make_signflip_noise.py | 137 ++++++++++++----------- 2 files changed, 107 insertions(+), 71 deletions(-) diff --git a/pipeline/bundling/coadder.py b/pipeline/bundling/coadder.py index abf2eeb..8334788 100644 --- a/pipeline/bundling/coadder.py +++ b/pipeline/bundling/coadder.py @@ -312,6 +312,7 @@ def __init__(self, bundle_db, freq_channel, car_map_template, map_dir, is_weights=True) for fname in self.fnames] print("weights: ", len(self.wmaps)) + self._weights_noise = None def signflip(self, seed=None): @@ -329,11 +330,23 @@ def signflip(self, seed=None): signflip_noise: np.array TQU map with th sign-flip noise realization. """ + class _NoInplaceScale: + """Wrapper that prevents in-place scaling of cached maps in utils.sum_maps.""" + def __init__(self, base): + self._base = base + + def __getattr__(self, name): + return getattr(self._base, name) + + def __imul__(self, other): + out = self._base.copy() + out *= other + return out + if seed is not None: np.random.seed(seed) perm_idx = np.random.permutation(len(self.wmaps)) - maps_list, weights_list = ([self.wmaps[i] for i in perm_idx], - [self.weights[i] for i in perm_idx]) + maps_list = [_NoInplaceScale(self.wmaps[i]) for i in perm_idx] mean_wQU_list = [self.ws[i] for i in perm_idx] abscal = self.full_abscal[perm_idx] @@ -342,10 +355,24 @@ def signflip(self, seed=None): pm_one = np.random.choice([-1, 1]) sign_list = np.where(weight_cumsum < 0.5, pm_one, -pm_one) - signflip_noise, weights_noise = utils.coadd_maps( - maps_list, weights_list, pix_type=self.pix_type, - sign_list=sign_list, car_template_map=self.car_map_template, - abscal=abscal + if self._weights_noise is None: + weights_list = [self.weights[i] for i in perm_idx] + signflip_noise, weights_noise = utils.coadd_maps( + maps_list, weights_list, pix_type=self.pix_type, + sign_list=sign_list, car_template_map=self.car_map_template, + abscal=abscal + ) + self._weights_noise = weights_noise + return signflip_noise, weights_noise + + signflip_noise = utils.sum_maps( + maps_list, + self._weights_noise, + self.pix_type, + mult=sign_list * abscal**-1, + fields_hp=self.fields_hp ) - return signflip_noise, weights_noise + good_weights = self._weights_noise > 0 + signflip_noise[good_weights] /= self._weights_noise[good_weights] + return signflip_noise, self._weights_noise diff --git a/pipeline/bundling/make_signflip_noise.py b/pipeline/bundling/make_signflip_noise.py index d7d3394..21bb8ce 100644 --- a/pipeline/bundling/make_signflip_noise.py +++ b/pipeline/bundling/make_signflip_noise.py @@ -73,85 +73,94 @@ def _make_signflip(args, size, rank, comm, split_intra_obs=None, split_inter_obs task_ids = mpi.distribute_tasks(size, rank, n_missing,) #local_mpi_list = [mpi_shared_list[i] for i in task_ids] - # Loop over only missing tasks - #for task_id in mpi.taskrange(len(missing_tasks)): - #for task_id in mpi.taskrange(n_missing - 1): + # Group tasks by bundle_id so we can reuse SignFlipper. SignFlipper reads the + # per-bundle atomics into memory, and reusing it avoids repeating that work + # for every sim_id. + sim_ids_by_bundle = {} for task_id in task_ids: bundle_id, sim_id = missing_tasks[task_id] - print(f"Running bundle_id={bundle_id} sim_id={sim_id}") + sim_ids_by_bundle.setdefault(bundle_id, []).append(sim_id) - combined_map = None - combined_weight = None - - # Now construct the signflipper (only for needed bundle_id) + # Loop over only missing tasks for this rank, grouped by bundle_id. + for bundle_id, sim_ids in sim_ids_by_bundle.items(): + signflippers = [] for bundle_db, map_dir_i in zip(sat_bundle_dbs, sat_map_dirs): - signflipper = SignFlipper( - bundle_db=bundle_db, - freq_channel=args.freq_channel, - wafer=args.wafer, - bundle_id=bundle_id, - null_prop_val=split_inter_obs, - pix_type=args.pix_type, - car_map_template=args.car_map_template, - split_label=split_intra_obs, - map_dir=map_dir_i, - atomic_list=args.atomic_list - ) + signflippers.append(( + bundle_db, + SignFlipper( + bundle_db=bundle_db, + freq_channel=args.freq_channel, + wafer=args.wafer, + bundle_id=bundle_id, + null_prop_val=split_inter_obs, + pix_type=args.pix_type, + car_map_template=args.car_map_template, + split_label=split_intra_obs, + map_dir=map_dir_i, + atomic_list=args.atomic_list, + abscal=args.abscal + ) + )) - print(len(signflipper.fnames)) + for sim_id in sim_ids: + print(f"Running bundle_id={bundle_id} sim_id={sim_id}") - # NEW check here - if not signflipper.fnames: - print(f"Skipping bundle_id={bundle_id} sim_id={sim_id} (no atomic files)") - continue + combined_map = None + combined_weight = None - # Make noise map - try: - result = signflipper.signflip(seed=12345 * bundle_id + sim_id) - if not isinstance(result, (tuple, list)) or len(result) != 2: - print(f"SignFlipper returned unexpected result for bundle_id={bundle_id} sim_id={sim_id} from {bundle_db}: {result}") + for bundle_db, signflipper in signflippers: + print(len(signflipper.fnames)) + if not signflipper.fnames: + print(f"Skipping bundle_id={bundle_id} sim_id={sim_id} (no atomic files)") continue - noise_map, noise_weight = result - except Exception as e: - print(f"SignFlipper crashed for bundle_id={bundle_id} sim_id={sim_id} from {bundle_db}: {e}") - continue - # Initialize or accumulate - if combined_map is None: - combined_map = noise_map.copy() - combined_weight = noise_weight.copy() - else: - combined_map += noise_map - combined_weight += noise_weight + # Make noise map + try: + result = signflipper.signflip(seed=12345 * bundle_id + sim_id) + if not isinstance(result, (tuple, list)) or len(result) != 2: + print(f"SignFlipper returned unexpected result for bundle_id={bundle_id} sim_id={sim_id} from {bundle_db}: {result}") + continue + noise_map, noise_weight = result + except Exception as e: + print(f"SignFlipper crashed for bundle_id={bundle_id} sim_id={sim_id} from {bundle_db}: {e}") + continue + + # Initialize or accumulate + if combined_map is None: + combined_map = noise_map.copy() + combined_weight = noise_weight.copy() + else: + combined_map += noise_map + combined_weight += noise_weight if combined_map is None: print(f"No valid maps for bundle_id={bundle_id} sim_id={sim_id}, skipping save.") continue - # Output filenames - out_fname = os.path.join( - out_dir, - args.map_string_format.format( - split=split_tag, - bundle_id=bundle_id, - wafer=wafer_tag, - patch=patch_tag, - freq_channel=args.freq_channel, - map_type='{map_type}' + # Output filenames + out_fname = os.path.join( + out_dir, + args.map_string_format.format( + split=split_tag, + bundle_id=bundle_id, + wafer=wafer_tag, + patch=patch_tag, + freq_channel=args.freq_channel, + map_type='{map_type}' + ) ) - ) - out_fname = out_fname.replace("__", "_") - out_fname = out_fname.replace("{map_type}", f"{sim_id:04d}"+"_{}") - - # Skip existing maps if overwrite=False - if (not args.overwrite) and os.path.exists(out_fname.format("map")): - if rank == 0: - print(f"Skipping existing: {out_fname}") - continue - - # Save maps - print('writing maps: '+out_fname) - utils.write_maps(out_fname, args.pix_type, combined_map, combined_weight, dtype=np.float32) + out_fname = out_fname.replace("__", "_") + out_fname = out_fname.replace("{map_type}", f"{sim_id:04d}"+"_{}") + + # Skip existing maps if overwrite=False + if (not args.overwrite) and os.path.exists(out_fname.format("map")): + if rank == 0: + print(f"Skipping existing: {out_fname}") + continue + + # Save maps + print('writing maps: '+out_fname) + utils.write_maps(out_fname, args.pix_type, combined_map, combined_weight, dtype=np.float32) # Quickplots #if sim_id % (n_sims // 3) == 0: From 67aa850d9cacdbe2da1ef17d5396879b6450142a Mon Sep 17 00:00:00 2001 From: susannaaz Date: Sun, 24 May 2026 23:55:27 -0400 Subject: [PATCH 09/11] fixed logs --- pipeline/filtering/filter_sims_sotodlib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipeline/filtering/filter_sims_sotodlib.py b/pipeline/filtering/filter_sims_sotodlib.py index 7ec464e..0d15867 100644 --- a/pipeline/filtering/filter_sims_sotodlib.py +++ b/pipeline/filtering/filter_sims_sotodlib.py @@ -285,15 +285,15 @@ def main(args): ignore_cfg_check=True ) except loader.LoaderError: - logger.warning(f"NO METADATA: " + logger.warning(f"NO METADATA: ", f"({patch}, {freq_channel}, {obs_id}, {wafer})") continue except OSError as err: - logger.warning(f"{err}: " + logger.warning(f"{err}: ", ignore_cfg_check=True, ) except loader.LoaderError: - logger.warning(f"NO METADATA IN DATA_AMAN: " + logger.warning(f"NO METADATA IN DATA_AMAN: ", f"({patch}, {freq_channel}, {obs_id}, {wafer})") continue From bd46c801beb143df60210315c4dce598bebeca91 Mon Sep 17 00:00:00 2001 From: susannaaz Date: Wed, 10 Jun 2026 13:53:42 -0400 Subject: [PATCH 10/11] v4_paper scripts --- ...omics_planck_bundle0_20260425-noazss.slurm | 36 +++++ ...omics_planck_bundle0_20260520-noazss.slurm | 36 +++++ ...config_planck_bundle0_20260425-noazss.yaml | 36 +++++ ...tering_config_planck_bundle0_20260520.yaml | 36 +++++ ...config_planck_bundle1_20260425-noazss.yaml | 36 +++++ ...tering_config_planck_bundle1_20260520.yaml | 36 +++++ .../satp3/filtering_config_wmap_bundle0.yaml | 35 +++++ ...ering_planck_bundle0_20250425-noazss.slurm | 36 +++++ .../filtering_planck_bundle0_20250520.slurm | 36 +++++ .../filtering_planck_bundle1_20250520.slurm | 36 +++++ .../satp3/filtering_wmap_bundle0.slurm | 32 ++++ .../preprocessing_config_20260417_init.yaml | 1 + ...rocessing_config_20260425_noazss_proc.yaml | 1 + ...ile_south_science_filtered_satp3_f090.yaml | 142 ++++++++++++++++++ ...ile_south_science_filtered_satp3_f150.yaml | 142 ++++++++++++++++++ .../satp3/spectra_computation_steps.sh | 69 +++++++++ 16 files changed, 746 insertions(+) create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm create mode 100644 towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm create mode 120000 towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml create mode 120000 towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml create mode 100644 towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml create mode 100644 towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml create mode 100755 towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm new file mode 100644 index 0000000..dfc049e --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260425-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=satp3-coadd-pk-0 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_20260417-noazss.log" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260425-noazss.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm new file mode 100644 index 0000000..419426b --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/coadd_atomics_planck_bundle0_20260520-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=4 +#SBATCH --ntasks=16 +#SBATCH --cpus-per-task=28 +#SBATCH --time=8:00:00 +#SBATCH --job-name=1-satp3-coadd-pk +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_coadd_20260520-noazss.log" + +export OMP_NUM_THREADS=28 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 16 -c 28 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/coadd_filtered_ext.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml new file mode 100644 index 0000000..463a737 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260425-noazss.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260417_noazss/bundles_{patch}_seed66_20260417.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260417_noazss/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260417_noazss/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml +preprocess_config_proc: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml new file mode 100644 index 0000000..2678651 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle0_20260520.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260520/bundles_{patch}_seed66_20260520.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260520/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260520/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_init.yaml +preprocess_config_proc: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml new file mode 100644 index 0000000..1611199 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260425-noazss.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260417_noazss/bundles_{patch}_seed66_20260417.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260417_noazss/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260417_noazss/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 1 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml +preprocess_config_proc: /home/sa5705/software/bb-awg-scripts/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml new file mode 100644 index 0000000..d2a3569 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_planck_bundle1_20260520.yaml @@ -0,0 +1,36 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/bundling/satp3_20260520/bundles_{patch}_seed66_20260520.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/mapmaking/satp3_20260520/atomic_db.sqlite' +#output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data_20260304/planck/f{sim_type}/satp3/{patch}/{freq_channel} +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/filtered_ext_data_20260520/planck/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/planck/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "planck_car_{sim_type}_bundle1_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +#sim_types: ["030", "100", "143", "217", "353"] # ext data freq channel +sim_types: ["100", "143", "353"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_in, det_out] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 1 +query_restrict: "pwv < 2 AND number_dets > 10 AND ((f_hwp > 1.9 AND f_hwp < 2.3) OR (f_hwp > -2.3 AND f_hwp < -1.9))" + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_init.yaml +preprocess_config_proc: /home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260520_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_in, det_out] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml new file mode 100644 index 0000000..7d5f6ab --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_config_wmap_bundle0.yaml @@ -0,0 +1,35 @@ +## Required shared arguments ## +bundle_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/bundling/satp3_20251216/bundles_{patch}_seed66_20251216.db' +atomic_db: '/scratch/gpfs/SIMONSOBS/sat-iso/v4/mapmaking/satp3_20251216/atomic_db.sqlite' +output_dir: &output_dir /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/filtered_ext_data/wmap/f{sim_type}/satp3/{patch}/{freq_channel} +sim_dir: /scratch/gpfs/SIMONSOBS/external_data_e2e/unfiltered_data/wmap/maps +atomic_sim_dir: !path [*output_dir, atomic_maps] +sim_string_format: "wmap_car_{sim_type}_bundle0_res1.0amin_coords_c.fits" + +# For filtering on 4 tiger nodes, 2 sims take <~ 3 hours. +sim_types: ["K", "Ka"] # ext data freq channel +sim_ids: [0] # should be there, but not used in the filter_ext_sotodib.py script +patches: [south] +freq_channels: [f090, f150] # freq channel defining the filtering +t2p_template: False + +# (Required for coadd_filtered_sims) +intra_obs_splits: [det_lower, det_upper] + +## Optional shared arguments ## +pix_type: car +car_map_template: "/scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits" +bundle_id: 0 +query_restrict: " pwv < 2 AND number_dets > 10 " + +## Required arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +preprocess_config_init: preprocessing_config_20251216_init.yaml +preprocess_config_proc: preprocessing_config_20251216_proc.yaml + +## Optional arguments for filter_sims_sotodlib / filter_ext_sotodlib ## +fp_thin: 8 + +## Optional arguments for coadd_filtered_sims ## +intra_obs_pair: [det_lower, det_upper] +inter_obs_splits: [] +nbatch_atomics: 1 diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm new file mode 100644 index 0000000..ae04bce --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250425-noazss.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b1 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" #"0" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260425-noazss.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260425-noazss.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm new file mode 100644 index 0000000..ff0843a --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle0_20250520.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b0 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="0" #"1" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260520.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm new file mode 100644 index 0000000..5a83f05 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_planck_bundle1_20250520.slurm @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=satp3-filtering-pk-b1 +#SBATCH --mail-user sazzoni@princeton.edu + +set -e + +module use --append /scratch/gpfs/SIMONSOBS/modules +module load soconda/3.12/v0.6.8 + +export PYTHONPATH="${PYTHONPATH}:/home/sa5705/software/sotodlib/" + +# Log file +tel_ext_data="planck" # planck # wmap +tel_filter="satp3" +bundle="1" #"0" +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering_20260520.log" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/sa5705/software/bb-awg-scripts ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4_paper/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}_20260520.yaml + +echo "Launching pipeline at $(date)" + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm new file mode 100644 index 0000000..ad56d12 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/filtering_wmap_bundle0.slurm @@ -0,0 +1,32 @@ +#!/bin/bash -l + +#SBATCH --account=simonsobs +#SBATCH --nodes=20 +#SBATCH --ntasks=1120 +#SBATCH --cpus-per-task=2 +#SBATCH --time=10:00:00 +#SBATCH --job-name=wmap-satp3-ext-filtering-bundle0 +#SBATCH --mail-user arianna.rizzieri@physics.ox.ac.uk + +set -e + +# Log file +tel_ext_data="wmap" # planck # wmap +tel_filter="satp3" +bundle="0" # 0 # 1 +log="./logs/log_${tel_ext_data}_${bundle}_${tel_filter}_ext_filtering" + +export OMP_NUM_THREADS=2 + +bb_awg_scripts_dir=/home/ar3186/awg-clean/bb-awg-scripts/ ## YOUR bb_awg_scripts DIRECTORY + +filtering_config=${bb_awg_scripts_dir}/towards_r/v4/external-data/filtering_configs/${tel_filter}/filtering_config_${tel_ext_data}_bundle${bundle}.yaml + +srun -n 1120 -c 2 --cpu_bind=cores \ + python -u ${bb_awg_scripts_dir}/pipeline/filtering/filter_ext_sotodlib.py --config_file $filtering_config \ + > ${log} 2>&1 + +echo ${com} +echo "Launching pipeline at $(date)" +eval ${com} > ${log} 2>&1 +echo "Ending batch script at $(date)" diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml new file mode 120000 index 0000000..4353a4a --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260417_init.yaml @@ -0,0 +1 @@ +/home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260417_init.yaml \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml new file mode 120000 index 0000000..9c8f973 --- /dev/null +++ b/towards_r/v4_paper/external-data/filtering_configs/satp3/preprocessing_config_20260425_noazss_proc.yaml @@ -0,0 +1 @@ +/home/sa5705/software/iso-sat/v4_paper/preprocessing/satp3/preprocessing_config_20260425_noazss_proc.yaml \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml new file mode 100644 index 0000000..8f65d83 --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f090.yaml @@ -0,0 +1,142 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper +output_directory: &out_dir !path [*ext_data_dir, /spectra/filtered_satp3_f090/south/full-dataset_20260417_noazss/science] +transfer_dir: &transfer_dir !path [*iso_dir, /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp3_20260417_noazss/signflip/south/] + +map_sets: + satp3_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" + filtering_tag: "satp3_f090_south_science" + planck_f100_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f100/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f143/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp3_f090_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f353/satp3/south/f090/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f090_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f090_south_science" # tag only used for couplings and transfer function + satp3_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp3" + hits_tag: "satp3" + filtering_tag: "satp3_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*transfer_dir, transfer_functions] + + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 19 + sim_id_start: 1 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml new file mode 100644 index 0000000..6fb8c6b --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/paramfile_south_science_filtered_satp3_f150.yaml @@ -0,0 +1,142 @@ +ext_data_directory: &ext_data_dir /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/ +iso_directory: &iso_dir /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper +output_directory: &out_dir !path [*ext_data_dir, /spectra/filtered_satp3_f150/south/full-dataset_20260417_noazss/science] +transfer_dir: &transfer_dir !path [*iso_dir, /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science] +inputs_dir: &inputs_dir !path [*iso_dir, soopercool_inputs] +signflip_dir: &sf_dir !path [*iso_dir, noise/satp3_20260417_noazss/signflip/south/] + +map_sets: + satp3_f090_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f090_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 # Number of bundles + freq_tag: 090 # Freq. tag (e.g. useful to coadd freqs) + exp_tag: "satp3" # Experiment tag (useful to get noise-bias free cross-split spectra) + hits_tag: "satp3" + filtering_tag: "satp3_f090_south_science" + planck_f100_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f100/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_100_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_100_long.txt" + n_bundles: 2 + freq_tag: 100 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + planck_f143_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f143/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_143_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_143_long.txt" + n_bundles: 2 + freq_tag: 143 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + planck_f353_filtered_satp3_f150_south_science: + map_dir: !path [*ext_data_dir, /filtered_ext_data_20260417_noazss/planck/f353/satp3/south/f150/coadded_maps] + beam_dir: !path [*ext_data_dir, unfiltered_data/planck/beam_window_functions/] + map_template: "planck_car_353_bundle{id_bundle}_res1.0amin_coords_c_bundle{id_bundle}_f150_science_{filtered|filtered_weights}.fits" + beam_file: "beam_pol_353_long.txt" + n_bundles: 2 + freq_tag: 353 + exp_tag: "planck" + hits_tag: "flat" + filtering_tag: "satp3_f150_south_science" # tag only used for couplings and transfer function + satp3_f150_south_science: + map_dir: !path [*iso_dir, bundling/satp3_20260417_noazss/south] + beam_dir: !path [*inputs_dir, beams] + map_template: "satp3_f150_science_bundle{id_bundle}_{map|hits}.fits" + beam_file: "beam_unity.dat" + n_bundles: 4 + freq_tag: 150 + exp_tag: "satp3" + hits_tag: "satp3" + filtering_tag: "satp3_f150_south_science" + + +#################### +# Masking metadata # +#################### +masks: + # Specify coordinates if using a rectangular mask + box_mask: null + # Set to True if mask is calculated from the weights, + # otherwise set to False to use the hits + use_weights: False + analysis_mask: !path [*out_dir, masks/analysis_mask.fits] + # When specifying this file, the mapset/bundle-specific hits files will + # be ignored and instead a global hits file will be used (testing puroposes) + global_hits: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + # Path to products (binary) + galactic_mask: !path [*inputs_dir, masks/mask_GAL070_coords_c_sat_f090_geometry.fits] + point_source_catalog: null + point_source_mask: null + external_mask: null + # Apodization details + apod_radius: 10.0 + smoothing_radius: 10.0 + apod_radius_point_source: 1.0 + apod_type: "C1" + +#################################### +# Metadata related to the analysis # +#################################### +## General parameters +general_pars: + pix_type: car + car_template: /scratch/gpfs/SIMONSOBS/so/science-readiness/geometry/sat_f090.fits + nside: 512 + lmin: 2 + lmax: 803 + binning_file: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/spectra/satp3/south/full-dataset_20260417_noazss/science/binning/binning_car_lmax2700_deltal15.npz + pure_B: False + # Where the beam window is lower than beam_floor, set it to beam_floor + beam_floor: 1.e-2 + compute_Dl: True + +covariance: + hits_files: + satp3: !path [*iso_dir, spectra/satp3/south/full-dataset_20260417_noazss/science/masks/normalized_hits.fits] + flat: /scratch/gpfs/SIMONSOBS/users/sa5705/SO/ISO_v4_paper/homogeneous_hits_for_planck.fits + cov_num_sims: 200 + fiducial_cmb: null + fiducial_dust: null + fiducial_synch: null + +transfer_settings: + transfer_directory: !path [*transfer_dir, transfer_functions] + + external_couplings_dir: !path [*transfer_dir, couplings] + tf_est_num_sims: 19 + sim_id_start: 1 + # # For estimation + # ## Number of sims for tf estimation and validation + # ## Parameters of the PL sims used for TF estimation + # power_law_pars_tf_est: + # amp: 1.0 + # delta_ell: 15 + # power_law_index: 2. + # ## Optional beams applied on PL sims + # # If true, beams will be applied only on the validation simulations. By default (false) + # # beam are applied to both the estimation and validation sims, + # # to account for potential effect of the beam on the TF (e.g. commutativity) + # do_not_beam_est_sims: True + # beams_list: [] + + ## Path to the sims for TF estimation + unfiltered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4/transfer_function/input_sims/ + unfiltered_map_template: + satp3_f090_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + satp3_f150_south_science: "{pure_type}_4.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR.fits" + filtered_map_dir: + satp3_f090_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f090/full-dataset/coadded_sims + satp3_f150_south_science: /scratch/gpfs/SIMONSOBS/sat-iso/v4_paper/transfer_function/satp3_20260417_noazss/south/f150/full-dataset/coadded_sims + filtered_map_template: + satp3_f090_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f090_science_filtered.fits" + satp3_f150_south_science: "{pure_type}_1.0arcmin_fwhm30.0_sim{id_sim:04d}_CAR_bundle0_f150_science_filtered.fits" \ No newline at end of file diff --git a/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh b/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh new file mode 100755 index 0000000..b28b161 --- /dev/null +++ b/towards_r/v4_paper/external-data/soopercool_configs/satp3/spectra_computation_steps.sh @@ -0,0 +1,69 @@ +#!/bin/bash -l + +set -e + +# General +instruments=("satp3") # ("satp1" "satp3") +sat_freqs=("f090" "f150") + +# For now, we only consider south patch. +patches=("south") + +soopercool_dir=/home/sa5705/software/SOOPERCOOL # Insert path to your local SOOPERCOOL directory + +# You can add more/take away splits here +splits=("science") + +# Run each step for each instrument and split +for sat in "${instruments[@]}"; do + for sat_freq in "${sat_freqs[@]}"; do + for patch in "${patches[@]}"; do + for split in "${splits[@]}"; do + echo "=== Running for sat: $sat | split: $split ===" + + paramfile="towards_r/v4_paper/external-data/soopercool_configs/${sat}/paramfile_${patch}_${split}_filtered_${sat}_${sat_freq}.yaml" + + echo "Using paramfile: $paramfile" + + # Uncomment the steps you want to run + # This following script was taken from soopercool's "sa_dev" branch + #srun -n 1 -c 112 --cpu_bind=cores python ${soopercool_dir}/pipeline/get_analysis_mask.py --globals ${paramfile} + #echo "=== Done with get_analysis_mask.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_mode_coupling.py --globals ${paramfile} + #echo "=== Done with get_mode_coupling.py ===" + #srun -n 12 -c 8 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_pseudo_cells_tf_estimation.py --globals ${paramfile} + #echo "=== Done with compute_pseudo_cells_tf_estimation.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/transfer/compute_transfer_function.py --globals ${paramfile} + #echo "=== Done with compute_transfer_function.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/get_full_couplings.py --globals ${paramfile} + #echo "=== Done with get_full_couplings.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_pseudo_cells.py --globals ${paramfile} --verbose + #echo "=== Done with compute_pseudo_cells.py ===" + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/coadd_pseudo_cells.py --globals ${paramfile} --no_plots + #echo "=== Done with coadd_pseudo_cells.py ===" + + #srun -n 16 -c 7 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/precompute_cov_couplings.py --globals ${paramfile} + #echo "=== Done with precompute_cov_couplings.py ===" + + #srun -n 1 -c 112 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/prepare_cov_inputs.py --globals ${paramfile} + #echo "=== Done with prepare_cov_inputs.py ===" + #srun -n 112 -c 1 --cpu_bind=cores python -u ${soopercool_dir}/pipeline/compute_covariance.py --globals ${paramfile} + #echo "=== Done with compute_covariance.py ===" + srun -n 1 -c 112 python -u ${soopercool_dir}/pipeline/create_sacc_file_analytic.py --globals ${paramfile} --data + echo "=== Done with create_sacc_file_analytic.py ===" + # + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_cals_and_beams.py ===" + + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_EB_angles.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated.fits \ + # --lmin-fit 150 --lmax-fit 600 --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_EB_angles.py ===" + # + #srun -n 1 -c 112 python ${soopercool_dir}/pipeline/postproc/fit_cals_and_beams.py --sacc-file /scratch/gpfs/SIMONSOBS/external_data_e2e/v4/spectra/filtered_${sat}_${sat_freq}_20260304_fixed_exp_tag/south/20251216_full-dataset/science/saccs/cl_and_cov_sacc_calibrated_rotated.fits --lmin-fit 150 --lmax-fit 600 --map-sets-ref planck_f100_filtered_${sat}_${sat_freq}_south_science,planck_f143_filtered_${sat}_${sat_freq}_south_science --map-sets-to-fit ${sat}_f090_south_science,${sat}_f150_south_science --lmax-sacc 650 + #echo "=== Done with fit_cals_and_beams.py (round 2) ===" + + echo "=== Done with sat: $sat | split: $split | filtered as: $sat $sat_freq ===" + done + done + done +done \ No newline at end of file From f061f4566e090d02a5c7a2226e66ff81d1bdb2d7 Mon Sep 17 00:00:00 2001 From: susannaaz Date: Thu, 11 Jun 2026 13:41:43 -0400 Subject: [PATCH 11/11] added missing script --- pipeline/filtering/coadd_filtered_ext.py | 394 +++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 pipeline/filtering/coadd_filtered_ext.py diff --git a/pipeline/filtering/coadd_filtered_ext.py b/pipeline/filtering/coadd_filtered_ext.py new file mode 100644 index 0000000..518f407 --- /dev/null +++ b/pipeline/filtering/coadd_filtered_ext.py @@ -0,0 +1,394 @@ +""" +Based on coadd_filtered_ext.py, +adapted to run on the external data. +""" +import numpy as np +import argparse +import sqlite3 +import os +import sys +import time +import tracemalloc +from itertools import product + +import sotodlib.preprocess.preprocess_util as pp_util + +# TODO: Make it an actual module +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'bundling')) +) +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'filtering')) +) +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'misc')) +) +import mpi_utils as mpi # noqa +import bundling_utils as bu # noqa +import filtering_utils as fu # noqa +import coordinator as coord # noqa + + +def main(args): + """ + """ + if args.pix_type not in ["hp", "car"]: + raise ValueError("Unknown pixel type, must be 'car' or 'hp'.") + if "{patch" not in args.bundle_db: + raise ValueError("bundle_db does not have \ + required placeholder 'patch'") + for required_tag in ["{sim_type"]: + if required_tag not in args.sim_string_format: + raise ValueError(f"sim_string_format does not have \ + required placeholder '{required_tag}'") + for required_tag in ["{patch", "{freq_channel"]: + if required_tag not in args.output_dir: + raise ValueError(f"output_dir does not have \ + required placeholder '{required_tag}'") + for required_tag in ["{patch", "{freq_channel"]: + if required_tag not in args.atomic_sim_dir: + raise ValueError(f"atomic_sim_dir does not have \ + required placeholder '{required_tag}'") + + # MPI related initialization + rank, size, comm = mpi.init(True) + + # Initialize the logger + logger = pp_util.init_logger("benchmark", verbosity=3) + if rank == 0: + start = time.time() + + # Ensure that freq_channels for the metadata follow the "f090" convention. + # We keep the original labels in a dict called freq_labels. + freq_labels = {} + for f in args.freq_channels: # If these don't contain the "f", add + freq_channel = f"f{f}" if "f" not in f else f + freq_labels[freq_channel] = f # dict values are the original labels + freq_channels = list(freq_labels.keys()) + + # Sim related arguments + sim_types = args.sim_types + sim_string_format = args.sim_string_format + + # Input directory + atomic_sim_dir = args.atomic_sim_dir + + # Output directories + patches = args.patches + out_dirs = { + (patch, freq_channel, sim_type): + args.output_dir.format( + patch=patch, freq_channel=freq_labels[freq_channel], sim_type=sim_type + ) + for patch, freq_channel, sim_type in product(patches, freq_channels, sim_types) + } + coadded_dirs = {key: f"{out_dir}/coadded_maps" + for key, out_dir in out_dirs.items()} + plot_dirs = {key: f"{out_dir}/plots" for key, out_dir in out_dirs.items()} + + for key in out_dirs: + os.makedirs(coadded_dirs[key], exist_ok=True) + os.makedirs(plot_dirs[key], exist_ok=True) + + # Pixelization arguments + pix_type = args.pix_type + if pix_type == "hp": + mfmt = ".fits" # TODO: test fits.gz for HEALPix + car_map_template = None + elif pix_type == "car": + mfmt = ".fits" + car_map_template = args.car_map_template + + # Databases + atom_db = args.atomic_db + bundle_dbs = {patch: args.bundle_db.format(patch=patch) + for patch in patches} + + # Bundle query arguments + freq_channel = args.freq_channels + inter_obs_splits = args.inter_obs_splits + if inter_obs_splits is None: + inter_obs_splits = [] + if isinstance(inter_obs_splits, str): + if "," in inter_obs_splits: + inter_obs_splits = inter_obs_splits.split(",") + else: + inter_obs_splits = [inter_obs_splits] + + # Load all data from bundle dbs without filtering + bundle_id = args.bundle_id + bundles = { + patch: + coord.BundleCoordinator.from_dbfile( + bundle_dbs[patch], bundle_id=bundle_id + ) for patch in patches + } + + # Gather all split labels of atomics to be coadded + intra_obs_splits = args.intra_obs_splits + intra_obs_pair = args.intra_obs_pair + if intra_obs_splits is not None: + if isinstance(intra_obs_splits, str): + if "," not in intra_obs_splits: + intra_obs_splits = [intra_obs_splits] + else: + intra_obs_splits = (intra_obs_splits).split(",") + if intra_obs_pair is not None: + if isinstance(intra_obs_pair, str): + if "," not in intra_obs_pair: + raise ValueError("You must pass a comma-separated string list " + "to 'intra_obs_pair'.") + else: + intra_obs_pair = intra_obs_pair.split(",") + if ((intra_obs_splits, intra_obs_pair) == (None, None)): + raise ValueError("You must pass at least one of the two: " + "'intra_obs_pair' or 'intra_obs_splits'.") + + logger.info(f"Split labels to coadd individually: {intra_obs_splits}") + logger.info(f"Split labels to coadd together: {intra_obs_pair}") + + # Extract list of ctimes from bundle database for the given + # bundle_id and without atomic batches - inter obs null label + ctimes = { + (patch, inter_obs_split, None): + bundles[patch].get_ctimes( + bundle_id=bundle_id, null_prop_val=inter_obs_split + ) for inter_obs_split in inter_obs_splits + for patch in patches + } + # Add the science split without atomic batches + for patch in patches: + ctimes[patch, "science", None] = bundles[patch].get_ctimes( + bundle_id=bundle_id + ) + + # Randomly split science ctimes into batches (optional) + nbatches = args.nbatch_atomics + if nbatches is not None: + nbatches_dict = {} + for patch in patches: + # Limit number of batches to one half the number of ctimes + if nbatches > len(ctimes[patch, "science", None]) // 2: + nbatches_dict[patch] = len(ctimes[patch, "science", None]) // 2 + # Must have at least two batches + if nbatches < 2: + nbatches_dict[patch] = None + batches = {} + for patch in patches: + batches[patch] = [None] + if nbatches_dict[patch] is None: + pass + elif nbatches_dict[patch] > 1: + nbatch = nbatches_dict[patch] + batches[patch] = range(nbatch) + nctimes = len(ctimes[patch, 'science', None]) + logger.info( + f"{patch}: splitting atomics into {nbatch} random " + f"batches with {nctimes // nbatch} ctimes in each." + ) + idx_rand = np.random.permutation(range(nctimes)) + for ib in batches[patch]: + ctimes[patch, "science", ib] = [ + ctimes[patch, "science", None][i] + for i in idx_rand if (i+ib) % nbatch + ] + + # Restrict the inter-obs null splits to the ctimes of the "science" split + for patch in patches: + for inter_obs_split, ib in product(inter_obs_splits, batches[patch]): + ctimes[patch, inter_obs_split, ib] = [ + ct + for ct in ctimes[patch, inter_obs_split, None] + if ct in ctimes[patch, "science", ib] + ] + + # Connect the the atomic map DB + db_con = sqlite3.connect(atom_db) + db_cur = db_con.cursor() + + # TODO: check if query_restrict is channel- or patch-specific + query_restrict = args.query_restrict + + relevant_splits = list(set(["science"] + intra_obs_splits + intra_obs_pair)) # noqa + queries = { + (patch, freq_channel, split_label, ib): fu.get_query_atomics( + freq_channel, ctimes[patch, "science", ib], + split_label=split_label, query_restrict=query_restrict + ) + for ib in batches[patch] + for patch, freq_channel, split_label in product(patches, + freq_channels, + relevant_splits) + } + atomic_metadata = {key: [] for key in queries} + + # Query all atomics used for science, filtering ctimes + for (patch, freq_channel, split_label, ib), query in queries.items(): + if split_label == "science": + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, freq_channel, "science", ib] = [ + (obs_id, wafer) for obs_id, wafer in res + ] + + # Query all atomics used for intra-obs splits + # filtering ctimes and split labels + for (patch, freq_channel, split_label, ib), query in queries.items(): + if split_label != "science": + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, + freq_channel, + split_label, + ib] = [ + (obs_id, wafer) for obs_id, wafer in res + if (obs_id, wafer) in atomic_metadata[patch, + freq_channel, + "science", + ib] + ] + + # Query all atomics used for inter-obs splits + # filtering ctimes w.r.t to the null prop considered + # for the two intra-obs splits to be coadded + if len(intra_obs_pair) != 0: + for patch, freq_channel in product(patches, freq_channels): + for inter_obs_split, intra_obs_split, ib in product(inter_obs_splits, # noqa + intra_obs_pair, # noqa + batches[patch]): # noqa + query = fu.get_query_atomics( + freq_channel, ctimes[patch, inter_obs_split, ib], + split_label=intra_obs_split + ) + res = db_cur.execute(query) + res = res.fetchall() + atomic_metadata[patch, + freq_channel, + inter_obs_split, + intra_obs_split, + ib] = [ + (obs_id, wafer) for obs_id, wafer in res + if (obs_id, wafer) in atomic_metadata[patch, + freq_channel, + "science", + ib] + ] + db_con.close() + + split_labels_all = ["science"] + intra_obs_splits + inter_obs_splits + split_labels_all = list(dict.fromkeys(split_labels_all)) # no duplicates + mpi_shared_list = [(patch, freq_channel, split_label, sim_type) + for patch in patches + for freq_channel in freq_channels + for split_label in split_labels_all + for sim_type in sim_types] + + # Every rank must have the same shared list + mpi_shared_list = comm.bcast(mpi_shared_list, root=0) + task_ids = mpi.distribute_tasks(size, rank, len(mpi_shared_list), + logger=logger) + local_mpi_list = [mpi_shared_list[i] for i in task_ids] + loop_over = [(patch, freq, split, sim_type, ib) + for ib in batches[patch] + for patch, freq, split, sim_type in local_mpi_list] + + + for patch, freq_channel, split_label, sim_type, ib in loop_over: + map_dir = atomic_sim_dir.format( + patch=patch, + freq_channel=freq_labels[freq_channel], + sim_type=sim_type + ) + assert os.path.isdir(map_dir), map_dir + + if not ib: + logger.info(f"Loading atomics for ({patch}, {freq_channel}, " + f"{split_label})" + f" to filter {sim_type}, {split_label}") + + w_list, wmap_list = ([], []) + + if split_label == "science": + tracemalloc.start() + for coadd in intra_obs_pair: + wmap_l, w_l = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, coadd, ib], + freq_labels[freq_channel], map_dir, coadd, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + wmap_list += wmap_l + w_list += w_l + current_gb, peak_gb = [1024**(-3) * c + for c in tracemalloc.get_traced_memory()] + logger.info("Traced Memory for 'science' (Current, Peak): " + f"{current_gb:.2f} GB, {peak_gb:.2f} GB") + tracemalloc.stop() + elif split_label in inter_obs_splits: + for coadd in intra_obs_pair: + wmap_l, w_l = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, split_label, coadd, ib], # noqa + freq_channel, map_dir, coadd, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + wmap_list += wmap_l + w_list += w_l + else: + wmap_list, w_list = fu.get_atomics_maps_list( + None, sim_type, + atomic_metadata[patch, freq_channel, split_label, ib], + freq_channel, map_dir, split_label, + sim_string_format, mfmt=mfmt, pix_type=pix_type, + logger=logger + ) + + if not ib: + logger.info(f"Coadding atomics for ({patch}, {freq_channel}, " + f"{split_label})" + f" to filter {sim_type}") + + map_filtered, weights = bu.coadd_maps( + wmap_list, w_list, pix_type=pix_type, + car_template_map=car_map_template + ) + out_fname = sim_string_format.format( + sim_type=sim_type, + freq_channel=freq_labels[freq_channel] + ).split("/")[-1] + batch_label = "" if ib is None else f"_batch{ib}of{nbatches}" + out_fname = out_fname.replace( + ".fits", + f"_bundle{bundle_id}_{freq_channel}_{split_label}{batch_label}_filtered.fits" # noqa + ) + fu.save_and_plot_map( + map_filtered, out_fname, + coadded_dirs[patch, freq_channel, sim_type], + plot_dirs[patch, freq_channel, sim_type], + pix_type=pix_type + ) + fu.save_and_plot_map( + weights, out_fname.replace(".fits", "_weights.fits"), + coadded_dirs[patch, freq_channel, sim_type], + plot_dirs[patch, freq_channel, sim_type], + pix_type=pix_type, do_plot=False + ) + comm.Barrier() + if rank == 0: + end = time.time() + print(f"Coadding completed in (wall time) {int(end - start)}s.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--config_file", type=str, help="yaml file with configuration." + ) + + args = parser.parse_args() + config = fu.Cfg.from_yaml(args.config_file) + config.update(vars(args)) + + main(config)