From 85925f70eb7ecbc4b11dc5d7542ba452abf84347 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 20 Jun 2026 16:32:04 +0200 Subject: [PATCH 01/11] docs: record the ngmix PSF/shape column migration plan Downstream of CosmoStat/shapepipe#761, which makes the ngmix output columns a single source of truth (true original-PSF fit stored separately, ELL split into scalar G1/G2, ESTIMATOR_COMPONENT_OBJECT grammar). Captures the old->new column map and the sp_validation consumer checklist. Blocked on #761 landing. Co-Authored-By: Claude Opus 4.8 --- docs/ngmix_psf_column_migration.md | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/ngmix_psf_column_migration.md diff --git a/docs/ngmix_psf_column_migration.md b/docs/ngmix_psf_column_migration.md new file mode 100644 index 00000000..d7a75770 --- /dev/null +++ b/docs/ngmix_psf_column_migration.md @@ -0,0 +1,66 @@ +# ngmix PSF / shape column migration (shapepipe → sp_validation) + +**Status:** planning — blocked on [CosmoStat/shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761) +(which lands in #741, the ngmix v2.0 PR). + +shapepipe#761 makes the ngmix shape-measurement output columns a single source of +truth: it restores an explicit fit to the *original* PSF (stored alongside the +metacal-*reconvolved* one), splits the 2-D ellipticity into scalar `G1`/`G2`, and +renames every ngmix column to a consistent `ESTIMATOR_COMPONENT_OBJECT` grammar. +This document records the downstream migration `sp_validation` needs once #761 +(and #741) land. + +## Column map + +`{shear} ∈ {NOSHEAR, 1P, 1M, 2P, 2M}`. The moments branch prefixes `NGMIXm_`. + +| Old (current ngmix_v2 catalogue) | New (#761) | Note | +|---|---|---| +| `NGMIX_ELL_{shear}` (2-vector) | `NGMIX_G1_GAL_{shear}`, `NGMIX_G2_GAL_{shear}` | ⚠ **dim change**: 2-D vector → two scalars | +| `NGMIX_ELL_ERR_{shear}` | `NGMIX_G1_ERR_GAL_{shear}`, `NGMIX_G2_ERR_GAL_{shear}` | ⚠ dim change | +| `NGMIX_T_{shear}` | `NGMIX_T_GAL_{shear}` | gains `_GAL` | +| `NGMIX_T_ERR_{shear}` | `NGMIX_T_ERR_GAL_{shear}` | gains `_GAL` | +| `NGMIX_FLUX_{shear}` / `_ERR` | `NGMIX_FLUX_GAL_{shear}` / `_ERR_GAL` | gains `_GAL` | +| `NGMIX_FLAGS_{shear}` | `NGMIX_FLAGS_GAL_{shear}` | gains `_GAL` | +| `NGMIX_Tpsf_{shear}` | `NGMIX_T_PSF_RECONV_{shear}` | value identical (still the reconvolved-kernel size — the `Tgal/Tpsf` cut) | +| `NGMIX_ELL_PSFo_{shear}_0/_1` | `NGMIX_G1_PSF_ORIG_{shear}`, `NGMIX_G2_PSF_ORIG_{shear}` | ⚠⚠ **value changed** + dim split | +| `NGMIX_T_PSFo_{shear}` | `NGMIX_T_PSF_ORIG_{shear}` | ⚠⚠ **value changed** | +| — | `NGMIX_G1/G2_PSF_RECONV_{shear}` + PSF `_ERR` columns | new | +| `r50`, `r50psf` (+ `_err`) | — | removed (never shipped; use `cs_util.size.T_to_r50(T)`) | +| `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH`, `NGMIX_MOM_FAIL` | unchanged | OBJECT/SHEAR-less metadata | + +### Two changes a mechanical `sed` would silently corrupt +1. **`*_PSF_ORIG` now carries a *true* original-PSF fit**, not the reconvolved-kernel + alias it used to (the shapepipe#749 fix). Any PSF-leakage / α-size analysis reading + these columns **changes value** — re-validate, don't assume. +2. **`NGMIX_ELL_*` → `G1`/`G2` is a dimensionality change**, not a rename: code that + indexes `ELL[..., 0/1]` must switch to the two scalar columns. + +## Consumer sites in sp_validation (@ develop) + +**PSF-leakage / calibration** — the `*_PSFo` value change matters here, re-validate: +- [ ] `config/calibration/mask_v1.X.{2..11}.yaml` (×10) — `NGMIX_ELL_PSFo_NOSHEAR_0/_1` → `NGMIX_G1/G2_PSF_ORIG_NOSHEAR` +- [ ] `src/sp_validation/calibration.py` +- [ ] `src/sp_validation/cat.py` +- [ ] `src/sp_validation/galaxy.py` + +**Size cuts / masking** — `Tpsf` rename, value-safe: +- [ ] `cosmo_inference/scripts/masking.py` — `NGMIX_Tpsf_NOSHEAR` → `NGMIX_T_PSF_RECONV_NOSHEAR` +- [ ] `scripts/apply_alpha_snr_size_bin.py` + +**Scripts / paper figures** — verify each: +- [ ] `scripts/calibration/{calibrate_comprehensive_cat,extract_info,params}.py` +- [ ] `scripts/examples/demo_{calibrate_minimal_cat,comprehensive_to_minimal_cat}.py` +- [ ] `papers/catalog/{2025_09_19_alpha_leakage_correction,hist_mag}.py` +- [ ] `scratch/kilbinger/plot_binned_quantities.py` + +**Unchanged — no action:** `NGMIX_MOM_FAIL`, `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH`. + +## Sequencing +1. shapepipe#761 merges into #741 (ngmix v2.0). +2. #741 merges to shapepipe `develop`; a catalogue is regenerated with the new columns. +3. Apply this migration and validate against the regenerated catalogue — in particular, + confirm the `*_PSF_ORIG` value change behaves as intended for α-leakage, rather than + assuming a name-only swap. + +— Claude on behalf of Cail From 9c552af73b9d9e8ab10db46a9e3123e3f1323bcd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 14:27:56 +0200 Subject: [PATCH 02/11] docs: rewrite column-migration map to the as-shipped v2 grammar The plan committed 2026-06-20 predated the "drop GAL" decision on shapepipe#761: it showed every galaxy column gaining a _GAL token and said nothing about HSM. Rewrite it to the authoritative as-shipped grammar, grounded in the #761 producer source: - galaxy is the implicit object (no _GAL token); ellipticity splits into named scalars NGMIX_G1/G2_{shear}, errors likewise; - NGMIX_Tpsf -> NGMIX_T_PSF_RECONV (value-safe); ELL_PSFo/T_PSFo -> G1/G2/T_PSF_ORIG (value change, shapepipe#749); - HSM E*_{PSF,STAR} -> HSM_G*_{PSF,STAR} (native g, pure rename), SIGMA_*_HSM -> HSM_T_* (units: now T=2sigma^2), FLAG_*_HSM -> HSM_FLAG_*; - spread_model removed; sizes route through cs_util.size; square_size retired. e->g conversion is not needed anywhere (both estimators store g). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01M8BwwtExg961NknUeu5GCv --- docs/ngmix_psf_column_migration.md | 184 ++++++++++++++++++++--------- 1 file changed, 127 insertions(+), 57 deletions(-) diff --git a/docs/ngmix_psf_column_migration.md b/docs/ngmix_psf_column_migration.md index d7a75770..27520d99 100644 --- a/docs/ngmix_psf_column_migration.md +++ b/docs/ngmix_psf_column_migration.md @@ -1,66 +1,136 @@ -# ngmix PSF / shape column migration (shapepipe → sp_validation) +# ShapePipe-v2 column-grammar migration (shapepipe → sp_validation) -**Status:** planning — blocked on [CosmoStat/shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761) -(which lands in #741, the ngmix v2.0 PR). +**Status:** code staged on `migrate/ngmix-psf-column-names` (draft PR +[#201](https://github.com/CosmoStat/sp_validation/pull/201)). The renames, the +σ→T units change and the `spread_model` removal are complete and the test suite +is green against synthetic catalogues carrying the new columns. The one item +that genuinely needs a *regenerated* catalogue — re-validating the `*_PSF_ORIG` +value change — is deferred until [CosmoStat/shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761) +lands in #741 and a v2 catalogue is produced. -shapepipe#761 makes the ngmix shape-measurement output columns a single source of -truth: it restores an explicit fit to the *original* PSF (stored alongside the -metacal-*reconvolved* one), splits the 2-D ellipticity into scalar `G1`/`G2`, and -renames every ngmix column to a consistent `ESTIMATOR_COMPONENT_OBJECT` grammar. -This document records the downstream migration `sp_validation` needs once #761 -(and #741) land. +shapepipe#761 turns the shape-measurement output into **one column grammar for +the whole catalogue**: every estimator names its outputs +`ESTIMATOR_COMPONENT[_ERR][_OBJECT]_SHEAR` (uppercase), stores a single size +`T = 2σ²` sourced from `cs_util.size`, and splits ellipticity into named scalar +components. The galaxy is the implicit default object and carries **no token** +(`NGMIX_G1_NOSHEAR`, never `..._GAL_...`). This document is the authoritative +old→new map for the sp_validation consumer side. + +## What is *not* a value change (safe renames) + +Both shape estimators now store native **`g`** (reduced shear). HSM always stored +galsim's `observed_shape.g1/.g2` under the `E*` names — name and value disagreed; +#761 renamed the columns to match the value (`HSM_G1/G2_*`), moving **zero +numbers**. So there is **no e→g conversion anywhere in this migration** — the ρ/τ +leakage code (`shear_psf_leakage`) reads `g` straight into treecorr, which is what +it was always implicitly validated against. `cs_util.shape` exists for whichever +repo needs a genuine e↔g conversion; sp_validation does not. ## Column map -`{shear} ∈ {NOSHEAR, 1P, 1M, 2P, 2M}`. The moments branch prefixes `NGMIXm_`. +`{shear} ∈ {NOSHEAR, 1P, 1M, 2P, 2M}` (uppercase). Moments branch prefixes `NGMIXm_`. + +### ngmix — galaxy (implicit object, no token) + +| Old | New | Note | +|---|---|---| +| `NGMIX_ELL_{shear}` (2-vec) | `NGMIX_G1_{shear}`, `NGMIX_G2_{shear}` | split into named scalars | +| `NGMIX_ELL_ERR_{shear}` (2-vec) | `NGMIX_G1_ERR_{shear}`, `NGMIX_G2_ERR_{shear}` | split; value-safe | +| `NGMIX_T_{shear}` / `_T_ERR_` | unchanged | no `_GAL` token added | +| `NGMIX_FLUX_{shear}` / `_ERR` | unchanged | | +| `NGMIX_FLAGS_{shear}`, `NGMIX_SNR_{shear}` | unchanged | | +| `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH` | unchanged | OBJECT/SHEAR-less metadata | + +### ngmix — reconvolved PSF (metacal kernel; value-safe rename) + +| Old | New | Note | +|---|---|---| +| `NGMIX_Tpsf_{shear}` | `NGMIX_T_PSF_RECONV_{shear}` | same value (the `T/Tpsf` size-ratio cut) | + +### ngmix — original PSF (⚠ **value change** — shapepipe#749 fix) + +`*_PSF_ORIG` now carries a *true* fit to the original image PSF, no longer the +reconvolved-kernel alias the old `ELL_PSFo`/`T_PSFo` columns silently held. A +mechanical rename is correct for the *names*; the *numbers* move, so any size-ratio +or leakage cut fed by these must be re-validated against a regenerated catalogue. + +| Old | New | +|---|---| +| `NGMIX_ELL_PSFo_{shear}_0` (or `[:, 0]`) | `NGMIX_G1_PSF_ORIG_{shear}` | +| `NGMIX_ELL_PSFo_{shear}_1` (or `[:, 1]`) | `NGMIX_G2_PSF_ORIG_{shear}` | +| `NGMIX_T_PSFo_{shear}` | `NGMIX_T_PSF_ORIG_{shear}` | + +### HSM — star / PSF validation catalogue -| Old (current ngmix_v2 catalogue) | New (#761) | Note | +`SIGMA_*_HSM` → `HSM_T_*` is a **units change**: the new column stores the area +`T = 2σ²` directly (the producer applied `sigma_to_T`). Downstream must stop +squaring — the stored value is already `T`. + +| Old | New | Note | |---|---|---| -| `NGMIX_ELL_{shear}` (2-vector) | `NGMIX_G1_GAL_{shear}`, `NGMIX_G2_GAL_{shear}` | ⚠ **dim change**: 2-D vector → two scalars | -| `NGMIX_ELL_ERR_{shear}` | `NGMIX_G1_ERR_GAL_{shear}`, `NGMIX_G2_ERR_GAL_{shear}` | ⚠ dim change | -| `NGMIX_T_{shear}` | `NGMIX_T_GAL_{shear}` | gains `_GAL` | -| `NGMIX_T_ERR_{shear}` | `NGMIX_T_ERR_GAL_{shear}` | gains `_GAL` | -| `NGMIX_FLUX_{shear}` / `_ERR` | `NGMIX_FLUX_GAL_{shear}` / `_ERR_GAL` | gains `_GAL` | -| `NGMIX_FLAGS_{shear}` | `NGMIX_FLAGS_GAL_{shear}` | gains `_GAL` | -| `NGMIX_Tpsf_{shear}` | `NGMIX_T_PSF_RECONV_{shear}` | value identical (still the reconvolved-kernel size — the `Tgal/Tpsf` cut) | -| `NGMIX_ELL_PSFo_{shear}_0/_1` | `NGMIX_G1_PSF_ORIG_{shear}`, `NGMIX_G2_PSF_ORIG_{shear}` | ⚠⚠ **value changed** + dim split | -| `NGMIX_T_PSFo_{shear}` | `NGMIX_T_PSF_ORIG_{shear}` | ⚠⚠ **value changed** | -| — | `NGMIX_G1/G2_PSF_RECONV_{shear}` + PSF `_ERR` columns | new | -| `r50`, `r50psf` (+ `_err`) | — | removed (never shipped; use `cs_util.size.T_to_r50(T)`) | -| `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH`, `NGMIX_MOM_FAIL` | unchanged | OBJECT/SHEAR-less metadata | - -### Two changes a mechanical `sed` would silently corrupt -1. **`*_PSF_ORIG` now carries a *true* original-PSF fit**, not the reconvolved-kernel - alias it used to (the shapepipe#749 fix). Any PSF-leakage / α-size analysis reading - these columns **changes value** — re-validate, don't assume. -2. **`NGMIX_ELL_*` → `G1`/`G2` is a dimensionality change**, not a rename: code that - indexes `ELL[..., 0/1]` must switch to the two scalar columns. - -## Consumer sites in sp_validation (@ develop) - -**PSF-leakage / calibration** — the `*_PSFo` value change matters here, re-validate: -- [ ] `config/calibration/mask_v1.X.{2..11}.yaml` (×10) — `NGMIX_ELL_PSFo_NOSHEAR_0/_1` → `NGMIX_G1/G2_PSF_ORIG_NOSHEAR` -- [ ] `src/sp_validation/calibration.py` -- [ ] `src/sp_validation/cat.py` -- [ ] `src/sp_validation/galaxy.py` - -**Size cuts / masking** — `Tpsf` rename, value-safe: -- [ ] `cosmo_inference/scripts/masking.py` — `NGMIX_Tpsf_NOSHEAR` → `NGMIX_T_PSF_RECONV_NOSHEAR` -- [ ] `scripts/apply_alpha_snr_size_bin.py` - -**Scripts / paper figures** — verify each: -- [ ] `scripts/calibration/{calibrate_comprehensive_cat,extract_info,params}.py` -- [ ] `scripts/examples/demo_{calibrate_minimal_cat,comprehensive_to_minimal_cat}.py` -- [ ] `papers/catalog/{2025_09_19_alpha_leakage_correction,hist_mag}.py` -- [ ] `scratch/kilbinger/plot_binned_quantities.py` - -**Unchanged — no action:** `NGMIX_MOM_FAIL`, `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH`. - -## Sequencing -1. shapepipe#761 merges into #741 (ngmix v2.0). -2. #741 merges to shapepipe `develop`; a catalogue is regenerated with the new columns. -3. Apply this migration and validate against the regenerated catalogue — in particular, - confirm the `*_PSF_ORIG` value change behaves as intended for α-leakage, rather than - assuming a name-only swap. +| `E1_PSF_HSM` / `E2_PSF_HSM` | `HSM_G1_PSF` / `HSM_G2_PSF` | native `g`, pure rename | +| `E1_STAR_HSM` / `E2_STAR_HSM` | `HSM_G1_STAR` / `HSM_G2_STAR` | native `g`, pure rename | +| `SIGMA_PSF_HSM` | `HSM_T_PSF` | **units:** now `T = 2σ²`, an area | +| `SIGMA_STAR_HSM` | `HSM_T_STAR` | **units:** now `T = 2σ²`, an area | +| `FLAG_PSF_HSM` | `HSM_FLAG_PSF` | singular `FLAG` for HSM | +| `FLAG_STAR_HSM` | `HSM_FLAG_STAR` | singular `FLAG` for HSM | + +The `piff_T` / DES block in `cat_config.yaml` uses Piff columns, **not** this +grammar — left untouched. + +### `spread_model` — removed + +shapepipe no longer writes `SPREAD_MODEL` / `SPREADERR_MODEL` / `SPREAD_CLASS` +(Axel's review: these must never be cut on). The `do_spread_model` branch, +`classification_galaxy_base`'s parameter, and `catalog.match_spread_class` are +removed; star/galaxy classification uses the size-based path. + +## Size arithmetic routes through `cs_util.size` + +`T = 2σ²` is the only stored size. Anything needing σ / FWHM / r50 calls +`cs_util.size` (`T_to_fwhm`, `T_to_r50`, …) — never inline `SIGMA ** 2` or +`sqrt(T)`. The pre-migration `papers/catalog/2025_12_*` hand-rolled +`T = SIGMA_*_HSM ** 2` (a factor-of-2 error even then, since `SIGMA_*_HSM` never +truly held σ) now reads `HSM_T_*` directly. + +## `square_size` is retired + +The σ→T change makes the old squaring dead: `HSM_T_*` (and DES's `piff_T`) already +hold `T`, so nothing squares. The per-dataset `square_size:` flags in +`cat_config.yaml` are dropped, the `not_square_size` list in `rho_tau.py` is +removed, and both param builders (`rho_tau.get_params_rho_tau`, +`cosmo_val/compute_theory_cov.py`) set `square_size = False`. The `square_size` +argument to `shear_psf_leakage`'s `build_cat_to_compute_{rho,tau}` is now always +`False`; passing a `T`-column with no squaring is correct independently of that +repo's own migration. + +## Consumer sites in sp_validation + +- **Core** — `src/sp_validation/{galaxy.py, calibration.py, catalog.py}`. +- **Configs** — `cosmo_val/cat_config.yaml` (HSM blocks; DES/piff spared); + `config/calibration/mask_v1.X.*.yaml` ×10 (`NGMIX_ELL_PSFo_NOSHEAR_0/_1`). +- **rho/τ + covariance** — `src/sp_validation/rho_tau.py`, + `src/sp_validation/cosmo_val/psf_systematics.py`, `cosmo_val/compute_theory_cov.py`, + `src/sp_validation/glass_mock.py`. +- **Scripts** — `scripts/calibration/{extract_info,params,calibrate_comprehensive_cat}.py`, + `scripts/apply_alpha_snr_size_bin.py`, `scripts/examples/demo_*.py`. +- **Papers** — `papers/catalog/2025_12_*`, `papers/catalog/{hist_mag,2025_09_19_alpha_leakage_correction}.py`, + `papers/harmonic/2025_09_11_psf_leakage_cell.py`. +- **Tests** — `src/sp_validation/tests/{test_calibration,test_cosmo_val}.py` + (synthetic catalogues + configs updated in lock-step; this is the migration's + internal-consistency check). + +## Deferred / coordinated + +- **`*_PSF_ORIG` value re-validation** — blocked on a #761-regenerated catalogue. + The rename is staged; the "does the size-ratio / leakage cut still behave" + check waits for real v2 data. +- **`NGMIX_MOM_FAIL`** (read at `galaxy.py`) was not found in the #761 producer + code; left as-is pending confirmation against a regenerated header. +- **galsim family** (`GALSIM_*` reads in `calibration.py`, `galaxy.py`) is renamed + on the producer side too but is out of this constitution's ngmix+HSM scope and + is left untouched here — flagged for a follow-up if the galsim path is live. +- **`shear_psf_leakage`** ρ/τ internals are Sacha Guerrini's separate PR; this + migration only sets the `square_size=False` interface and passes `T`-columns. — Claude on behalf of Cail From 23b81220442b9ca9bdfd08e2756dd5608199f182 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 14:28:37 +0200 Subject: [PATCH 03/11] refactor(src): adopt ShapePipe-v2 column grammar in the package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consumer half of shapepipe#761. Every shape-column read in the package moves onto the one grammar: - metacal (calibration.py): galaxy ellipticity read as named scalars NGMIX_G1/G2_{shear} and errors NGMIX_G1/G2_ERR_{shear} (was packed NGMIX_ELL_*[:, c]); NGMIX_Tpsf -> NGMIX_T_PSF_RECONV. The obsolete col_2d flag (packed-vs-split ellipticity) is removed from the metacal ctor and get_variance_ivweights — v2 is always named scalars. - galaxy.py: NGMIX_ELL_PSFo[:, 0] -> NGMIX_G1_PSF_ORIG scalar; the spread_model branch/param dropped (shapepipe stopped writing it) — classification falls back to the size-based path. - catalog.py: match_spread_class deleted (SPREAD_CLASS gone); match_subsample takes two scalar keys (g1_key, g2_key) instead of a packed ell_key; check_invalid drops the now-meaningless comp arg. - glass_mock.py: HSM E1/E2_PSF_HSM -> HSM_G1/G2_PSF. - rho_tau.py: sizes are always T=2sigma^2 now, so square_size is False and the not_square_size list is gone. - tests: synthetic catalogues + configs rebuilt on the v2 grammar in lock-step (the migration's internal-consistency check). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01M8BwwtExg961NknUeu5GCv --- src/sp_validation/calibration.py | 61 +++++++-------------- src/sp_validation/catalog.py | 48 +++------------- src/sp_validation/galaxy.py | 14 +---- src/sp_validation/glass_mock.py | 4 +- src/sp_validation/rho_tau.py | 12 +--- src/sp_validation/tests/test_calibration.py | 14 ++--- src/sp_validation/tests/test_cosmo_val.py | 36 ++++++------ 7 files changed, 58 insertions(+), 131 deletions(-) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index b61e7261..971f05d9 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -182,10 +182,12 @@ def fill_cat_gal(cat_gal, dat, g_uncorr, gal_metacal, mask1, mask2, purpose="wei cat_gal["R_g22"] = gal_metacal.R22 cat_gal["NGMIX_T_NOSHEAR"] = sp_cat.get_col(dat, "NGMIX_T_NOSHEAR", mask1, mask2) - cat_gal["NGMIX_Tpsf_NOSHEAR"] = sp_cat.get_col( - dat, "NGMIX_Tpsf_NOSHEAR", mask1, mask2 + cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] = sp_cat.get_col( + dat, "NGMIX_T_PSF_RECONV_NOSHEAR", mask1, mask2 + ) + cat_gal["size_ratio"] = ( + cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] ) - cat_gal["size_ratio"] = cat_gal["NGMIX_T_NOSHEAR"] / cat_gal["NGMIX_Tpsf_NOSHEAR"] cat_gal["snr"] = sp_cat.get_col( dat, "NGMIX_FLUX_NOSHEAR", mask1, mask2 @@ -322,8 +324,8 @@ def get_alpha_leakage_per_object(cat_gal, num_bins, weight_type="des"): """ assert weight_type in ["des", "iv"], "weight_type must be either 'des' or 'iv'" # Compute the size ratio - size_ratio = cat_gal["NGMIX_Tpsf_NOSHEAR"] / ( - cat_gal["NGMIX_T_NOSHEAR"] + cat_gal["NGMIX_Tpsf_NOSHEAR"] + size_ratio = cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] / ( + cat_gal["NGMIX_T_NOSHEAR"] + cat_gal["NGMIX_T_PSF_RECONV_NOSHEAR"] ) df_gal = pd.DataFrame( @@ -715,9 +717,6 @@ class metacal: sigma_eps : float, optional ellipticity dispersion (one component) for computation of weights; default is 0.34 - col_2d : bool, optional - if `True` (default, ellipticity in one 2D column; - if `False`, ellipticity in two columns ELL_0, ELL_1 verbose : bool, optional, default=False verbose output if True @@ -737,7 +736,6 @@ def __init__( size_corr_ell=True, global_R_weight=None, sigma_eps=0.34, - col_2d=True, verbose=False, ): @@ -761,7 +759,6 @@ def __init__( self._global_R_weight = global_R_weight self._sigma_eps = sigma_eps - self._col_2d = col_2d self._verbose = verbose @@ -803,7 +800,7 @@ def _read_data(self, data, mask): print("FHP/MK hack using p1 PSF for ns in cuts") indices = np.where(mask)[0] - col_1p = f"{self._prefix}_Tpsf_1P" + col_1p = f"{self._prefix}_T_PSF_RECONV_1P" new_psf = data[col_1p][indices] # Overwriting incorrect no-shear PSF size to the one from 1p @@ -830,32 +827,24 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): dict_tmp["flag"] = masked_data[f"{self._prefix}_FLAGS_{name_shear}"] - if self._col_2d: - # Ellipticity in one 2D column - for comp in (0, 1): - dict_tmp[f"g{comp + 1}"] = masked_data[ - f"{self._prefix}_ELL_{name_shear}" - ][:, comp] - else: - # Ellipcitiy in two different columns - for comp in (0, 1): - dict_tmp[f"g{comp + 1}"] = masked_data[ - f"{self._prefix}_ELL_{name_shear}_{comp}" - ] + # Ellipticity in named scalar components (ShapePipe-v2 grammar) + for comp in (0, 1): + dict_tmp[f"g{comp + 1}"] = masked_data[ + f"{self._prefix}_G{comp + 1}_{name_shear}" + ] for key in ("flux", "flux_err", "T", "T_err"): dict_tmp[key] = masked_data[ f"{self._prefix}_{key.upper()}_{name_shear}" ] - dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_Tpsf_{name_shear}"] + dict_tmp["Tpsf"] = masked_data[f"{self._prefix}_T_PSF_RECONV_{name_shear}"] ns["C11"], ns["C22"], ns["w"] = self.get_variance_ivweights( masked_data, self._sigma_eps, self._prefix, mask=None, - col_2d=self._col_2d, ) self._n_input = len(masked_data) @@ -870,7 +859,7 @@ def _read_data_ngmix(self, masked_data, m1, p1, m2, p2, ns): return m1, p1, m2, p2, ns @staticmethod - def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=True): + def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None): """Get Variance IVWEIGHTS. Compute variance and inverse-variance weights. @@ -886,9 +875,6 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=Tr mask : list, optional indicates valid objects with ``True`` values; default is ``None`` = use all objects type has to be bool - col_2d : bool, optional - if ``True`` (default), ellipticity is given in single 2D column; - if ``False``, ellipticity is expected in two 1D columns. Returns ------- @@ -900,20 +886,11 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None, col_2d=Tr weight """ + C11 = data[f"{prefix}_G1_ERR_NOSHEAR"] + C22 = data[f"{prefix}_G2_ERR_NOSHEAR"] if mask is not None: - if col_2d: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 0][mask] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 1][mask] - else: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR_0"][mask] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR_1"][mask] - else: - if col_2d: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 0] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR"][:, 1] - else: - C11 = data[f"{prefix}_ELL_ERR_NOSHEAR_0"] - C22 = data[f"{prefix}_ELL_ERR_NOSHEAR_1"] + C11 = C11[mask] + C22 = C22[mask] iv_w = 1 / (2 * sigma_eps**2 + C11 + C22) diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index 4fe45476..fd90ff1a 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -217,7 +217,7 @@ def check_matching( return ind, mask_area_tiles, n_tot -def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): +def check_invalid(dd, key, val, stats_file, name=None, verbose=False): """Check invalid objects. Check whether objects have invalid values. @@ -227,9 +227,7 @@ def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): dd : dict catalog key : list - key names of columns to check - comp : array of int - components for above columns + key names of (scalar) columns to check val : array of float values for above columns indicating invalid entries stats_file : file handler @@ -245,7 +243,7 @@ def check_invalid(dd, key, comp, val, stats_file, name=None, verbose=False): n_all = len(dd) for i in range(len(key)): - w = dd[key[i]][:, comp[i]] == val[i] + w = dd[key[i]] == val[i] n_inv_psf = len(np.where(w)[0]) msg = "Invalid {} found for {}/{} = {:.1g}% objects".format( name[i], n_inv_psf, n_all, n_inv_psf / n_all @@ -258,7 +256,8 @@ def match_subsample( ind, mask, pos_key, - ell_key, + g1_key, + g2_key, n_ref, stats_file, verbose=False, @@ -277,8 +276,8 @@ def match_subsample( boolean mask pos_key : list key names for position columns - ell_key : str - key name for ellipticity column + g1_key, g2_key : str + key names for the two scalar ellipticity components n_ref : int reference number of objects stats_file : file handler @@ -302,42 +301,13 @@ def match_subsample( ra = dd[pos_key[0]][ind][mask] dec = dd[pos_key[1]][ind][mask] - g1 = dd[ell_key][:, 0][ind][mask] - g2 = dd[ell_key][:, 1][ind][mask] + g1 = dd[g1_key][ind][mask] + g2 = dd[g2_key][ind][mask] g = np.array([g1, g2]) return ra, dec, g -def match_spread_class(dd, ind, mask, stats_file, n_ref, verbose=False): - """Match spread class. - - Match - """ - tot_star = n_ref - tot_as_star = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 0)[0]) - tot_as_gal = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 1)[0]) - tot_as_other = len(np.where(dd["SPREAD_CLASS"][ind][mask] == 2)[0]) - - msg = ( - "Number of stars selected as star (SPREAD_CLASS=0) = " - + f"{tot_as_star}/{tot_star} = {tot_as_star / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - msg = ( - "Number of stars selected as galaxy (SPREAD_CLASS=1) = " - + f"{tot_as_gal}/{tot_star} = {tot_as_gal / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - msg = ( - "Number of stars selected as other (SPREAD_CLASS=2) = " - + f"{tot_as_other}/{tot_star} = {tot_as_other / tot_star * 100:.1f}%" - ) - io.print_stats(msg, stats_file, verbose=verbose) - - def match_stars2(ra_gal, dec_gal, ra_star, dec_star, thresh=0.0002): """Add docstring. diff --git a/src/sp_validation/galaxy.py b/src/sp_validation/galaxy.py index 616f0ad5..fb346782 100644 --- a/src/sp_validation/galaxy.py +++ b/src/sp_validation/galaxy.py @@ -141,23 +141,12 @@ def classification_galaxy_base( gal_mag_faint=26, flags_keep=None, n_epoch_min=1, - do_spread_model=True, ): """Classification Galaxy Base. Return mask corresponding to basic classification for galaxies. """ - if do_spread_model: - # spread model class, add two times the uncertainty to be conservative - sm_classif = dd["SPREAD_MODEL"] + 2 * dd["SPREADERR_MODEL"] - cut_sm = sm_classif > 0.0035 - - cut_sm_all = cut_sm & (dd["SPREAD_MODEL"] > 0) & (dd["SPREAD_MODEL"] < 0.03) - else: - # Do not use spread model - cut_sm_all = True - # SExtractor flags # Keep some flags if specified if flags_keep: @@ -181,7 +170,6 @@ def classification_galaxy_base( cut_common = ( cut_overlap & cut_flags - & cut_sm_all & (dd["MAG_AUTO"] <= gal_mag_faint) & (dd["MAG_AUTO"] >= gal_mag_bright) & (dd["IMAFLAGS_ISO"] == 0) @@ -204,7 +192,7 @@ def classification_galaxy_ngmix( m_gal_ngmix = ( cut_common & (dd["NGMIX_MCAL_FLAGS"] == 0) - & (dd["NGMIX_ELL_PSFo_NOSHEAR"][:, 0] != -10) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"] != -10) & (dd["NGMIX_MOM_FAIL"] == 0) ) diff --git a/src/sp_validation/glass_mock.py b/src/sp_validation/glass_mock.py index 7ba0057b..de8312c7 100644 --- a/src/sp_validation/glass_mock.py +++ b/src/sp_validation/glass_mock.py @@ -493,8 +493,8 @@ def compute_leakage_harmony(cat, cat_star, nside=1024, lmin=8, n_bins=32): e1 = cat["e1"] e2 = cat["e2"] - e1_psf = cat_star["E1_PSF_HSM"] - e2_psf = cat_star["E2_PSF_HSM"] + e1_psf = cat_star["HSM_G1_PSF"] + e2_psf = cat_star["HSM_G2_PSF"] ra = cat["ra"] dec = cat["dec"] diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 0a311432..5cd43973 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -10,15 +10,6 @@ # `from sp_validation.rho_tau import SquareRootScale` keeps working. from sp_validation.plots import SquareRootScale # noqa: F401 -not_square_size = [ - "DES", - "SP_v1.3_LFmask_8k", - "SP_v1.3_LFmask_8k_no_alpha", - "SP_v1.3_LFmask_8k_li_2024", - "SP_v1.3_LFmask_8k_SN8", - "SP_v1.3_LFmask_8k_F2", -] - def _extract_xip(correlations): """Return flattened array of xip values from a list of correlations.""" @@ -49,7 +40,8 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - params["square_size"] = survey not in not_square_size + # ShapePipe-v2 stores every size as T = 2σ² (an area); nothing to square. + params["square_size"] = False if survey != "DES": params["PSF_flag"] = cat["psf"]["PSF_flag"] params["star_flag"] = cat["psf"]["star_flag"] diff --git a/src/sp_validation/tests/test_calibration.py b/src/sp_validation/tests/test_calibration.py index b1b0dd26..40621c19 100644 --- a/src/sp_validation/tests/test_calibration.py +++ b/src/sp_validation/tests/test_calibration.py @@ -259,20 +259,20 @@ def _build_ngmix_catalog(slope_11=2.0, slope_22=3.0, step=0.01): cols = {} for var in _VARIANTS: dg1, dg2 = shifts[var] - ell = np.zeros((n, 2)) - ell[:, 0] = g1_base + dg1 - ell[:, 1] = g2_base + dg2 - cols[f"NGMIX_ELL_{var}"] = ell + # ShapePipe-v2 grammar: ellipticity in named scalar components. + cols[f"NGMIX_G1_{var}"] = np.full(n, g1_base + dg1) + cols[f"NGMIX_G2_{var}"] = np.full(n, g2_base + dg2) cols[f"NGMIX_FLAGS_{var}"] = np.zeros(n, dtype=int) cols[f"NGMIX_FLUX_{var}"] = np.full(n, 50.0) cols[f"NGMIX_FLUX_ERR_{var}"] = np.full(n, 1.0) - # T / Tpsf = 1.0 -> rel_size inside [0.5, 3.0]. + # T / T_PSF_RECONV = 1.0 -> rel_size inside [0.5, 3.0]. cols[f"NGMIX_T_{var}"] = np.full(n, 1.0) cols[f"NGMIX_T_ERR_{var}"] = np.full(n, 0.1) - cols[f"NGMIX_Tpsf_{var}"] = np.full(n, 1.0) + cols[f"NGMIX_T_PSF_RECONV_{var}"] = np.full(n, 1.0) # Per-component ellipticity errors -> inverse-variance weights. - cols["NGMIX_ELL_ERR_NOSHEAR"] = np.full((n, 2), 0.25) + cols["NGMIX_G1_ERR_NOSHEAR"] = np.full(n, 0.25) + cols["NGMIX_G2_ERR_NOSHEAR"] = np.full(n, 0.25) return Table(cols), n diff --git a/src/sp_validation/tests/test_cosmo_val.py b/src/sp_validation/tests/test_cosmo_val.py index 0a618e9a..f50992d4 100644 --- a/src/sp_validation/tests/test_cosmo_val.py +++ b/src/sp_validation/tests/test_cosmo_val.py @@ -418,14 +418,14 @@ def _write_synthetic_catalogs( { "RA": rng.uniform(*ra_range, n_star), "Dec": rng.uniform(*dec_range, n_star), - "E1_PSF_HSM": rng.normal(0, 0.03, n_star), - "E2_PSF_HSM": rng.normal(0, 0.03, n_star), - "E1_STAR_HSM": rng.normal(0, 0.03, n_star), - "E2_STAR_HSM": rng.normal(0, 0.03, n_star), - "SIGMA_PSF_HSM": rng.uniform(0.4, 0.6, n_star), - "SIGMA_STAR_HSM": rng.uniform(0.4, 0.6, n_star), - "FLAG_PSF_HSM": np.zeros(n_star, dtype=int), - "FLAG_STAR_HSM": np.zeros(n_star, dtype=int), + "HSM_G1_PSF": rng.normal(0, 0.03, n_star), + "HSM_G2_PSF": rng.normal(0, 0.03, n_star), + "HSM_G1_STAR": rng.normal(0, 0.03, n_star), + "HSM_G2_STAR": rng.normal(0, 0.03, n_star), + "HSM_T_PSF": rng.uniform(0.4, 0.6, n_star), + "HSM_T_STAR": rng.uniform(0.4, 0.6, n_star), + "HSM_FLAG_PSF": np.zeros(n_star, dtype=int), + "HSM_FLAG_STAR": np.zeros(n_star, dtype=int), } ).write(star_path, overwrite=True) @@ -449,8 +449,8 @@ def _write_synthetic_catalogs( "path": "star.fits", "ra_col": "RA", "dec_col": "Dec", - "e1_col": "E1_PSF_HSM", - "e2_col": "E2_PSF_HSM", + "e1_col": "HSM_G1_PSF", + "e2_col": "HSM_G2_PSF", } version_cfg = { "subdir": str(cat_dir), @@ -464,14 +464,14 @@ def _write_synthetic_catalogs( "hdu": 1, "ra_col": "RA", "dec_col": "Dec", - "e1_PSF_col": "E1_PSF_HSM", - "e2_PSF_col": "E2_PSF_HSM", - "e1_star_col": "E1_STAR_HSM", - "e2_star_col": "E2_STAR_HSM", - "PSF_size": "SIGMA_PSF_HSM", - "star_size": "SIGMA_STAR_HSM", - "PSF_flag": "FLAG_PSF_HSM", - "star_flag": "FLAG_STAR_HSM", + "e1_PSF_col": "HSM_G1_PSF", + "e2_PSF_col": "HSM_G2_PSF", + "e1_star_col": "HSM_G1_STAR", + "e2_star_col": "HSM_G2_STAR", + "PSF_size": "HSM_T_PSF", + "star_size": "HSM_T_STAR", + "PSF_flag": "HSM_FLAG_PSF", + "star_flag": "HSM_FLAG_STAR", } config_data = { From ba7b663914d7d2195c7a246c00c72a806d59be19 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 14:28:51 +0200 Subject: [PATCH 04/11] refactor(config): rename HSM/ngmix columns to v2 grammar, drop square_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cat_config.yaml: every HSM block onto the v2 grammar — E*_{PSF,STAR}_HSM -> HSM_G*_{PSF,STAR}, SIGMA_*_HSM / T_*_HSM -> HSM_T_*, FLAG_*_HSM -> HSM_FLAG_*. The DES/piff block is untouched (Piff isn't this grammar). The dead per-dataset square_size flags are dropped: sizes are always T=2sigma^2 now, so nothing squares (this also fixes a latent factor error in the SP_v1.3 block, which squared its already-T T_PSF_HSM). - mask_v1.X.{2..11}.yaml: NGMIX_ELL_PSFo_NOSHEAR_0/_1 -> NGMIX_G1/G2_PSF_ORIG_NOSHEAR. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01M8BwwtExg961NknUeu5GCv --- config/calibration/mask_v1.X.10.yaml | 4 +- config/calibration/mask_v1.X.11.yaml | 4 +- config/calibration/mask_v1.X.2.yaml | 4 +- config/calibration/mask_v1.X.3.yaml | 4 +- config/calibration/mask_v1.X.4.yaml | 4 +- config/calibration/mask_v1.X.5.yaml | 4 +- config/calibration/mask_v1.X.6.yaml | 4 +- config/calibration/mask_v1.X.7.yaml | 4 +- config/calibration/mask_v1.X.8.yaml | 4 +- config/calibration/mask_v1.X.9.yaml | 4 +- cosmo_val/cat_config.yaml | 471 +++++++++++++-------------- 11 files changed, 242 insertions(+), 269 deletions(-) diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 6a926216..1f002fcf 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml index 9562d524..d224be34 100644 --- a/config/calibration/mask_v1.X.11.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index b50e2ed2..81e98208 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index 9f8442ff..feaf582f 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 077a9f6d..8c7dd186 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index e2d9e871..bce8b6d2 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index e01d3d3f..72ed62b7 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index bbe52699..7add0cc0 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index 2071f651..418cab72 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.9.yaml b/config/calibration/mask_v1.X.9.yaml index 95234bcd..077ec7e8 100644 --- a/config/calibration/mask_v1.X.9.yaml +++ b/config/calibration/mask_v1.X.9.yaml @@ -48,11 +48,11 @@ dat: value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/cosmo_val/cat_config.yaml b/cosmo_val/cat_config.yaml index 0b949372..cc1326df 100644 --- a/cosmo_val/cat_config.yaml +++ b/cosmo_val/cat_config.yaml @@ -12,7 +12,6 @@ DES: sigma_e: 0.29 psf: PSF_size: piff_T - square_size: false star_size: obs_T hdu: 1 path: psf_y3a1-v29.fits @@ -52,32 +51,31 @@ SP_axel_v0.0: n_psf: 0.47 sigma_e: 0.3 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: star_cat.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: shapepipe_1500_goldshape_v1.fits w_col: w e1_col: g1 - e1_PSF_col: E1_PSF_HSM + e1_PSF_col: HSM_G1_PSF e2_col: g2 - e2_PSF_col: E2_PSF_HSM + e2_PSF_col: HSM_G2_PSF star: ra_col: RA dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM + e1_col: HSM_G1_STAR + e2_col: HSM_G2_STAR path: star_cat.fits SP_v0.1.1: subdir: /n17data/mkilbing/astro/data/CFIS/v0.0 @@ -87,18 +85,18 @@ SP_v0.1.1: ls: dashdot marker: v psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: star_cat.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_extended_2022_v0.0_recon_theli_8192.fits @@ -109,8 +107,8 @@ SP_v0.1.1: star: ra_col: RA dec_col: DEC - e1_col: E1_STAR_HSM - e2_col: E2_STAR_HSM + e1_col: HSM_G1_STAR + e2_col: HSM_G2_STAR path: star_cat.fits SP_v1.3: subdir: /n17data/mkilbing/astro/data/CFIS/v1.0/ShapePipe @@ -120,19 +118,18 @@ SP_v1.3: ls: solid marker: p psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: T_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: T_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_star_2022_v1.3.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -162,19 +159,18 @@ SP_v1.3.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2022_v1.3.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits @@ -206,19 +202,18 @@ SP_v1.4.5: sigma_e: 0.39740419232655594 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/sguerrini/unions_shapepipe_cut_struc_2024_v1.4.5_rerun.fits @@ -249,19 +244,18 @@ SP_v1.4.5.A: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: shapepipe_SPv1.fits @@ -290,19 +284,18 @@ SP_v1.4.5_bright: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_bright.fits @@ -331,19 +324,18 @@ SP_v1.4.5_faint: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_faint.fits @@ -373,19 +365,18 @@ SP_v1.4.6_glass_mock: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n09data/guerrini/glass_mock_v1.4.6/results/unions_glass_sim_00001_4096.fits @@ -415,19 +406,18 @@ SP_v1.4.5_intermediate: n_psf: 0.752316232272063 sigma_e: 0.30961528707207325 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: unions_shapepipe_cut_struc_2024_v1.4.5_intermediate.fits @@ -456,19 +446,18 @@ SP_v1.4.6: n_psf: 0.752316232272063 sigma_e: 0.379587601488189 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits @@ -501,19 +490,18 @@ SP_v1.4.6.3: n_psf: 0.752316232272063 sigma_e: 0.3783979556382064 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -545,19 +533,18 @@ SP_v1.3.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2022_v1.3.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.3.6/unions_shapepipe_cut_struc_2022_v1.3.6.fits @@ -589,19 +576,18 @@ SP_v1.4.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 0.92 path: v1.4.6/unions_shapepipe_cut_struc_2024_v1.4.6.fits @@ -633,19 +619,18 @@ SP_v1.4.6.3_B: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -677,19 +662,18 @@ SP_v1.4.6.3_C: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -721,19 +705,18 @@ SP_v1.4.6.3_A: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.3/unions_shapepipe_cut_struc_2024_v1.4.6.3.fits @@ -765,19 +748,18 @@ SP_v1.4.6_ecut07: sigma_e: 0.34187904441900674 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.6_ecut07.fits @@ -809,19 +791,18 @@ SP_v1.4.6.1: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.6.1/unions_shapepipe_cut_struc_2024_v1.4.6.1.fits @@ -854,19 +835,18 @@ SP_v1.4.7: sigma_e: 0.3797419081703805 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.7/unions_shapepipe_cut_struc_2024_v1.4.7.fits @@ -900,19 +880,18 @@ SP_v1.4.8: sigma_e: 0.3797397825805285 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_starhalo_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.4.8/unions_shapepipe_cut_struc_2024_v1.4.8.fits @@ -944,19 +923,18 @@ SP_v1.4.11.2: sigma_e: 0.380499698062307 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.2/unions_shapepipe_cut_struc_2024_v1.4.11.2.fits @@ -988,19 +966,18 @@ SP_v1.4.11.3: sigma_e: 0.380499698062307 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/UNIONS/WL/v1.4.x/v1.4.11.3/unions_shapepipe_cut_struc_2024_v1.4.11.3.fits @@ -1032,19 +1009,18 @@ SP_v1.4.11.3_ecut07: sigma_e: 0.34058513153783426 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: /n17data/UNIONS/WL/v1.4.x/unions_shapepipe_psf_2024_v1.4.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: /n17data/cdaley/unions/pure_eb/results/ecut/SP_v1.4.11.3_ecut07.fits @@ -1165,19 +1141,18 @@ SP_v1.4_LFmask_8k: n_psf: 0.5434016250405327 sigma_e: 0.31509572849714534 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -1206,19 +1181,18 @@ SP_v1.4_LFmask_8k_noalpha: n_psf: 0.5434016250405327 sigma_e: 0.31509572849714534 psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_conv_2022_v1.4.0_mtheli8k.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR label: SP_LFmask_psf shear: R: 1.0 @@ -1255,19 +1229,18 @@ SP_v1.6.6: sigma_e: 0.379587601488189 mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_footprint_nside_4096.fits psf: - PSF_flag: FLAG_PSF_HSM - PSF_size: SIGMA_PSF_HSM - square_size: true - star_flag: FLAG_STAR_HSM - star_size: SIGMA_STAR_HSM + PSF_flag: HSM_FLAG_PSF + PSF_size: HSM_T_PSF + star_flag: HSM_FLAG_STAR + star_size: HSM_T_STAR hdu: 1 path: unions_shapepipe_psf_2024_v1.6.a.fits ra_col: RA dec_col: Dec - e1_PSF_col: E1_PSF_HSM - e1_star_col: E1_STAR_HSM - e2_PSF_col: E2_PSF_HSM - e2_star_col: E2_STAR_HSM + e1_PSF_col: HSM_G1_PSF + e1_star_col: HSM_G1_STAR + e2_PSF_col: HSM_G2_PSF + e2_star_col: HSM_G2_STAR shear: R: 1.0 path: v1.6.6/unions_shapepipe_cut_struc_2024_v1.6.6.fits From c84e4040bb9d9a7d4a4972b3065237783201c91a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 14:29:13 +0200 Subject: [PATCH 05/11] refactor(scripts): migrate scripts, papers, cosmo_inference to v2 grammar Non-library consumers onto the same grammar: - scripts/calibration/{extract_info,params,calibrate_comprehensive_cat}.py: Tpsf -> T_PSF_RECONV; ELL_PSFo -> G1/G2_PSF_ORIG; T_PSFo -> T_PSF_ORIG; bare galaxy ELL -> G1/G2 scalars (match_subsample/check_invalid callers updated for the new signatures); HSM E*_PSF -> HSM_G*_PSF; spread_model wiring removed; col_2d args dropped. - scripts/apply_alpha_snr_size_bin.py, scripts/examples/demo_*.py: same Tpsf/PSFo renames, col_2d dropped. - cosmo_val/compute_theory_cov.py, cosmo_inference/scripts/masking.py: Tpsf -> T_PSF_RECONV; square_size hardcoded False. - papers/catalog/*, papers/harmonic/*: HSM ellipticity renames and the sigma->T units fix (hand-rolled SIGMA_*_HSM**2 -> read HSM_T_* directly). Out of scope, left as-is: GALSIM_* columns, scratch/, and the cosmo_inference notebooks (own migration fiber). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01M8BwwtExg961NknUeu5GCv --- cosmo_inference/scripts/masking.py | 4 +- cosmo_val/compute_theory_cov.py | 3 +- .../2025_09_19_alpha_leakage_correction.py | 2 +- .../catalog/2025_12_09_residual_star_plot.py | 12 ++-- papers/catalog/2025_12_10_plot_maps.py | 6 +- .../2025_12_11_plot_star_properties.py | 6 +- ...025_12_11_plot_tau_stats_dependence_mag.py | 4 +- papers/catalog/hist_mag.py | 15 +++-- .../harmonic/2025_09_11_psf_leakage_cell.py | 4 +- scripts/apply_alpha_snr_size_bin.py | 2 +- .../calibrate_comprehensive_cat.py | 9 +-- scripts/calibration/extract_info.py | 57 ++++++++----------- scripts/calibration/params.py | 13 ++--- .../examples/demo_calibrate_minimal_cat.py | 5 +- .../demo_comprehensive_to_minimal_cat.py | 4 +- 15 files changed, 69 insertions(+), 77 deletions(-) diff --git a/cosmo_inference/scripts/masking.py b/cosmo_inference/scripts/masking.py index 08a04feb..ba2f3c4c 100644 --- a/cosmo_inference/scripts/masking.py +++ b/cosmo_inference/scripts/masking.py @@ -118,9 +118,9 @@ def apply_masks(data, data_ext, mask_config, footprint_only=False): if not footprint_only: rel_size = np.divide( data["NGMIX_T_NOSHEAR"], - data["NGMIX_Tpsf_NOSHEAR"], + data["NGMIX_T_PSF_RECONV_NOSHEAR"], out=np.zeros_like(data["NGMIX_T_NOSHEAR"]), - where=(data["NGMIX_Tpsf_NOSHEAR"] > 0), + where=(data["NGMIX_T_PSF_RECONV_NOSHEAR"] > 0), ) rel_min = mask_config["metacal"]["gal_rel_size_min"] diff --git a/cosmo_val/compute_theory_cov.py b/cosmo_val/compute_theory_cov.py index b65fadb2..47d570b6 100644 --- a/cosmo_val/compute_theory_cov.py +++ b/cosmo_val/compute_theory_cov.py @@ -18,7 +18,8 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - params["square_size"] = cat["psf"]["square_size"] + # ShapePipe-v2 stores every size as T = 2σ² (an area); nothing to square. + params["square_size"] = False params["R11"] = np.array([1]) params["R22"] = np.array([1]) if survey != "DES": diff --git a/papers/catalog/2025_09_19_alpha_leakage_correction.py b/papers/catalog/2025_09_19_alpha_leakage_correction.py index de411ed8..135b1aab 100644 --- a/papers/catalog/2025_09_19_alpha_leakage_correction.py +++ b/papers/catalog/2025_09_19_alpha_leakage_correction.py @@ -33,7 +33,7 @@ cat_gal = fits.getdata(path_cat) # %% -psf_size = "NGMIX_Tpsf_NOSHEAR" +psf_size = "NGMIX_T_PSF_RECONV_NOSHEAR" gal_size = "NGMIX_T_NOSHEAR" size_ratio = cat_gal[psf_size] / (cat_gal[gal_size] + cat_gal[psf_size]) diff --git a/papers/catalog/2025_12_09_residual_star_plot.py b/papers/catalog/2025_12_09_residual_star_plot.py index 30085eef..79e2f521 100644 --- a/papers/catalog/2025_12_09_residual_star_plot.py +++ b/papers/catalog/2025_12_09_residual_star_plot.py @@ -27,12 +27,12 @@ cat_star = fits.getdata(path_star) # To be checked -e1_star = cat_star["E1_STAR_HSM"] -e2_star = cat_star["E2_STAR_HSM"] -T_star = cat_star["SIGMA_STAR_HSM"] ** 2 -e1_psf = cat_star["E1_PSF_HSM"] -e2_psf = cat_star["E2_PSF_HSM"] -T_psf = cat_star["SIGMA_PSF_HSM"] ** 2 +e1_star = cat_star["HSM_G1_STAR"] +e2_star = cat_star["HSM_G2_STAR"] +T_star = cat_star["HSM_T_STAR"] +e1_psf = cat_star["HSM_G1_PSF"] +e2_psf = cat_star["HSM_G2_PSF"] +T_psf = cat_star["HSM_T_PSF"] M_4_1_star = cat_star["M_4_STAR_1"] M_4_2_star = cat_star["M_4_STAR_2"] M_4_1_psf = cat_star["M_4_PSF_1"] diff --git a/papers/catalog/2025_12_10_plot_maps.py b/papers/catalog/2025_12_10_plot_maps.py index 2ad0be7d..ad8da0ba 100644 --- a/papers/catalog/2025_12_10_plot_maps.py +++ b/papers/catalog/2025_12_10_plot_maps.py @@ -599,9 +599,9 @@ def get_kappa_kaiser_squire(gamma_1, gamma_2): # %% # Maps of star/PSF information ra_star, dec_star = cat_star["RA"], cat_star["DEC"] -e1_star, e2_star = cat_star["E1_STAR_HSM"], cat_star["E2_STAR_HSM"] -e1_psf, e2_psf = cat_star["E1_PSF_HSM"], cat_star["E2_PSF_HSM"] -t_star, t_psf = cat_star["SIGMA_STAR_HSM"] ** 2, cat_star["SIGMA_PSF_HSM"] ** 2 +e1_star, e2_star = cat_star["HSM_G1_STAR"], cat_star["HSM_G2_STAR"] +e1_psf, e2_psf = cat_star["HSM_G1_PSF"], cat_star["HSM_G2_PSF"] +t_star, t_psf = cat_star["HSM_T_STAR"], cat_star["HSM_T_PSF"] weights_star = np.ones_like(ra_star) # %% diff --git a/papers/catalog/2025_12_11_plot_star_properties.py b/papers/catalog/2025_12_11_plot_star_properties.py index be992521..2aef02f4 100644 --- a/papers/catalog/2025_12_11_plot_star_properties.py +++ b/papers/catalog/2025_12_11_plot_star_properties.py @@ -34,9 +34,9 @@ # Load catalog cat_stars = fits.open(path_cat_star)[1].data -e1_star, e2_star = cat_stars["E1_STAR_HSM"], cat_stars["E2_STAR_HSM"] -e1_psf, e2_psf = cat_stars["E1_PSF_HSM"], cat_stars["E2_PSF_HSM"] -t_star, t_psf = cat_stars["SIGMA_STAR_HSM"] ** 2, cat_stars["SIGMA_PSF_HSM"] ** 2 +e1_star, e2_star = cat_stars["HSM_G1_STAR"], cat_stars["HSM_G2_STAR"] +e1_psf, e2_psf = cat_stars["HSM_G1_PSF"], cat_stars["HSM_G2_PSF"] +t_star, t_psf = cat_stars["HSM_T_STAR"], cat_stars["HSM_T_PSF"] mag = cat_stars["MAG"] # %% diff --git a/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py b/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py index 4b359e15..911a8449 100644 --- a/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py +++ b/papers/catalog/2025_12_11_plot_tau_stats_dependence_mag.py @@ -65,8 +65,8 @@ tc_psf = treecorr.Catalog( ra=cat_star["RA"], dec=cat_star["DEC"], - g1=cat_star["E1_PSF_HSM"], - g2=cat_star["E2_PSF_HSM"], + g1=cat_star["HSM_G1_PSF"], + g2=cat_star["HSM_G2_PSF"], ra_units="degrees", dec_units="degrees", npatch=100, diff --git a/papers/catalog/hist_mag.py b/papers/catalog/hist_mag.py index 262c38cd..60a1a2b6 100644 --- a/papers/catalog/hist_mag.py +++ b/papers/catalog/hist_mag.py @@ -300,8 +300,8 @@ def plot_all_hists( "FLAGS", "IMAFLAGS_ISO", "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", "8_Manual", "1024_Maximask", @@ -316,8 +316,8 @@ def plot_all_hists( "IMAFLAGS_ISO", "FLAGS", "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", "8_Manual", "1024_Maximask", @@ -330,8 +330,8 @@ def plot_all_hists( combine_cols = { "ngmix failures": [ "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", ] } @@ -415,7 +415,7 @@ def get_info_for_metacal_masking(dat, mask, prefix="NGMIX", name_shear="NOSHEAR" for key in ("flux", "flux_err", "T"): res[key] = dat[mask][f"{prefix}_{key.upper()}_{name_shear}"] - res["Tpsf"] = dat[mask][f"{prefix}_Tpsf_{name_shear}"] + res["Tpsf"] = dat[mask][f"{prefix}_T_PSF_RECONV_{name_shear}"] return res @@ -439,7 +439,6 @@ def get_info_for_metacal_masking(dat, mask, prefix="NGMIX", name_shear="NOSHEAR" size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], global_R_weight=cm["global_R_weight"], - col_2d=False, verbose=True, ) diff --git a/papers/harmonic/2025_09_11_psf_leakage_cell.py b/papers/harmonic/2025_09_11_psf_leakage_cell.py index c388affb..45f13362 100644 --- a/papers/harmonic/2025_09_11_psf_leakage_cell.py +++ b/papers/harmonic/2025_09_11_psf_leakage_cell.py @@ -103,7 +103,7 @@ def get_shape_noise(e1, e2, w): f_psf = nmt.NmtFieldCatalog( positions=[cat_star["RA"], cat_star["DEC"]], weights=np.ones_like(cat_star["RA"]), - field=[cat_star["E1_PSF_HSM"], -cat_star["E2_PSF_HSM"]], + field=[cat_star["HSM_G1_PSF"], -cat_star["HSM_G2_PSF"]], lmax=b_lmax, lmax_mask=b_lmax, spin=2, @@ -235,7 +235,7 @@ def compute_iNKA_covariance(field1, field2, coupled=False): # %% -field_psf_map = get_field_catalog(cat_star, "E1_PSF_HSM", "E2_PSF_HSM", nside=1024) +field_psf_map = get_field_catalog(cat_star, "HSM_G1_PSF", "HSM_G2_PSF", nside=1024) field_gal_map = get_field_catalog(cat_gal, "e1", "e2", nside=1024, w_col="w_des") field_gal_corrected_map = get_field_catalog( cat_gal, "e1_leak_corrected", "e2_leak_corrected", nside=1024, w_col="w_des" diff --git a/scripts/apply_alpha_snr_size_bin.py b/scripts/apply_alpha_snr_size_bin.py index abb541bf..12261e31 100644 --- a/scripts/apply_alpha_snr_size_bin.py +++ b/scripts/apply_alpha_snr_size_bin.py @@ -40,7 +40,7 @@ def params_default(): "e2_PSF_col": "e2_PSF", "snr_col": "snr", "w_col": "w", - "TPSF_col": "NGMIX_Tpsf_NOSHEAR", + "TPSF_col": "NGMIX_T_PSF_RECONV_NOSHEAR", "T_col": "NGMIX_T_NOSHEAR", "input_path_shear": "SP/unions_shapepipe_extended_2022_v1.0.fits", "output_path": "shape_cat_cor_alpha.fits", diff --git a/scripts/calibration/calibrate_comprehensive_cat.py b/scripts/calibration/calibrate_comprehensive_cat.py index 38d67a53..4b15ed89 100644 --- a/scripts/calibration/calibrate_comprehensive_cat.py +++ b/scripts/calibration/calibrate_comprehensive_cat.py @@ -92,7 +92,6 @@ size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], global_R_weight=cm["global_R_weight"], - col_2d=False, verbose=True, ) @@ -185,7 +184,7 @@ "FLUX_APER", "FLUXERR_APER", "NGMIX_T_NOSHEAR", - "NGMIX_Tpsf_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", "fwhm_PSF", ] add_cols_data = {} @@ -197,8 +196,10 @@ "FHP/MK hack: explicit copying of the metacal no-shear (updated from 1p)" + " PSF size" ) -add_cols_data["NGMIX_Tpsf_NOSHEAR_orig"] = add_cols_data["NGMIX_Tpsf_NOSHEAR"] -add_cols_data["NGMIX_Tpsf_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] +add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR_orig"] = add_cols_data[ + "NGMIX_T_PSF_RECONV_NOSHEAR" +] +add_cols_data["NGMIX_T_PSF_RECONV_NOSHEAR"] = gal_metacal.ns["Tpsf"][mask_metacal] # %% # Additional post-processing columns to write to output cat diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index 3868080b..78647209 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -79,14 +79,21 @@ # PSF keys key_base = shape.upper() -key_PSF_ell = f"{key_base}_ELL_PSFo_NOSHEAR" -key_PSF_size = f"{key_base}_T_PSFo_NOSHEAR" +key_PSF_g1 = f"{key_base}_G1_PSF_ORIG_NOSHEAR" +key_PSF_g2 = f"{key_base}_G2_PSF_ORIG_NOSHEAR" +key_PSF_size = f"{key_base}_T_PSF_ORIG_NOSHEAR" size_to_fwhm = T_to_fwhm print_stats("Galaxies:", stats_file, verbose=verbose) n_tot = spv_cat.print_some_quantities(dd, stats_file, verbose=verbose) spv_cat.print_mean_ellipticity( - dd, f"{key_base}_ELL_NOSHEAR", 2, n_tot, stats_file, invalid=-10, verbose=verbose + dd, + [f"{key_base}_G1_NOSHEAR", f"{key_base}_G2_NOSHEAR"], + 1, + n_tot, + stats_file, + invalid=-10, + verbose=verbose, ) # #### Survey area and potential missing tiles @@ -110,7 +117,7 @@ n_tot = spv_cat.print_some_quantities(d_star, stats_file, verbose=verbose) spv_cat.print_mean_ellipticity( d_star, - ["E1_PSF_HSM", "E2_PSF_HSM"], + ["HSM_G1_PSF", "HSM_G2_PSF"], 1, n_tot, stats_file, @@ -121,7 +128,7 @@ # ### 3. Matching of stars # ### Matching of star catalogues -# Match the star catalogue `d_star` (selected on individual exposures using size-magnitude diagram) to catalogue from tile. Uses some simple criteria to select stars from tile catalogue such as SPREAD_CLASS. +# Match the star catalogue `d_star` (selected on individual exposures using size-magnitude diagram) to catalogue from tile. Uses some simple criteria to select stars from tile catalogue. # # This is mainly for testing, this match will not be used later. @@ -148,7 +155,7 @@ (dd["FLAGS"][ind_star] == 0) & (dd["IMAFLAGS_ISO"][ind_star] == 0) & (dd["NGMIX_MCAL_FLAGS"][ind_star] == 0) - & (dd["NGMIX_ELL_PSFo_NOSHEAR"][:, 0][ind_star] != -10) + & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"][ind_star] != -10) ) ra_star, dec_star, g_star_psf = spv_cat.match_subsample( @@ -156,7 +163,8 @@ ind_star, m_star, [col_name_ra, col_name_dec], - key_PSF_ell, + key_PSF_g1, + key_PSF_g2, n_star_tot, stats_file, verbose=verbose, @@ -175,24 +183,11 @@ g_star_psf[1], ) -#### Refine: Match to SPREAD_CLASS samples -if "SPREAD_CLASS" in dd.dtype.names: - spv_cat.match_spread_class( - dd, ind_star, m_star, stats_file, len(ra_star), verbose=verbose - ) -else: - print_stats( - "No SPREAD_CLASS in input, skipping star-gal matching", - stats_file, - verbose=verbose, - ) - # ## Check for objects with invalid PSF spv_cat.check_invalid( dd, - [key_PSF_ell, f"{key_base}_ELL_NOSHEAR"], - [0, 0], + [key_PSF_g1, f"{key_base}_G1_NOSHEAR"], [-10, -10], stats_file, name=["`PSF", "galaxy ellipticity"], @@ -243,16 +238,16 @@ add_cols_pre_cal_format["overlap"] = "I" # Additional columns {e1, e2, size}_PSF -ext_cols_pre_cal["e1_PSF"] = dd[key_PSF_ell][:, 0] -ext_cols_pre_cal["e2_PSF"] = dd[key_PSF_ell][:, 1] +ext_cols_pre_cal["e1_PSF"] = dd[key_PSF_g1] +ext_cols_pre_cal["e2_PSF"] = dd[key_PSF_g2] ext_cols_pre_cal["fwhm_PSF"] = size_to_fwhm(dd[key_PSF_size]) -_, _, iv_w = metacal.get_variance_ivweights(dd, sigma_eps_prior, mask=None, col_2d=True) +_, _, iv_w = metacal.get_variance_ivweights(dd, sigma_eps_prior, mask=None) mag = spv_cat.get_col(dd, "MAG_AUTO", None, None) snr = spv_cat.get_snr(shape, dd, None, None) -g1_uncal = dd[f"{key_base}_ELL_NOSHEAR"][:, 0] -g2_uncal = dd[f"{key_base}_ELL_NOSHEAR"][:, 1] +g1_uncal = dd[f"{key_base}_G1_NOSHEAR"] +g2_uncal = dd[f"{key_base}_G2_NOSHEAR"] # Comprehensive catalogue without cuts nor mask applied if verbose: @@ -300,7 +295,6 @@ gal_mag_faint=gal_mag_faint, flags_keep=flags_keep, n_epoch_min=n_epoch_min, - do_spread_model=do_spread_model, ) if shape == "ngmix": m_gal = classification_galaxy_ngmix( @@ -608,7 +602,7 @@ labels = [] if shape == "ngmix": # Do not apply `mask_ns`, so use all galaxies - xs = [dd["NGMIX_T_NOSHEAR"][m_gal] / dd["NGMIX_Tpsf_NOSHEAR"][m_gal]] + xs = [dd["NGMIX_T_NOSHEAR"][m_gal] / dd["NGMIX_T_PSF_RECONV_NOSHEAR"][m_gal]] labels.append("size ratio") else: @@ -878,8 +872,7 @@ n_bin = 250 # + -key = key_PSF_ell -xs = [dd[key][:, 0][mask_ns_stars], dd[key][:, 1][mask_ns_stars]] +xs = [dd[key_PSF_g1][mask_ns_stars], dd[key_PSF_g2][mask_ns_stars]] title = "PSF" out_name = f"ell_PSF_{shape}.pdf" out_path = os.path.join(plot_dir, out_name) @@ -990,8 +983,8 @@ # + # Additional columns: # {e1, e2, size}_PSF -ext_cols["e1_PSF"] = dd[key_PSF_ell][:, 0][m_gal][mask] -ext_cols["e2_PSF"] = dd[key_PSF_ell][:, 1][m_gal][mask] +ext_cols["e1_PSF"] = dd[key_PSF_g1][m_gal][mask] +ext_cols["e2_PSF"] = dd[key_PSF_g2][m_gal][mask] ext_cols["fwhm_PSF"] = size_to_fwhm(dd[key_PSF_size][m_gal][mask]) if mask_external_path: ext_cols["mask_extern"] = m_extern diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index 363b36f7..ed59958c 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -119,7 +119,7 @@ "FLUX_APER", "FLUXERR_APER", "NGMIX_T_NOSHEAR", - "NGMIX_Tpsf_NOSHEAR", + "NGMIX_T_PSF_RECONV_NOSHEAR", ] ## Pre-calibration catalogue, including masked objects and mask flags @@ -132,8 +132,10 @@ "NGMIX_MOM_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", - "NGMIX_ELL_PSFo_NOSHEAR", - "NGMIX_ELL_ERR_NOSHEAR", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", + "NGMIX_G1_ERR_NOSHEAR", + "NGMIX_G2_ERR_NOSHEAR", ] ### Set flag columns as integer format @@ -155,7 +157,7 @@ # Create key names for metacal information prefix = "NGMIX" suffixes = ["1M", "1P", "2M", "2P", "NOSHEAR"] -centers = ["FLAGS", "ELL", "FLUX", "FLUX_ERR", "T", "T_ERR", "Tpsf"] +centers = ["FLAGS", "G1", "G2", "FLUX", "FLUX_ERR", "T", "T_ERR", "T_PSF_RECONV"] for center in centers: for suffix in suffixes: add_cols_pre_cal.append(f"{prefix}_{center}_{suffix}") @@ -185,9 +187,6 @@ gal_mag_bright = 15 gal_mag_faint = 30 -### Spread-model -do_spread_model = False - ### SExtractor flags to keep in addition to FLAGS=0 ### (bit-coded; list of powers of 2); ### Empty list if no flags diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 85a0f536..9599d8ba 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -53,8 +53,8 @@ "N_EPOCH", "mag", "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", ] masks, labels = sp_joint.get_masks_from_config( @@ -104,7 +104,6 @@ rel_size_max=cm["gal_rel_size_max"], size_corr_ell=cm["gal_size_corr_ell"], sigma_eps=cm["sigma_eps_prior"], - col_2d=False, verbose=True, ) # - diff --git a/scripts/examples/demo_comprehensive_to_minimal_cat.py b/scripts/examples/demo_comprehensive_to_minimal_cat.py index 243feda7..6d0651cf 100644 --- a/scripts/examples/demo_comprehensive_to_minimal_cat.py +++ b/scripts/examples/demo_comprehensive_to_minimal_cat.py @@ -54,8 +54,8 @@ "overlap", "IMAFLAGS_ISO", "NGMIX_MOM_FAIL", - "NGMIX_ELL_PSFo_NOSHEAR_0", - "NGMIX_ELL_PSFo_NOSHEAR_1", + "NGMIX_G1_PSF_ORIG_NOSHEAR", + "NGMIX_G2_PSF_ORIG_NOSHEAR", "8_Manual", ] From e0eaa9f3cae0acb60fd9d0313c1095e89d42ebad Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 15:00:01 +0200 Subject: [PATCH 06/11] refactor(src): remove dead galsim path, finish v2 grammar tightening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tightening-pass changes to complete ShapePipe-v2 column adoption in the package: - Remove the galsim estimator path as dead code. `shape` is hardcoded to "ngmix", extract_info.py raises for any other value, and nothing outside scratch/ instantiates metacal(prefix="GALSIM") or calls classification_galaxy_galsim. Migrating it would carry an untestable path whose shared `col_1p = {prefix}_T_PSF_RECONV_1P` read never matched the galsim producer output (GALSIM_T_PSF_*, not ..._T_PSF_RECONV_*) — already broken. Removed: metacal._read_data_galsim, the prefix=="GALSIM" dispatch (now else: raise — unknown prefixes fail loudly), the two galsim ellipticity sign flips in _shear_response/_selection_response, galaxy.classification_galaxy_galsim, the sh=="galsim" branch in catalog.get_snr (now else: raise), and the unused shape_method arg on get_calibrated_quantities/get_calibrated_m_c. ngmix is the sole estimator; the `prefix` param stays (names the column family; a future NGMIXm moments family could reuse it). - Rename the galaxy failure-flag read NGMIX_MOM_FAIL -> NGMIX_MCAL_TYPES_FAIL in classification_galaxy_ngmix. The v1 column counted moments-initial-guess failures (get_guess, gone in v2); the producer reused the slot for a failed-metacal-types count. sp_validation cuts on == 0 either way (keep fully-measured objects), so the read migrates cleanly; the underlying failure mode changed, so this is the first line to check if the post-cut galaxy count looks off against a regenerated v2 catalogue. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019f8w2twg4b3Ga36dSxNbLW --- src/sp_validation/calibration.py | 67 ++++---------------------------- src/sp_validation/catalog.py | 4 +- src/sp_validation/galaxy.py | 24 +----------- 3 files changed, 11 insertions(+), 84 deletions(-) diff --git a/src/sp_validation/calibration.py b/src/sp_validation/calibration.py index 971f05d9..9032bc63 100644 --- a/src/sp_validation/calibration.py +++ b/src/sp_validation/calibration.py @@ -19,7 +19,7 @@ from sp_validation.statistics import jackknif_weighted_average2 -def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): +def get_calibrated_quantities(gal_metacal): """Get Calibrated Quantities. Return catalogue quantities for objects calibrated for multiplicative @@ -29,8 +29,6 @@ def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): ---------- gal_metacal : dict galaxy metacalibration catalogue - shape_method : string, optional, default='ngmix' - shape measurement method, one in 'ngmix', 'galsim' Returns ------- @@ -58,7 +56,7 @@ def get_calibrated_quantities(gal_metacal, shape_method="ngmix"): return g_corr, g_uncorr, w, mask -def get_calibrated_m_c(gal_metacal, shape_method="ngmix"): +def get_calibrated_m_c(gal_metacal): """Get Calibrated C. Return catalogue quantities for objects calibrated for multiplicative and @@ -68,8 +66,6 @@ def get_calibrated_m_c(gal_metacal, shape_method="ngmix"): ---------- gal_metacal : dict galaxy metacalibration catalogue - shape_method : string, optional, default='ngmix' - shape measurement method, one in 'ngmix', 'galsim' Returns ------- @@ -788,14 +784,9 @@ def _read_data(self, data, mask): p2, ns, ) - elif self._prefix == "GALSIM": - m1, p1, m2, p2, ns = self._read_data_galsim( - masked_data, - m1, - p1, - m2, - p2, - ns, + else: + raise ValueError( + f"Unsupported shape prefix '{self._prefix}'; only 'NGMIX' is supported" ) print("FHP/MK hack using p1 PSF for ns in cuts") @@ -896,40 +887,6 @@ def get_variance_ivweights(data, sigma_eps, prefix="NGMIX", mask=None): return C11, C22, iv_w - def _read_data_galsim(self, masked_data, m1, p1, m2, p2, ns): - """Read Data Galsim. - - Read data from galsim catalogue. - - """ - prefix_mom = "GALSIM_GAL" - - for name_shear, dict_tmp in zip( - ["1m", "1p", "2m", "2p", "noshear"], [m1, p1, m2, p2, ns] - ): - if self._verbose: - print("Extracting {}".format(name_shear)) - - dict_tmp["flag"] = masked_data[f"{self._prefix}_FLAGS_{name_shear.upper()}"] - dict_tmp["g1"] = masked_data[ - f"{prefix_mom}_ELL_UNCORR_{name_shear.upper()}" - ][:, 0] - dict_tmp["g2"] = masked_data[ - f"{prefix_mom}_ELL_UNCORR_{name_shear.upper()}" - ][:, 1] - - dict_tmp["T"] = masked_data[f"{prefix_mom}_SIGMA_{name_shear.upper()}"] - dict_tmp["Tpsf"] = masked_data[ - f"{self._prefix}_PSF_SIGMA_{name_shear.upper()}" - ] - - self.snr_sextractor = masked_data["SNR_WIN"] - ns["C11"] = masked_data[f"{prefix_mom}_ELL_ERR_NOSHEAR"][:, 0] - ns["C22"] = masked_data[f"{prefix_mom}_ELL_ERR_NOSHEAR"][:, 1] - ns["w"] = 1.0 / (2 * self._sigma_eps**2 + dict_tmp["C11"] + dict_tmp["C22"]) - - return m1, p1, m2, p2, ns - def _compute_calibration(self): """Compute Calibration. @@ -1071,15 +1028,11 @@ def _shear_response(self): Compute shear response matrix """ - sign = 1 - if self._prefix == "GALSIM": - sign = -1 - ma = self.mask_dict["ns"] h2 = 2 * self._step self.R11 = (self.p1["g1"][ma] - self.m1["g1"][ma]) / h2 - self.R22 = sign * (self.p2["g2"][ma] - self.m2["g2"][ma]) / h2 + self.R22 = (self.p2["g2"][ma] - self.m2["g2"][ma]) / h2 self.R12 = (self.p2["g1"][ma] - self.m2["g1"][ma]) / h2 self.R21 = (self.p1["g2"][ma] - self.m1["g2"][ma]) / h2 @@ -1121,10 +1074,6 @@ def _selection_response(self): ... """ - sign = 1 - if self._prefix == "GALSIM": - sign = -1 - ma_p1 = self.mask_dict["p1"] ma_m1 = self.mask_dict["m1"] ma_p2 = self.mask_dict["p2"] @@ -1135,8 +1084,8 @@ def _selection_response(self): np.mean(self.ns["g1"][ma_p1]) - np.mean(self.ns["g1"][ma_m1]) ) / h2 self.R22_s = ( - sign * (np.mean(self.ns["g2"][ma_p2]) - np.mean(self.ns["g2"][ma_m2])) / h2 - ) + np.mean(self.ns["g2"][ma_p2]) - np.mean(self.ns["g2"][ma_m2]) + ) / h2 self.R12_s = ( np.mean(self.ns["g1"][ma_p2]) - np.mean(self.ns["g1"][ma_m2]) ) / h2 diff --git a/src/sp_validation/catalog.py b/src/sp_validation/catalog.py index fd90ff1a..b1b175d9 100644 --- a/src/sp_validation/catalog.py +++ b/src/sp_validation/catalog.py @@ -855,7 +855,7 @@ def get_snr(sh, dat, m_sel, m_flg): my_snr = get_col(dat, "NGMIX_FLUX_NOSHEAR", m_sel, m_flg) / get_col( dat, "NGMIX_FLUX_ERR_NOSHEAR", m_sel, m_flg ) - elif sh == "galsim": - my_snr = get_col(dat, "SNR_WIN", m_sel, m_flg) + else: + raise ValueError(f"Unsupported shape method '{sh}'; only 'ngmix' is supported") return my_snr diff --git a/src/sp_validation/galaxy.py b/src/sp_validation/galaxy.py index fb346782..3c49a96b 100644 --- a/src/sp_validation/galaxy.py +++ b/src/sp_validation/galaxy.py @@ -193,7 +193,7 @@ def classification_galaxy_ngmix( cut_common & (dd["NGMIX_MCAL_FLAGS"] == 0) & (dd["NGMIX_G1_PSF_ORIG_NOSHEAR"] != -10) - & (dd["NGMIX_MOM_FAIL"] == 0) + & (dd["NGMIX_MCAL_TYPES_FAIL"] == 0) ) n_gal_ngmix = len(np.where(m_gal_ngmix)[0]) @@ -211,28 +211,6 @@ def classification_galaxy_ngmix( return m_gal_ngmix -def classification_galaxy_galsim(dd, cut_common, stats_file, verbose=False): - """Classification Galaxy Galsim. - - Return mask corresponding to galsim classification of galaxies - - """ - m_gal_galsim = cut_common & (dd["GALSIM_PSF_ELL_ORIGINAL_PSF"][:, 0] != -10) - - n_gal_galsim = len(np.where(m_gal_galsim)[0]) - n_tot = len(dd) - - io.print_ratio( - "galsim: Objects selected as galaxies", - n_gal_galsim, - n_tot, - stats_file, - verbose=verbose, - ) - - return m_gal_galsim - - def mask_overlap(ra, dec, tile_id_in, region_file_path, n_jobs=-1): """Mask Overlap. From 3bcd5017907adea2d11e0551a782d6f61ad8f688 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 15:00:16 +0200 Subject: [PATCH 07/11] refactor(config): NGMIX_MOM_FAIL -> NGMIX_MCAL_TYPES_FAIL in mask configs The v1 failure-flag column no longer exists in the ShapePipe-v2 header; the producer renamed the slot to NGMIX_MCAL_TYPES_FAIL. Propagate the rename into all ten calibration mask configs so the flag column resolves against a v2 catalogue. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019f8w2twg4b3Ga36dSxNbLW --- config/calibration/mask_v1.X.10.yaml | 2 +- config/calibration/mask_v1.X.11.yaml | 2 +- config/calibration/mask_v1.X.2.yaml | 2 +- config/calibration/mask_v1.X.3.yaml | 2 +- config/calibration/mask_v1.X.4.yaml | 2 +- config/calibration/mask_v1.X.5.yaml | 2 +- config/calibration/mask_v1.X.6.yaml | 2 +- config/calibration/mask_v1.X.7.yaml | 2 +- config/calibration/mask_v1.X.8.yaml | 2 +- config/calibration/mask_v1.X.9.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 1f002fcf..703ea91b 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml index d224be34..88e01643 100644 --- a/config/calibration/mask_v1.X.11.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index 81e98208..846f14ff 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index feaf582f..bcbbc620 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 8c7dd186..903b9098 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index bce8b6d2..5f56b135 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index 72ed62b7..1b5f7da6 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index 7add0cc0..cbc802be 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index 418cab72..58234d64 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 diff --git a/config/calibration/mask_v1.X.9.yaml b/config/calibration/mask_v1.X.9.yaml index 077ec7e8..4a448ce7 100644 --- a/config/calibration/mask_v1.X.9.yaml +++ b/config/calibration/mask_v1.X.9.yaml @@ -42,7 +42,7 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 From 34c7cc6f918425e9996fb109f1ccae5a6f60ea46 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 15:00:25 +0200 Subject: [PATCH 08/11] refactor(scripts,papers): finish v2 grammar tightening in scripts - NGMIX_MOM_FAIL -> NGMIX_MCAL_TYPES_FAIL in params.add_cols_pre_cal (and its int-format set), the two demo column lists, and hist_mag's read lists. - Drop the galsim mentions left in params.py (shape comment) and extract_info.py (the stale "cuts common to ngmix and galsim" comment, whose spread-model line also went with the earlier spread_model removal). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019f8w2twg4b3Ga36dSxNbLW --- papers/catalog/hist_mag.py | 6 +++--- scripts/calibration/extract_info.py | 3 +-- scripts/calibration/params.py | 9 ++++----- scripts/examples/demo_calibrate_minimal_cat.py | 2 +- scripts/examples/demo_comprehensive_to_minimal_cat.py | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/papers/catalog/hist_mag.py b/papers/catalog/hist_mag.py index 60a1a2b6..b0bc33ef 100644 --- a/papers/catalog/hist_mag.py +++ b/papers/catalog/hist_mag.py @@ -299,7 +299,7 @@ def plot_all_hists( [ "FLAGS", "IMAFLAGS_ISO", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "NGMIX_G1_PSF_ORIG_NOSHEAR", "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", @@ -315,7 +315,7 @@ def plot_all_hists( [ "IMAFLAGS_ISO", "FLAGS", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "NGMIX_G1_PSF_ORIG_NOSHEAR", "NGMIX_G2_PSF_ORIG_NOSHEAR", "4_Stars", @@ -329,7 +329,7 @@ def plot_all_hists( combine_cols = { "ngmix failures": [ - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "NGMIX_G1_PSF_ORIG_NOSHEAR", "NGMIX_G2_PSF_ORIG_NOSHEAR", ] diff --git a/scripts/calibration/extract_info.py b/scripts/calibration/extract_info.py index 78647209..1cbf836f 100644 --- a/scripts/calibration/extract_info.py +++ b/scripts/calibration/extract_info.py @@ -279,8 +279,7 @@ # ## 4. Select galaxies # #### Common flags and cuts -# First, set cuts common to ngmix and galsim: -# - spread model: select objects well larger than the PSF +# First, set cuts common to all objects: # - magnitude: cut galaxies that are too faint (= too noisy, likely to be # artefacts), and too bright (might be too large for postage stamp) # - flags: cut objects that were flagged as invalid or masked diff --git a/scripts/calibration/params.py b/scripts/calibration/params.py index ed59958c..2d0bd45f 100644 --- a/scripts/calibration/params.py +++ b/scripts/calibration/params.py @@ -35,9 +35,8 @@ ## Pixel size in arcsec pixel_size = 0.187 -## Shape measurement method, implemented is -## 'ngix': multi-epoch model fitting -## 'galsim': stacked-image moments (experimental) +## Shape measurement method (only 'ngmix' is supported): +## 'ngmix': multi-epoch model fitting shape = "ngmix" # Paths @@ -129,7 +128,7 @@ "IMAFLAGS_ISO", "FLAGS", "NGMIX_MCAL_FLAGS", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", "NGMIX_G1_PSF_ORIG_NOSHEAR", @@ -145,7 +144,7 @@ "IMAFLAGS_ISO", "FLAGS", "NGMIX_MCAL_FLAGS", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "N_EPOCH", "NGMIX_N_EPOCH", ): diff --git a/scripts/examples/demo_calibrate_minimal_cat.py b/scripts/examples/demo_calibrate_minimal_cat.py index 9599d8ba..a90025dd 100644 --- a/scripts/examples/demo_calibrate_minimal_cat.py +++ b/scripts/examples/demo_calibrate_minimal_cat.py @@ -52,7 +52,7 @@ "1024_Maximask", "N_EPOCH", "mag", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "NGMIX_G1_PSF_ORIG_NOSHEAR", "NGMIX_G2_PSF_ORIG_NOSHEAR", ] diff --git a/scripts/examples/demo_comprehensive_to_minimal_cat.py b/scripts/examples/demo_comprehensive_to_minimal_cat.py index 6d0651cf..11e5ed5a 100644 --- a/scripts/examples/demo_comprehensive_to_minimal_cat.py +++ b/scripts/examples/demo_comprehensive_to_minimal_cat.py @@ -53,7 +53,7 @@ masks_to_apply = [ "overlap", "IMAFLAGS_ISO", - "NGMIX_MOM_FAIL", + "NGMIX_MCAL_TYPES_FAIL", "NGMIX_G1_PSF_ORIG_NOSHEAR", "NGMIX_G2_PSF_ORIG_NOSHEAR", "8_Manual", From 427e723dfae46cbcd152d14fca6865f8ddf0f64e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 15:00:47 +0200 Subject: [PATCH 09/11] refactor(notebooks): migrate cfis_analysis HSM PSF columns to v2 grammar cfis_analysis.ipynb read E1/E2_PSF_HSM into treecorr for the C_sys PSF test; rename to HSM_G1/G2_PSF (native g, pure rename). A sweep of all tracked notebooks found no other old column tokens. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019f8w2twg4b3Ga36dSxNbLW --- cosmo_inference/notebooks/cfis_analysis.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cosmo_inference/notebooks/cfis_analysis.ipynb b/cosmo_inference/notebooks/cfis_analysis.ipynb index 74816206..ee93f4ec 100644 --- a/cosmo_inference/notebooks/cfis_analysis.ipynb +++ b/cosmo_inference/notebooks/cfis_analysis.ipynb @@ -776,8 +776,8 @@ "cat_psf = treecorr.Catalog(\n", " ra=df_psf[\"RA\"],\n", " dec=df_psf[\"DEC\"],\n", - " g1=df_psf[\"E1_PSF_HSM\"],\n", - " g2=df_psf[\"E2_PSF_HSM\"],\n", + " g1=df_psf[\"HSM_G1_PSF\"],\n", + " g2=df_psf[\"HSM_G2_PSF\"],\n", " ra_units=\"degrees\",\n", " dec_units=\"degrees\",\n", " npatch=50,\n", From d536715bc0e55977701075e024f81c5c2899aca8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 15:00:49 +0200 Subject: [PATCH 10/11] docs: record galsim removal, MOM_FAIL rename, and *_PSF_ORIG reframe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the authoritative migration map for the tightening pass: - *_PSF_ORIG reframed as a value change that is NOT a code blocker (the rename is correct as-is; a v2 catalogue only enables a look-at-numbers check). The real merge gate is cutover timing — merging makes develop require v2 columns. - NGMIX_MOM_FAIL -> NGMIX_MCAL_TYPES_FAIL added to the ngmix map with a semantics-change note. - galsim path documented as removed dead code (was "left untouched, flagged"). - notebooks added to the consumer-sites list. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019f8w2twg4b3Ga36dSxNbLW --- docs/ngmix_psf_column_migration.md | 85 +++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/docs/ngmix_psf_column_migration.md b/docs/ngmix_psf_column_migration.md index 27520d99..99607bae 100644 --- a/docs/ngmix_psf_column_migration.md +++ b/docs/ngmix_psf_column_migration.md @@ -1,12 +1,23 @@ # ShapePipe-v2 column-grammar migration (shapepipe → sp_validation) -**Status:** code staged on `migrate/ngmix-psf-column-names` (draft PR -[#201](https://github.com/CosmoStat/sp_validation/pull/201)). The renames, the -σ→T units change and the `spread_model` removal are complete and the test suite -is green against synthetic catalogues carrying the new columns. The one item -that genuinely needs a *regenerated* catalogue — re-validating the `*_PSF_ORIG` -value change — is deferred until [CosmoStat/shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761) -lands in #741 and a v2 catalogue is produced. +**Status:** code-complete on `migrate/ngmix-psf-column-names` (draft PR +[#201](https://github.com/CosmoStat/sp_validation/pull/201)). Every shape-column +read in the live package, configs, calibration scripts, paper figures, and +notebooks uses the ShapePipe-v2 grammar; the σ→T units change and the +`spread_model` removal are in; the dead `galsim` estimator path is removed; and +the suite is green against synthetic catalogues carrying the new columns. + +**No code work is left waiting on a regenerated catalogue.** The one *value* +change — `*_PSF_ORIG` now holds a true original-PSF fit (shapepipe#749) rather +than the reconvolved-kernel alias the old columns silently held — is a straight +column rename at the code level and is already in place; the code does not care +that the numbers moved. All a v2 catalogue enables is a *look-at-the-numbers* +sanity check (do the α-leakage / size-ratio cuts still behave), which is analysis, +not code, and does not gate the PR. **The real merge gate is cutover timing:** +merging this branch makes `develop` *require* v2 columns and stop reading today's +catalogues, so #201 should land together with — or just after — +[shapepipe#761](https://github.com/CosmoStat/shapepipe/pull/761)→#741 and the +first v2 catalogue. shapepipe#761 turns the shape-measurement output into **one column grammar for the whole catalogue**: every estimator names its outputs @@ -40,6 +51,17 @@ repo needs a genuine e↔g conversion; sp_validation does not. | `NGMIX_FLUX_{shear}` / `_ERR` | unchanged | | | `NGMIX_FLAGS_{shear}`, `NGMIX_SNR_{shear}` | unchanged | | | `NGMIX_MCAL_FLAGS`, `NGMIX_N_EPOCH` | unchanged | OBJECT/SHEAR-less metadata | +| `NGMIX_MOM_FAIL` | `NGMIX_MCAL_TYPES_FAIL` | **renamed + semantics change** (see below) | + +`NGMIX_MOM_FAIL` → `NGMIX_MCAL_TYPES_FAIL` is more than a rename: in ngmix v1 the +column counted moments-initial-guess failures from `get_guess`, which no longer +exists in v2, so the producer reused the slot for a *failed-metacal-types* count. +sp_validation only ever cuts on it as `== 0` (keep objects with no failure), and +that cut stays correct — but the underlying failure mode changed, so if the +post-cut galaxy count looks off against a regenerated v2 catalogue, this is the +first line to check. The cut lives in `galaxy.classification_galaxy_ngmix`; the +column is also carried through `params.add_cols_pre_cal` and every +`config/calibration/mask_v1.X.*.yaml`. ### ngmix — reconvolved PSF (metacal kernel; value-safe rename) @@ -47,12 +69,14 @@ repo needs a genuine e↔g conversion; sp_validation does not. |---|---|---| | `NGMIX_Tpsf_{shear}` | `NGMIX_T_PSF_RECONV_{shear}` | same value (the `T/Tpsf` size-ratio cut) | -### ngmix — original PSF (⚠ **value change** — shapepipe#749 fix) +### ngmix — original PSF (value change — shapepipe#749 fix — *not a code blocker*) `*_PSF_ORIG` now carries a *true* fit to the original image PSF, no longer the -reconvolved-kernel alias the old `ELL_PSFo`/`T_PSFo` columns silently held. A -mechanical rename is correct for the *names*; the *numbers* move, so any size-ratio -or leakage cut fed by these must be re-validated against a regenerated catalogue. +reconvolved-kernel alias the old `ELL_PSFo`/`T_PSFo` columns silently held. The +rename is a straight column rename in sp_validation and is correct as-is — the +code does not care that the numbers moved. The only thing a regenerated catalogue +buys is a *look-at-the-numbers* check that the α-leakage / size-ratio cuts still +behave; that is analysis, not code, and it does not gate this PR (see Status). | Old | New | |---|---| @@ -116,21 +140,42 @@ repo's own migration. `scripts/apply_alpha_snr_size_bin.py`, `scripts/examples/demo_*.py`. - **Papers** — `papers/catalog/2025_12_*`, `papers/catalog/{hist_mag,2025_09_19_alpha_leakage_correction}.py`, `papers/harmonic/2025_09_11_psf_leakage_cell.py`. +- **Notebooks** — `cosmo_inference/notebooks/cfis_analysis.ipynb` + (`E1/E2_PSF_HSM` → `HSM_G1/G2_PSF`); a sweep of all tracked notebooks found no + other old tokens. - **Tests** — `src/sp_validation/tests/{test_calibration,test_cosmo_val}.py` (synthetic catalogues + configs updated in lock-step; this is the migration's internal-consistency check). -## Deferred / coordinated +## galsim estimator path — removed as dead code + +shapepipe#761 renamed the galsim column family too (`GALSIM_GAL_ELL_*` / +`GALSIM_*_SIGMA_*` → scalar `GALSIM_G1/G2_*`, `GALSIM_T*`), but sp_validation's +galsim reader is **dead code**: `shape` is hardcoded to `"ngmix"`, +`extract_info.py` raises for any other value, and nothing outside `scratch/` +instantiates `metacal(prefix="GALSIM")` or calls `classification_galaxy_galsim`. +Migrating it would produce an untestable path — and the shared +`col_1p = f"{prefix}_T_PSF_RECONV_1P"` read in `metacal._read_data` never matched +the galsim producer output (galsim writes `GALSIM_T_PSF_*`, not +`..._T_PSF_RECONV_*`), so the path was already broken. So instead of migrating it, +this branch **removes** it: + +- `calibration.metacal._read_data_galsim`, the `prefix == "GALSIM"` dispatch + branch (now `else: raise` — unknown prefixes fail loudly), and the two galsim + ellipticity-sign flips in `_shear_response` / `_selection_response`; +- `galaxy.classification_galaxy_galsim`; +- the `sh == "galsim"` branch in `catalog.get_snr` (now `else: raise`); +- the unused `shape_method` argument on `get_calibrated_quantities` / + `get_calibrated_m_c`, and the `galsim` mentions in `params.py` / `extract_info.py`. + +ngmix is the sole estimator sp_validation supports; the `prefix` parameter stays +(it names the column family and could serve a future `NGMIXm` moments family). + +## Coordinated (not this repo) -- **`*_PSF_ORIG` value re-validation** — blocked on a #761-regenerated catalogue. - The rename is staged; the "does the size-ratio / leakage cut still behave" - check waits for real v2 data. -- **`NGMIX_MOM_FAIL`** (read at `galaxy.py`) was not found in the #761 producer - code; left as-is pending confirmation against a regenerated header. -- **galsim family** (`GALSIM_*` reads in `calibration.py`, `galaxy.py`) is renamed - on the producer side too but is out of this constitution's ngmix+HSM scope and - is left untouched here — flagged for a follow-up if the galsim path is live. - **`shear_psf_leakage`** ρ/τ internals are Sacha Guerrini's separate PR; this migration only sets the `square_size=False` interface and passes `T`-columns. + Passing an already-`T` column with no squaring is correct independently of that + repo's own migration; the size-residual semantics there are worth a joint look. — Claude on behalf of Cail From 967b7e40d697be1740a884cb82895973092fedaf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 1 Jul 2026 17:27:06 +0200 Subject: [PATCH 11/11] refactor(rho_tau,cov): drop square_size, coordinate with leakage #27 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shear_psf_leakage #27 removed the square_size parameter from build_cat_to_compute_{rho,tau} and CovTauTh (HSM_T_* already stores the area T=2σ², so the flag was dead weight). sp_validation still threaded square_size into those leakage calls; once #27's container lands, passing it would raise TypeError. Drop the square_size key from both param builders (rho_tau.get_params_rho_tau, cosmo_val/compute_theory_cov.py) and stop passing it to the handlers. Behavior-identical against the current container (square_size defaults to False), required for the post-#27 one. Also correct the migration doc: galsim shapes are not producible by ShapePipe (no shape-measurement runner; make_cat's galsim mode reads an external catalogue nothing generates; production configs pin ngmix) — the decisive reason the dead consumer path was removed rather than migrated; and reframe the shear_psf_leakage coordination now that #27 is a sibling PR landing the same grammar. Suite: 137 passed, 1 skipped (unrelated cmss12.tfm font gap). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01GnCCTxKE13cHzPr8BUeKZE --- cosmo_val/compute_theory_cov.py | 2 - docs/ngmix_psf_column_migration.md | 61 ++++++++++++++++++++---------- src/sp_validation/rho_tau.py | 12 ------ 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/cosmo_val/compute_theory_cov.py b/cosmo_val/compute_theory_cov.py index 47d570b6..2868355f 100644 --- a/cosmo_val/compute_theory_cov.py +++ b/cosmo_val/compute_theory_cov.py @@ -18,8 +18,6 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - # ShapePipe-v2 stores every size as T = 2σ² (an area); nothing to square. - params["square_size"] = False params["R11"] = np.array([1]) params["R22"] = np.array([1]) if survey != "DES": diff --git a/docs/ngmix_psf_column_migration.md b/docs/ngmix_psf_column_migration.md index 99607bae..b24e9f83 100644 --- a/docs/ngmix_psf_column_migration.md +++ b/docs/ngmix_psf_column_migration.md @@ -122,11 +122,13 @@ truly held σ) now reads `HSM_T_*` directly. The σ→T change makes the old squaring dead: `HSM_T_*` (and DES's `piff_T`) already hold `T`, so nothing squares. The per-dataset `square_size:` flags in `cat_config.yaml` are dropped, the `not_square_size` list in `rho_tau.py` is -removed, and both param builders (`rho_tau.get_params_rho_tau`, -`cosmo_val/compute_theory_cov.py`) set `square_size = False`. The `square_size` -argument to `shear_psf_leakage`'s `build_cat_to_compute_{rho,tau}` is now always -`False`; passing a `T`-column with no squaring is correct independently of that -repo's own migration. +removed, and the `square_size` key is gone from both param builders +(`rho_tau.get_params_rho_tau`, `cosmo_val/compute_theory_cov.py`). The parameter +is also removed from `shear_psf_leakage`'s `build_cat_to_compute_{rho,tau}` and +`CovTauTh` ([PR #27](https://github.com/CosmoStat/shear_psf_leakage/pull/27)), so +sp_validation no longer *passes* it — the two migrations are coordinated. (Passing +`square_size=False` into the post-#27 leakage handlers would raise `TypeError`; +dropping the argument here is what keeps the container green once #27 lands.) ## Consumer sites in sp_validation @@ -149,16 +151,24 @@ repo's own migration. ## galsim estimator path — removed as dead code -shapepipe#761 renamed the galsim column family too (`GALSIM_GAL_ELL_*` / -`GALSIM_*_SIGMA_*` → scalar `GALSIM_G1/G2_*`, `GALSIM_T*`), but sp_validation's -galsim reader is **dead code**: `shape` is hardcoded to `"ngmix"`, -`extract_info.py` raises for any other value, and nothing outside `scratch/` -instantiates `metacal(prefix="GALSIM")` or calls `classification_galaxy_galsim`. -Migrating it would produce an untestable path — and the shared +**ShapePipe cannot produce galsim shapes.** There is no galsim shape-measurement +runner in the pipeline — `ngmix_runner` measures shapes and writes `NGMIX_*`, but +no galsim analogue exists. The only trace of galsim-as-shapes is a dormant +serialization hook in `make_cat` (`SHAPE_MEASUREMENT_TYPE=galsim` reads shapes from +an optional 5th input catalogue that nothing upstream fills), and every production +config sets `SHAPE_MEASUREMENT_TYPE=ngmix` with galsim commented out. shapepipe#761 +did rename the `GALSIM_*` family onto the grammar (`GALSIM_GAL_ELL_*` / +`GALSIM_*_SIGMA_*` → scalar `GALSIM_G1/G2_*`, `GALSIM_T*`), but that is cheap +serialization symmetry, not a live capability. + +On the consumer side the galsim reader was **dead and already broken**: `shape` is +hardcoded to `"ngmix"`, `extract_info.py` raises for any other value, nothing +outside `scratch/` instantiates `metacal(prefix="GALSIM")` or calls +`classification_galaxy_galsim`, and the shared `col_1p = f"{prefix}_T_PSF_RECONV_1P"` read in `metacal._read_data` never matched -the galsim producer output (galsim writes `GALSIM_T_PSF_*`, not -`..._T_PSF_RECONV_*`), so the path was already broken. So instead of migrating it, -this branch **removes** it: +the galsim producer output (`GALSIM_T_PSF_*`, not `..._T_PSF_RECONV_*`). Carrying +an untestable, already-broken path onto the new grammar is a worse end state than +deleting it, so this branch **removes** it: - `calibration.metacal._read_data_galsim`, the `prefix == "GALSIM"` dispatch branch (now `else: raise` — unknown prefixes fail loudly), and the two galsim @@ -170,12 +180,21 @@ this branch **removes** it: ngmix is the sole estimator sp_validation supports; the `prefix` parameter stays (it names the column family and could serve a future `NGMIXm` moments family). - -## Coordinated (not this repo) - -- **`shear_psf_leakage`** ρ/τ internals are Sacha Guerrini's separate PR; this - migration only sets the `square_size=False` interface and passes `T`-columns. - Passing an already-`T` column with no squaring is correct independently of that - repo's own migration; the size-residual semantics there are worth a joint look. +To revive galsim shapes, the capability must return **end-to-end**: a galsim +shape-measurement runner in ShapePipe (so the columns are producible at all), then +a migrated-and-tested consumer here — restore commit `e0eaa9f`, rename onto the +grammar, and add coverage — not the dead stub that was removed. + +## Coordinated repos + +- **`shear_psf_leakage`** ([PR #27](https://github.com/CosmoStat/shear_psf_leakage/pull/27), + "Adopt ShapePipe-v2 HSM column grammar, retire square_size") is the sibling + consumer migration. It lands on the same HSM grammar + (`HSM_G1/G2_{PSF,STAR}`, `HSM_T_*`, `HSM_FLAG_*`) and removed the `square_size` + parameter from `build_cat_to_compute_{rho,tau}` and `CovTauTh`; this PR drops the + matching argument, so the two land together (see "`square_size` is retired"). +- **shapepipe#761** (producer) still renames the `GALSIM_*` family onto the grammar + for columns nothing can create. If the goal is to simplify the grammar, + retiring that serialization is a producer-side follow-up worth raising there. — Claude on behalf of Cail diff --git a/src/sp_validation/rho_tau.py b/src/sp_validation/rho_tau.py index 5cd43973..6874a08c 100644 --- a/src/sp_validation/rho_tau.py +++ b/src/sp_validation/rho_tau.py @@ -40,8 +40,6 @@ def get_params_rho_tau(cat, survey="other"): params["e2_star_col"] = cat["psf"]["e2_star_col"] params["PSF_size"] = cat["psf"]["PSF_size"] params["star_size"] = cat["psf"]["star_size"] - # ShapePipe-v2 stores every size as T = 2σ² (an area); nothing to square. - params["square_size"] = False if survey != "DES": params["PSF_flag"] = cat["psf"]["PSF_flag"] params["star_flag"] = cat["psf"]["star_flag"] @@ -167,12 +165,10 @@ def get_rho_tau( rho_stat_handler.catalogs.set_params(params, outdir) mask = version != "DES" - square_size = params["square_size"] rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], catalog_id=catalog_id, - square_size=square_size, mask=mask, hdu=( config[version]["psf"]["hdu"] @@ -210,15 +206,12 @@ def get_rho_tau( mask = version != "DES" - square_size = params["square_size"] - # Build the different catalogs if necessary if f"psf_{version}" not in tau_stat_handler.catalogs.catalogs_dict.keys(): tau_stat_handler.build_cat_to_compute_tau( config[version]["psf"]["path"], cat_type="psf", catalog_id=version, - square_size=square_size, mask=mask, hdu=( config[version]["psf"]["hdu"] @@ -232,7 +225,6 @@ def get_rho_tau( config[version]["shear"]["path"], cat_type="gal", catalog_id=version, - square_size=square_size, mask=mask, ) @@ -346,8 +338,6 @@ def get_jackknife_cov( rho_stat_handler.catalogs.set_params(params, outdir) - square_size = params["square_size"] - tau_stat_handler = TauStat( catalogs=rho_stat_handler.catalogs, output=outdir, @@ -368,7 +358,6 @@ def get_jackknife_cov( rho_stat_handler.build_cat_to_compute_rho( config[version]["psf"]["path"], catalog_id=version + str(i), - square_size=square_size, mask=False, hdu=config[version]["psf"]["hdu"], ) @@ -382,7 +371,6 @@ def get_jackknife_cov( config[version]["shear"]["path"], cat_type="gal", catalog_id=version + str(i), - square_size=square_size, mask=False, )