From c450896dd4e77b50ae5839678f2a1d6f22f5e772 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 13:12:16 +0000 Subject: [PATCH 1/2] Add release-fidelity env profile (config/build/env_vars_release.yaml) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M3 of the Brain->Health->Heart release-validation redesign: PyAutoHeart's workspace-validation.yml mode=release path passes this file to Build's run_python.py via --env-config so release rehearsals run at release fidelity instead of smoke fidelity. Self-contained sibling of env_vars.yaml (the smoke profile) — no PyAutoBuild changes needed, since --env-config already accepted an arbitrary path. Spec + acceptance table: PyAutoHeart/docs/release_validation.md. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V4xvQJFsdgi2DtSdF89EqX --- config/build/env_vars_release.yaml | 115 +++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 config/build/env_vars_release.yaml diff --git a/config/build/env_vars_release.yaml b/config/build/env_vars_release.yaml new file mode 100644 index 0000000..17ae016 --- /dev/null +++ b/config/build/env_vars_release.yaml @@ -0,0 +1,115 @@ +# Per-script environment variable configuration for the RELEASE-FIDELITY +# validation run (Heart's workspace-validation.yml, mode=release — the M3 +# wheel-based release-fidelity path). Distinct from env_vars.yaml, which is the +# `smoke` profile used by the per-PR CI gate. +# +# The `release` profile runs REAL searches (PYAUTO_TEST_MODE=0 — each script's +# own n_like_max caps keep this bounded) at full resolution: PYAUTO_SMALL_DATASETS +# and PYAUTO_FAST_PLOTS are deliberately absent from "defaults" below (unset == +# full-res grids, real plots), unlike the smoke profile which caps both for +# speed. Spec + acceptance table: PyAutoHeart/docs/release_validation.md. +# +# "defaults" are applied to every script on top of the inherited environment. +# "overrides" selectively unset or replace vars for matching path patterns — +# identical in structure and intent to env_vars.yaml's overrides; they still +# layer on top of this profile's defaults rather than the smoke ones. Most +# PYAUTO_SMALL_DATASETS-unsetting overrides are no-ops here (already unset by +# default) but are kept so the two profiles stay easy to diff against each other. +# +# Pattern convention (same as no_run.yaml): +# - Patterns containing '/' do a substring match against the file path +# including extension (so patterns may end in `.py` to anchor against +# the script form, e.g. `imaging/visualization.py` matches only the +# `.py` script, not `imaging/visualization_jax.py`). +# - Patterns without '/' match the file stem exactly. + +defaults: + PYAUTO_TEST_MODE: "0" # real searches (each script caps its own n_like_max) + PYAUTO_DISABLE_JAX: "1" # force use_jax=False by default; JAX-specific folders re-enable below + PYAUTO_SKIP_WORKSPACE_VERSION_CHECK: "1" # TestPyPI dev version won't match the workspace pin + JAX_ENABLE_X64: "True" # enable 64-bit precision in JAX + NUMBA_CACHE_DIR: "/tmp/numba_cache" # writable cache dir for numba + MPLCONFIGDIR: "/tmp/matplotlib" # writable config dir for matplotlib + +overrides: + # JAX likelihood functions test JIT compilation — need JAX enabled and full-size datasets + - pattern: "jax_likelihood_functions/" + unset: [PYAUTO_SMALL_DATASETS, PYAUTO_DISABLE_JAX] + # JAX gradient scripts exercise jax.value_and_grad on the full likelihood path — + # need JAX enabled and full-size datasets, same as jax_likelihood_functions/. + - pattern: "jax_grad/" + unset: [PYAUTO_SMALL_DATASETS, PYAUTO_DISABLE_JAX] + # convolver_mixed_precision asserts the convolver dispatches mixed-precision + # arrays cleanly. It loads a pre-committed 80x80 array; PYAUTO_SMALL_DATASETS=1 + # would cap the mask to 15x15 and the assertion would fail with a shape mismatch + # before the precision check even ran. + - pattern: "jax_assertions/convolver_mixed_precision" + unset: [PYAUTO_SMALL_DATASETS] + # Model composition scripts assert exact prior_count for configured gaussian_per_basis. + # PYAUTO_SMALL_DATASETS=1 reduces total_gaussians and changes prior_count. + - pattern: "model_composition/" + unset: [PYAUTO_SMALL_DATASETS] + # Aggregator scripts assert grid metadata (e.g. over_sample_size_lp == 5) + # that depends on full-size datasets. PYAUTO_SMALL_DATASETS=1 reduces these. + - pattern: "aggregator/" + unset: [PYAUTO_SMALL_DATASETS] + # imaging/model_fit reads pre-committed FITS data at full resolution; the + # mask is built from the committed dataset shape, so PYAUTO_SMALL_DATASETS=1 + # would yield a 15x15 array against a full-size mask. + - pattern: "imaging/model_fit" + unset: [PYAUTO_SMALL_DATASETS] + # imaging/visualization.py asserts subplot PNG / FITS files land on disk — + # PYAUTO_FAST_PLOTS skips savefig and would break those assertions. It also + # reads the same full-resolution dataset as imaging/model_fit, so the + # PYAUTO_SMALL_DATASETS cap would produce a shape mismatch with the mask. + - pattern: "imaging/visualization.py" + unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # imaging/visualization_jax.py exercises the jit-cached fit_for_visualization + # path (registered model + fit pytree). It must run with JAX enabled — + # PYAUTO_DISABLE_JAX=1 would silently flip use_jax flags off and the script + # would no-op via the NumPy fallback. + - pattern: "imaging/visualization_jax" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # ellipse/visualization.py asserts subplot PNG / FITS files land on disk + # (same intent as imaging/visualization.py). Loads pre-committed FITS at + # full resolution and builds the mask from dataset.shape_native, so both + # PYAUTO_FAST_PLOTS and PYAUTO_SMALL_DATASETS must be unset. + - pattern: "ellipse/visualization.py" + unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # ellipse/visualization_jax exercises the JAX-capable ellipse visualization + # path and asserts plot files land on disk. + - pattern: "ellipse/visualization_jax" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # ellipse/modeling_visualization_jit — live Nautilus + JAX visualization path. + - pattern: "ellipse/modeling_visualization_jit" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] + # modeling_visualization_jit tests the JIT-cached visualization path: + # needs JAX enabled (Part 1 asserts log_likelihood is a jax.Array), the + # full-resolution mask, a real Nautilus run for Part 2, and savefig active + # (Part 2 asserts fit.png lands on disk). + - pattern: "imaging/modeling_visualization_jit" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] + # interferometer/visualization.py asserts subplot PNG / FITS files land on + # disk. Loads pre-committed FITS at full resolution and uses an explicit + # (256, 256) shape_native mask — both PYAUTO_FAST_PLOTS and + # PYAUTO_SMALL_DATASETS must be unset (the 15x15 cap caps even when + # shape_native is explicit). + - pattern: "interferometer/visualization.py" + unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # interferometer/visualization_jax exercises the jit-cached + # fit_for_visualization path on the interferometer side. + - pattern: "interferometer/visualization_jax" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # interferometer/modeling_visualization_jit — live Nautilus + JIT path. + - pattern: "interferometer/modeling_visualization_jit" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] + # quantity/visualization.py asserts subplot PNG files land on disk and + # builds an explicit (30, 30) shape_native mask which would mismatch the + # 15x15 cap from PYAUTO_SMALL_DATASETS=1. + - pattern: "quantity/visualization.py" + unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + # quantity/visualization_jax exercises the jit-cached + # fit_for_visualization path on the quantity side (wired in + # PyAutoGalaxy #404). + - pattern: "quantity/visualization_jax" + unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] From d9b54f1d30160bcca9d9a34f48daf48761a5c2b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Jul 2026 13:42:41 +0000 Subject: [PATCH 2/2] env_vars_release.yaml: address Copilot review on PR #67 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot correctly flagged: (1) PYAUTO_SMALL_DATASETS/PYAUTO_FAST_PLOTS were left absent from "defaults" (meant "full-res, real plots") rather than explicitly "0", so an inherited smoke-mode "1" would silently survive; (2) ellipse/modeling_visualization_jit, imaging/modeling_visualization_jit, and interferometer/modeling_visualization_jit all `unset: [PYAUTO_TEST_MODE, ...]` even though defaults already pins PYAUTO_TEST_MODE="0" — an unnecessary and risky re-exposure to inherited/ambient env. Fixed by pinning every var this profile cares about explicitly in defaults, and converting every override to `set:` only what genuinely differs from the pinned defaults — in every remaining case here, just PYAUTO_DISABLE_JAX. This dropped 10 of 17 override entries entirely (now fully redundant). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01V4xvQJFsdgi2DtSdF89EqX --- config/build/env_vars_release.yaml | 122 +++++++++-------------------- 1 file changed, 39 insertions(+), 83 deletions(-) diff --git a/config/build/env_vars_release.yaml b/config/build/env_vars_release.yaml index 17ae016..0bf4795 100644 --- a/config/build/env_vars_release.yaml +++ b/config/build/env_vars_release.yaml @@ -3,18 +3,27 @@ # wheel-based release-fidelity path). Distinct from env_vars.yaml, which is the # `smoke` profile used by the per-PR CI gate. # -# The `release` profile runs REAL searches (PYAUTO_TEST_MODE=0 — each script's -# own n_like_max caps keep this bounded) at full resolution: PYAUTO_SMALL_DATASETS -# and PYAUTO_FAST_PLOTS are deliberately absent from "defaults" below (unset == -# full-res grids, real plots), unlike the smoke profile which caps both for -# speed. Spec + acceptance table: PyAutoHeart/docs/release_validation.md. +# The `release` profile runs REAL searches (PYAUTO_TEST_MODE="0" — each +# script's own n_like_max caps keep this bounded) at full resolution. +# Spec + acceptance table: PyAutoHeart/docs/release_validation.md. # # "defaults" are applied to every script on top of the inherited environment. -# "overrides" selectively unset or replace vars for matching path patterns — -# identical in structure and intent to env_vars.yaml's overrides; they still -# layer on top of this profile's defaults rather than the smoke ones. Most -# PYAUTO_SMALL_DATASETS-unsetting overrides are no-ops here (already unset by -# default) but are kept so the two profiles stay easy to diff against each other. +# Every var this profile cares about is given an EXPLICIT value in "defaults" +# (not left absent) — the runner only ever *sets* keys it's given, it never +# clears unrelated inherited env vars, so an absent key silently falls through +# to whatever the calling process already had (a leftover smoke-mode "1" from +# an earlier step, a developer's local shell, ...). Pinning everything here +# — including PYAUTO_SMALL_DATASETS="0" / PYAUTO_FAST_PLOTS="0" for full-res, +# real plots — makes the profile self-contained regardless of the caller's +# environment. +# +# "overrides" then only ever `set:` a var to flip it away from this profile's +# own default for specific scripts — never `unset:` a var that "defaults" +# already pins, since unsetting just re-exposes the same inherited-env gap +# `defaults` exists to close. In practice that leaves PYAUTO_DISABLE_JAX as +# the only var any override here still needs to touch (PYAUTO_TEST_MODE="0" +# and the full-res/real-plot defaults already satisfy every other unset the +# smoke profile's env_vars.yaml needs). # # Pattern convention (same as no_run.yaml): # - Patterns containing '/' do a substring match against the file path @@ -24,7 +33,9 @@ # - Patterns without '/' match the file stem exactly. defaults: - PYAUTO_TEST_MODE: "0" # real searches (each script caps its own n_like_max) + PYAUTO_TEST_MODE: "0" # real searches (each script caps its own n_like_max) + PYAUTO_SMALL_DATASETS: "0" # full-res grids/masks (release fidelity, not smoke) + PYAUTO_FAST_PLOTS: "0" # real plots (release fidelity, not smoke) PYAUTO_DISABLE_JAX: "1" # force use_jax=False by default; JAX-specific folders re-enable below PYAUTO_SKIP_WORKSPACE_VERSION_CHECK: "1" # TestPyPI dev version won't match the workspace pin JAX_ENABLE_X64: "True" # enable 64-bit precision in JAX @@ -32,84 +43,29 @@ defaults: MPLCONFIGDIR: "/tmp/matplotlib" # writable config dir for matplotlib overrides: - # JAX likelihood functions test JIT compilation — need JAX enabled and full-size datasets + # JAX likelihood/gradient scripts test JIT compilation — need JAX enabled + # (already full-res/real-plots by default above). - pattern: "jax_likelihood_functions/" - unset: [PYAUTO_SMALL_DATASETS, PYAUTO_DISABLE_JAX] - # JAX gradient scripts exercise jax.value_and_grad on the full likelihood path — - # need JAX enabled and full-size datasets, same as jax_likelihood_functions/. + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "jax_grad/" - unset: [PYAUTO_SMALL_DATASETS, PYAUTO_DISABLE_JAX] - # convolver_mixed_precision asserts the convolver dispatches mixed-precision - # arrays cleanly. It loads a pre-committed 80x80 array; PYAUTO_SMALL_DATASETS=1 - # would cap the mask to 15x15 and the assertion would fail with a shape mismatch - # before the precision check even ran. - - pattern: "jax_assertions/convolver_mixed_precision" - unset: [PYAUTO_SMALL_DATASETS] - # Model composition scripts assert exact prior_count for configured gaussian_per_basis. - # PYAUTO_SMALL_DATASETS=1 reduces total_gaussians and changes prior_count. - - pattern: "model_composition/" - unset: [PYAUTO_SMALL_DATASETS] - # Aggregator scripts assert grid metadata (e.g. over_sample_size_lp == 5) - # that depends on full-size datasets. PYAUTO_SMALL_DATASETS=1 reduces these. - - pattern: "aggregator/" - unset: [PYAUTO_SMALL_DATASETS] - # imaging/model_fit reads pre-committed FITS data at full resolution; the - # mask is built from the committed dataset shape, so PYAUTO_SMALL_DATASETS=1 - # would yield a 15x15 array against a full-size mask. - - pattern: "imaging/model_fit" - unset: [PYAUTO_SMALL_DATASETS] - # imaging/visualization.py asserts subplot PNG / FITS files land on disk — - # PYAUTO_FAST_PLOTS skips savefig and would break those assertions. It also - # reads the same full-resolution dataset as imaging/model_fit, so the - # PYAUTO_SMALL_DATASETS cap would produce a shape mismatch with the mask. - - pattern: "imaging/visualization.py" - unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # imaging/visualization_jax.py exercises the jit-cached fit_for_visualization - # path (registered model + fit pytree). It must run with JAX enabled — - # PYAUTO_DISABLE_JAX=1 would silently flip use_jax flags off and the script - # would no-op via the NumPy fallback. + set: { PYAUTO_DISABLE_JAX: "0" } + # visualization_jax scripts (imaging/interferometer/ellipse/quantity) and + # modeling_visualization_jit scripts (imaging/interferometer/ellipse) + # exercise the jit-cached fit_for_visualization path and must run with JAX + # enabled — PYAUTO_DISABLE_JAX="1" would silently flip use_jax flags off. + # PYAUTO_TEST_MODE="0" (real Nautilus run) and full-res/real-plots are + # already this profile's defaults. - pattern: "imaging/visualization_jax" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # ellipse/visualization.py asserts subplot PNG / FITS files land on disk - # (same intent as imaging/visualization.py). Loads pre-committed FITS at - # full resolution and builds the mask from dataset.shape_native, so both - # PYAUTO_FAST_PLOTS and PYAUTO_SMALL_DATASETS must be unset. - - pattern: "ellipse/visualization.py" - unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # ellipse/visualization_jax exercises the JAX-capable ellipse visualization - # path and asserts plot files land on disk. + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "ellipse/visualization_jax" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # ellipse/modeling_visualization_jit — live Nautilus + JAX visualization path. + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "ellipse/modeling_visualization_jit" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] - # modeling_visualization_jit tests the JIT-cached visualization path: - # needs JAX enabled (Part 1 asserts log_likelihood is a jax.Array), the - # full-resolution mask, a real Nautilus run for Part 2, and savefig active - # (Part 2 asserts fit.png lands on disk). + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "imaging/modeling_visualization_jit" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] - # interferometer/visualization.py asserts subplot PNG / FITS files land on - # disk. Loads pre-committed FITS at full resolution and uses an explicit - # (256, 256) shape_native mask — both PYAUTO_FAST_PLOTS and - # PYAUTO_SMALL_DATASETS must be unset (the 15x15 cap caps even when - # shape_native is explicit). - - pattern: "interferometer/visualization.py" - unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # interferometer/visualization_jax exercises the jit-cached - # fit_for_visualization path on the interferometer side. + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "interferometer/visualization_jax" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # interferometer/modeling_visualization_jit — live Nautilus + JIT path. + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "interferometer/modeling_visualization_jit" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_SMALL_DATASETS, PYAUTO_TEST_MODE, PYAUTO_FAST_PLOTS] - # quantity/visualization.py asserts subplot PNG files land on disk and - # builds an explicit (30, 30) shape_native mask which would mismatch the - # 15x15 cap from PYAUTO_SMALL_DATASETS=1. - - pattern: "quantity/visualization.py" - unset: [PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] - # quantity/visualization_jax exercises the jit-cached - # fit_for_visualization path on the quantity side (wired in - # PyAutoGalaxy #404). + set: { PYAUTO_DISABLE_JAX: "0" } - pattern: "quantity/visualization_jax" - unset: [PYAUTO_DISABLE_JAX, PYAUTO_FAST_PLOTS, PYAUTO_SMALL_DATASETS] + set: { PYAUTO_DISABLE_JAX: "0" }