diff --git a/config/repos.yaml b/config/repos.yaml index 6bbf62d..ad6ef65 100644 --- a/config/repos.yaml +++ b/config/repos.yaml @@ -123,7 +123,12 @@ noise_globs: - "*model_*.json" # model_0.json, model_1.json, … - "*max_log_likelihood.json" # result snapshots - "*galaxies.json" # generated dataset galaxies (HowTo*, *_test) + - "*dataset.json" # generated dataset payloads (weak/simple/dataset.json) + - "*tracer_*.json" # tracer_imaging.json / tracer_point.json variants + - "*point_dataset*.csv" # csv twins of the point_dataset json artifacts + - "*.npy" # generated numpy payloads (los_halo_list.npy etc.) - "*summary.json" # generated profiling summaries (postfix_N*_summary.json) + - "*summary_v*.json" # versioned profiling summaries (imaging_summary_v2026.5.14.2.json) - "*comparison.json" # jax_profiling result comparisons - "*hpc_*.json" # jax_profiling HPC run results (hpc_a100_fp64/mp.json) - "*.png" # generated plots / figures (global; PNGs here are always generated) diff --git a/heart/noise.py b/heart/noise.py index c96ef79..a6e8547 100644 --- a/heart/noise.py +++ b/heart/noise.py @@ -59,14 +59,16 @@ def _is_noise(code: str, path: str, noise_globs: list[str]) -> bool: return False -def classify_dirty( +def split_lines( porcelain_lines: list[str], noise_globs: list[str] ) -> tuple[list[str], list[str]]: - """Split porcelain lines into (real_paths, noise_paths). + """Split porcelain lines into (real_lines, noise_lines), codes intact. Each line is the raw ``git status --porcelain`` form: a two-char status code, a space, then the path (which may itself contain spaces). Rename - entries (``R old -> new``) are reduced to their destination path. + entries (``R old -> new``) are classified by their destination path. + The full lines are returned so callers can still distinguish untracked + (``??``) from modified entries. """ real: list[str] = [] noise: list[str] = [] @@ -80,12 +82,27 @@ def classify_dirty( if " -> " in path: # rename/copy: keep the destination path = path.split(" -> ", 1)[1] if _is_noise(code, path, noise_globs): - noise.append(path) + noise.append(line) else: - real.append(path) + real.append(line) return real, noise +def _line_path(line: str) -> str: + path = line[3:] + if " -> " in path: + path = path.split(" -> ", 1)[1] + return path + + +def classify_dirty( + porcelain_lines: list[str], noise_globs: list[str] +) -> tuple[list[str], list[str]]: + """Split porcelain lines into (real_paths, noise_paths) — see split_lines.""" + real_lines, noise_lines = split_lines(porcelain_lines, noise_globs) + return [_line_path(l) for l in real_lines], [_line_path(l) for l in noise_lines] + + def build_sidecar( *, name: str, @@ -116,19 +133,49 @@ def build_sidecar( } +def run_batch(in_dir: Path, real_out: Path, config: str) -> int: + """Classify one porcelain file per repo: ``/`` holds that + repo's ``git status --porcelain`` output. Real-only lines are written to + ``/`` and one ``\\t\\t\\t`` + line per repo goes to stdout (untracked = ``??`` code). No sidecar is + written — this is the display path (health_sync.sh), not the census. + """ + noise_globs = load_noise_globs(config) + real_out.mkdir(parents=True, exist_ok=True) + for f in sorted(in_dir.iterdir()): + if not f.is_file(): + continue + real_lines, noise_lines = split_lines(f.read_text().splitlines(), noise_globs) + (real_out / f.name).write_text( + "\n".join(real_lines) + ("\n" if real_lines else "") + ) + real_untr = sum(1 for l in real_lines if l[:2] == "??") + print(f"{f.name}\t{len(real_lines) - real_untr}\t{real_untr}\t{len(noise_lines)}") + return 0 + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="heart.noise") - ap.add_argument("--name", required=True) + ap.add_argument("--name") ap.add_argument("--group", default="") ap.add_argument("--branch", default="") ap.add_argument("--ahead", type=int, default=0) ap.add_argument("--behind", type=int, default=0) ap.add_argument("--upstream", default="") ap.add_argument("--ts", default="") - ap.add_argument("--out", required=True, help="sidecar JSON path to write") + ap.add_argument("--out", help="sidecar JSON path to write") ap.add_argument("--config", default=str(DEFAULT_CONFIG)) + ap.add_argument("--batch", help="dir of per-repo porcelain files (no sidecar mode)") + ap.add_argument("--real-out", help="dir to write per-repo real-only porcelain (with --batch)") ns = ap.parse_args(argv) + if ns.batch: + if not ns.real_out: + ap.error("--batch requires --real-out") + return run_batch(Path(ns.batch), Path(ns.real_out), ns.config) + if not ns.name or not ns.out: + ap.error("--name and --out are required (unless using --batch)") + ts = ns.ts or datetime.datetime.now(datetime.timezone.utc).isoformat() porcelain_lines = sys.stdin.read().splitlines() noise_globs = load_noise_globs(ns.config) diff --git a/scripts/health.sh b/scripts/health.sh index 376fe8f..74df4ec 100644 --- a/scripts/health.sh +++ b/scripts/health.sh @@ -31,16 +31,21 @@ health() { cat <<'EOF' health — local health/dev shell dispatcher (mirrors the Claude /health door). -Usage: health [sync|release|audit] +Usage: health [sync|release|audit] [--all] health cross-repo git-sync dashboard (branch, behind/ahead, dirty) health sync same as bare `health` + health --all sync dashboard with the full raw dirty listing (noise included) health release last PyAutoBuild release-prep run dashboard health audit structural repo-health audit (non-repo dirs, stashes, dead branches) Release-run helpers: health-report / health-json / health-triage. EOF ;; + -*) + # Leading-dash first arg (e.g. `health --all`) is a sync flag, not a + # subcommand — route to the default sync view with the flag intact. + _health_sync "$sub" "$@" ;; *) echo "health: unknown subcommand '$sub' (try: health help)" >&2 return 2 diff --git a/scripts/health_sync.sh b/scripts/health_sync.sh index 51a5cfd..6557ac0 100644 --- a/scripts/health_sync.sh +++ b/scripts/health_sync.sh @@ -10,19 +10,29 @@ # Usage (normally sourced via ~/.bashrc, run through the `health` dispatcher): # source ~/Code/PyAutoLabs/PyAutoMind/scripts/health_sync.sh # health # (or: health sync) +# health --all # full raw dirty-file listing (noise included) # # Override the repo root (e.g. for testing) via PYAUTO_STATUS_ROOT. # +# Dirty counts are classified through Heart's real-vs-noise splitter +# (heart/noise.py + noise_globs in config/repos.yaml): MOD/UNTR count *real* +# source drift only, the NOISE column counts regenerated artifacts (*.fits, +# *data.json, *.png, …). If the classifier is unavailable (no PyAutoHeart +# checkout / python failure) the raw counts are shown with NOISE=? and the +# listing falls back to today's full form. +# # Flag glyphs (FLAGS column): # ↓ behind upstream # ↑ ahead of upstream -# * dirty (modified or untracked files) +# * dirty (real modified or untracked files) +# ~ noise only (regenerated artifacts; no real drift) # ! no upstream / fetch failed # b current branch ≠ upstream branch (forgotten feature branch) # -# After the main table, four optional sections may follow: -# - "Dirty files:" — per-repo `git status --porcelain` for any repo -# with mod or untr > 0. +# After the main table, optional sections may follow: +# - "Dirty files:" — per-repo listing of *real* dirty files, with +# noise collapsed to a one-line count per repo +# (`health --all` restores the raw listing). # - "Follow-up commands:" — copy-pasteable git invocations grouped by # category (pull / set-upstream / investigate). # Suppressed entirely when nothing is actionable. @@ -34,6 +44,10 @@ # ~/Code/PyAutoLabs/PyAutoBuild/test_results/*.json # (committed by the autobuild release pipeline). # Suppressed when no JSONs exist. +# - "Hygiene:" — nudge when the last /repo_cleanup audit stamp +# (~/.cache/pyauto/repo_cleanup_last_audit.json) +# is missing or older than PYAUTO_CLEANUP_NUDGE_DAYS +# (default 7). Suppressed when the stamp is fresh. # # Note: distinct from the Claude `/health status` command (the active-work # registry dashboard). This shell command shows git *sync* state across repos; @@ -49,6 +63,13 @@ _health_sync() { return 1 fi + local show_all=false arg + for arg in "$@"; do + case "$arg" in + --all) show_all=true ;; + esac + done + # Discover repos. `.git` is a directory in normal checkouts and a file in # linked worktrees, so accept both. mindepth/maxdepth 2 limits us to the # immediate children of $root. @@ -93,19 +114,16 @@ _health_sync() { wait ) 2>/dev/null - # Header. - local fmt='%-32s %-30s %-36s %6s %5s %4s %4s %s\n' - printf "$fmt" REPO BRANCH UPSTREAM BEHIND AHEAD MOD UNTR FLAGS - printf "$fmt" "--------------------------------" \ - "------------------------------" \ - "------------------------------------" \ - "------" "-----" "----" "----" "-----" - - # Per-repo row. Porcelain is cached so the dirty-files listing below can - # reuse it without a second `git status` per repo. Action arrays collect - # actionable follow-ups for the "Follow-up commands:" section printed at - # the end. - declare -A repo_porcelain + # Pass 1 — gather per-repo state without printing. The table itself is + # printed in pass 2, after the noise classifier has run over every repo's + # porcelain in one batch (per-row python calls would blow the <10s budget). + # Porcelain is cached so the dirty-files listing below can reuse it without + # a second `git status` per repo. Action arrays collect actionable + # follow-ups for the "Follow-up commands:" section printed at the end. + local noise_tmp="$fetch_status_dir/noise" + mkdir -p "$noise_tmp/p" "$noise_tmp/real" + declare -A repo_porcelain repo_branch repo_upstream repo_behind repo_ahead \ + repo_flags repo_mod_raw repo_untr_raw real_mod real_untr noise_n local actions_pull=() actions_set_upstream=() actions_manual=() local name branch upstream upstream_branch behind ahead mod untr flags counts porcelain branch_mismatch b_int a_int for repo in "${repos[@]}"; do @@ -149,6 +167,7 @@ _health_sync() { else untr="$(printf '%s\n' "$porcelain" | grep -c '^??' || true)" mod="$(printf '%s\n' "$porcelain" | grep -cv '^??' || true)" + printf '%s\n' "$porcelain" > "$noise_tmp/p/$name" fi # Branch-mismatch detection. With no upstream, the heuristic is @@ -163,10 +182,17 @@ _health_sync() { [[ "$behind" =~ ^[0-9]+$ ]] && (( behind > 0 )) && flags+="↓" [[ "$ahead" =~ ^[0-9]+$ ]] && (( ahead > 0 )) && flags+="↑" - (( mod + untr > 0 )) && flags+="*" [[ "$branch_mismatch" == "true" ]] && flags+="b" - printf "$fmt" "$name" "$branch" "$upstream" "$behind" "$ahead" "$mod" "$untr" "$flags" + # Dirty flags (* / ~) depend on the noise classification and are added in + # pass 2; everything else about the row is fixed here. + repo_branch["$name"]="$branch" + repo_upstream["$name"]="$upstream" + repo_behind["$name"]="$behind" + repo_ahead["$name"]="$ahead" + repo_flags["$name"]="$flags" + repo_mod_raw["$name"]="$mod" + repo_untr_raw["$name"]="$untr" # Categorise actionable follow-ups. Only the boring case (clean, behind, # not ahead) becomes an auto-runnable command; everything else is @@ -193,10 +219,64 @@ _health_sync() { fi done + # Noise classification — one batched heart.noise call over every dirty + # repo's porcelain. Real-only porcelain lands in $noise_tmp/real/, + # counts on stdout as "\t\t\t". Any + # failure (no PyAutoHeart checkout, no PyYAML, …) degrades to the raw + # behaviour: classify_ok=false, NOISE column shows "?". PYAUTO_HEART_HOME + # points at an alternative Heart checkout (e.g. a task worktree). + local heart_home="${PYAUTO_HEART_HOME:-$root/PyAutoHeart}" + local classify_ok=false + if [[ -f "$heart_home/heart/noise.py" ]]; then + if PYTHONPATH="$heart_home" python3 -m heart.noise \ + --batch "$noise_tmp/p" --real-out "$noise_tmp/real" \ + --config "$heart_home/config/repos.yaml" > "$noise_tmp/counts" 2>/dev/null; then + classify_ok=true + local cname crm cru cnn + while IFS=$'\t' read -r cname crm cru cnn; do + [[ -z "$cname" ]] && continue + real_mod["$cname"]="$crm" + real_untr["$cname"]="$cru" + noise_n["$cname"]="$cnn" + done < "$noise_tmp/counts" + fi + fi + + # Pass 2 — print the table. MOD/UNTR are real-file counts when the + # classifier ran (raw counts otherwise); NOISE carries the regenerated- + # artifact count. `*` marks real drift, `~` noise-only dirt. + local fmt='%-32s %-30s %-36s %6s %5s %4s %4s %5s %s\n' + printf "$fmt" REPO BRANCH UPSTREAM BEHIND AHEAD MOD UNTR NOISE FLAGS + printf "$fmt" "--------------------------------" \ + "------------------------------" \ + "------------------------------------" \ + "------" "-----" "----" "----" "-----" "-----" + + local noise nreal + for repo in "${repos[@]}"; do + name="$(basename "$repo")" + flags="${repo_flags[$name]}" + if [[ "$classify_ok" == "true" ]]; then + mod="${real_mod[$name]:-0}" + untr="${real_untr[$name]:-0}" + noise="${noise_n[$name]:-0}" + (( mod + untr > 0 )) && flags+="*" + (( noise > 0 )) && flags+="~" + else + mod="${repo_mod_raw[$name]}" + untr="${repo_untr_raw[$name]}" + noise="?" + (( mod + untr > 0 )) && flags+="*" + fi + printf "$fmt" "$name" "${repo_branch[$name]}" "${repo_upstream[$name]}" \ + "${repo_behind[$name]}" "${repo_ahead[$name]}" "$mod" "$untr" "$noise" "$flags" + done + # Per-repo dirty-file listing. Only repos with non-empty porcelain are # shown — keeps the output empty when everything is clean. The `??` and # ` M` etc. prefixes from porcelain are preserved so users can tell - # untracked from modified at a glance. + # untracked from modified at a glance. With the classifier available (and + # without --all) only *real* files are listed; noise collapses to a count. local printed_header=false for repo in "${repos[@]}"; do name="$(basename "$repo")" @@ -207,9 +287,24 @@ _health_sync() { echo "Dirty files:" printed_header=true fi - echo " $name:" - printf '%s\n' "$porcelain" | sed 's/^/ /' + if [[ "$show_all" == "true" || "$classify_ok" != "true" ]]; then + echo " $name:" + printf '%s\n' "$porcelain" | sed 's/^/ /' + continue + fi + nreal=$(( ${real_mod[$name]:-0} + ${real_untr[$name]:-0} )) + noise="${noise_n[$name]:-0}" + if (( nreal == 0 )); then + echo " $name: $noise noise artifact(s) (regenerated) — \`health --all\` to list" + else + echo " $name:" + sed 's/^/ /' "$noise_tmp/real/$name" 2>/dev/null + (( noise > 0 )) && echo " (+$noise noise artifact(s) — \`health --all\` to list)" + fi done + if [[ "$printed_header" == "true" && "$classify_ok" != "true" ]]; then + echo " (noise classification unavailable — raw listing shown)" + fi # Follow-up commands. Suppressed entirely when nothing is actionable so # the clean case stays quiet. The `git -C ` form means each @@ -309,4 +404,20 @@ print(f"{latest[:10]}|{num}|{len(projects)}|{total_p}|{total_f}|{total_s}") "$njobs" "$nproj" "$pab_p" "$pab_f" "$pab_s" fi fi + + # Hygiene nudge. /repo_cleanup writes this stamp after each audit; the + # mtime is the audit time. Missing or stale (> PYAUTO_CLEANUP_NUDGE_DAYS, + # default 7) prints a reminder — the cadence mechanism for the sweep. + local stamp="$HOME/.cache/pyauto/repo_cleanup_last_audit.json" + local nudge_days="${PYAUTO_CLEANUP_NUDGE_DAYS:-7}" age_days + if [[ ! -f "$stamp" ]]; then + echo "" + echo "Hygiene: no /repo_cleanup audit on record — consider running /repo_cleanup." + else + age_days=$(( ( $(date +%s) - $(stat -c %Y "$stamp" 2>/dev/null || echo 0) ) / 86400 )) + if (( age_days > nudge_days )); then + echo "" + echo "Hygiene: last /repo_cleanup audit ${age_days} days ago — consider running /repo_cleanup." + fi + fi } diff --git a/tests/test_noise.py b/tests/test_noise.py index 4fc6d5d..45d188a 100644 --- a/tests/test_noise.py +++ b/tests/test_noise.py @@ -20,7 +20,12 @@ "*model_*.json", "*max_log_likelihood.json", "*galaxies.json", + "*dataset.json", + "*tracer_*.json", + "*point_dataset*.csv", + "*.npy", "*summary.json", + "*summary_v*.json", "*comparison.json", "*hpc_*.json", "*.png", @@ -207,6 +212,86 @@ def test_load_noise_globs_missing_file_returns_empty(tmp_path): assert noise.load_noise_globs(tmp_path / "nope.yaml") == [] +# --- v1.6: line-level split + batch mode (the health_sync dashboard path) --- + + +def test_split_lines_preserves_codes(): + real_lines, noise_lines = noise.split_lines(SAMPLE, GLOBS) + assert " M util.py" in real_lines + assert "?? new_module.py" in real_lines + assert " M dataset/build/imaging/no_lens_light/data.fits" in noise_lines + assert "?? test_report.md" in noise_lines + + +def test_split_lines_rename_classified_by_destination(): + real_lines, noise_lines = noise.split_lines( + ["R old.py -> new.py", "R raw.txt -> plot.png"], GLOBS + ) + assert real_lines == ["R old.py -> new.py"] + assert noise_lines == ["R raw.txt -> plot.png"] + + +def test_classify_dirty_matches_split_lines(): + # classify_dirty is a path-projection of split_lines — same partition. + real, noise_files = noise.classify_dirty(SAMPLE, GLOBS) + real_lines, noise_lines = noise.split_lines(SAMPLE, GLOBS) + assert len(real) == len(real_lines) + assert len(noise_files) == len(noise_lines) + + +def test_sweep_glob_gaps_are_noise(): + # The 2026-07-09 repo_cleanup sweep found these slipping through. + sample = [ + " M dataset/imaging/los_halos/los_halo_list.npy", + " M dataset/point_source/deblending/tracer_imaging.json", + " M dataset/point_source/deblending/tracer_point.json", + " M dataset/point_source/simple/point_dataset_positions_only.csv", + " M dataset/weak/simple/dataset.json", + "?? results/simulators/imaging_summary_v2026.5.14.2.json", + " M scripts/scaling_relation.csv.py", # .py stays real + ] + real, noise_files = noise.classify_dirty(sample, GLOBS) + assert real == ["scripts/scaling_relation.csv.py"] + assert len(noise_files) == 6 + + +def test_batch_mode_roundtrip(tmp_path, capsys): + in_dir = tmp_path / "p" + out_dir = tmp_path / "real" + in_dir.mkdir() + (in_dir / "ws_mixed").write_text( + " M util.py\n M data.fits\n?? new_module.py\n?? plot.png\n" + ) + (in_dir / "ws_noise_only").write_text(" M a.fits\n M b.png\n") + + config = tmp_path / "repos.yaml" + config.write_text("noise_globs:\n - '*.fits'\n - '*.png'\n") + + rc = noise.run_batch(in_dir, out_dir, str(config)) + assert rc == 0 + + lines = capsys.readouterr().out.splitlines() + assert "ws_mixed\t1\t1\t2" in lines # 1 real mod, 1 real untracked, 2 noise + assert "ws_noise_only\t0\t0\t2" in lines + assert (out_dir / "ws_mixed").read_text() == " M util.py\n?? new_module.py\n" + assert (out_dir / "ws_noise_only").read_text() == "" + + +def test_batch_mode_via_cli(tmp_path, capsys): + in_dir = tmp_path / "p" + in_dir.mkdir() + (in_dir / "repo").write_text(" M x.fits\n M y.py\n") + config = tmp_path / "repos.yaml" + config.write_text("noise_globs:\n - '*.fits'\n") + + rc = noise.main( + ["--batch", str(in_dir), "--real-out", str(tmp_path / "real"), "--config", str(config)] + ) + assert rc == 0 + assert "repo\t1\t0\t1" in capsys.readouterr().out + assert (tmp_path / "real" / "repo").read_text() == " M y.py\n" + + def test_build_sidecar_shape(): sidecar = noise.build_sidecar( name="autolens_workspace_test",