Skip to content

feat: noise-filter the health sync dashboard + repo_cleanup hygiene nudge #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jammy2211 merged 1 commit into from
Jul 9, 2026
Merged

feat: noise-filter the health sync dashboard + repo_cleanup hygiene nudge #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions config/repos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,12 @@ noise_globs:
- "*model_*.json" # model_0.json, model_1.json, …
- "*max_log_likelihood.json" # result snapshots
- "*galaxies.json" # generated dataset galaxies (HowTo*, *_test)
- "*dataset.json" # generated dataset payloads (weak/simple/dataset.json)
- "*tracer_*.json" # tracer_imaging.json / tracer_point.json variants
- "*point_dataset*.csv" # csv twins of the point_dataset json artifacts
- "*.npy" # generated numpy payloads (los_halo_list.npy etc.)
- "*summary.json" # generated profiling summaries (postfix_N*_summary.json)
- "*summary_v*.json" # versioned profiling summaries (imaging_summary_v2026.5.14.2.json)
- "*comparison.json" # jax_profiling result comparisons
- "*hpc_*.json" # jax_profiling HPC run results (hpc_a100_fp64/mp.json)
- "*.png" # generated plots / figures (global; PNGs here are always generated)
Expand Down
61 changes: 54 additions & 7 deletions heart/noise.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,16 @@ def _is_noise(code: str, path: str, noise_globs: list[str]) -> bool:
return False


def classify_dirty(
def split_lines(
porcelain_lines: list[str], noise_globs: list[str]
) -> tuple[list[str], list[str]]:
"""Split porcelain lines into (real_paths, noise_paths).
"""Split porcelain lines into (real_lines, noise_lines), codes intact.

Each line is the raw ``git status --porcelain`` form: a two-char status
code, a space, then the path (which may itself contain spaces). Rename
entries (``R old -> new``) are reduced to their destination path.
entries (``R old -> new``) are classified by their destination path.
The full lines are returned so callers can still distinguish untracked
(``??``) from modified entries.
"""
real: list[str] = []
noise: list[str] = []
Expand All @@ -80,12 +82,27 @@ def classify_dirty(
if " -> " in path: # rename/copy: keep the destination
path = path.split(" -> ", 1)[1]
if _is_noise(code, path, noise_globs):
noise.append(path)
noise.append(line)
else:
real.append(path)
real.append(line)
return real, noise


def _line_path(line: str) -> str:
path = line[3:]
if " -> " in path:
path = path.split(" -> ", 1)[1]
return path


def classify_dirty(
porcelain_lines: list[str], noise_globs: list[str]
) -> tuple[list[str], list[str]]:
"""Split porcelain lines into (real_paths, noise_paths) — see split_lines."""
real_lines, noise_lines = split_lines(porcelain_lines, noise_globs)
return [_line_path(l) for l in real_lines], [_line_path(l) for l in noise_lines]


def build_sidecar(
*,
name: str,
Expand Down Expand Up @@ -116,19 +133,49 @@ def build_sidecar(
}


def run_batch(in_dir: Path, real_out: Path, config: str) -> int:
"""Classify one porcelain file per repo: ``<in_dir>/<name>`` holds that
repo's ``git status --porcelain`` output. Real-only lines are written to
``<real_out>/<name>`` and one ``<name>\\t<real_mod>\\t<real_untr>\\t<noise>``
line per repo goes to stdout (untracked = ``??`` code). No sidecar is
written — this is the display path (health_sync.sh), not the census.
"""
noise_globs = load_noise_globs(config)
real_out.mkdir(parents=True, exist_ok=True)
for f in sorted(in_dir.iterdir()):
if not f.is_file():
continue
real_lines, noise_lines = split_lines(f.read_text().splitlines(), noise_globs)
(real_out / f.name).write_text(
"\n".join(real_lines) + ("\n" if real_lines else "")
)
real_untr = sum(1 for l in real_lines if l[:2] == "??")
print(f"{f.name}\t{len(real_lines) - real_untr}\t{real_untr}\t{len(noise_lines)}")
return 0


def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="heart.noise")
ap.add_argument("--name", required=True)
ap.add_argument("--name")
ap.add_argument("--group", default="")
ap.add_argument("--branch", default="")
ap.add_argument("--ahead", type=int, default=0)
ap.add_argument("--behind", type=int, default=0)
ap.add_argument("--upstream", default="")
ap.add_argument("--ts", default="")
ap.add_argument("--out", required=True, help="sidecar JSON path to write")
ap.add_argument("--out", help="sidecar JSON path to write")
ap.add_argument("--config", default=str(DEFAULT_CONFIG))
ap.add_argument("--batch", help="dir of per-repo porcelain files (no sidecar mode)")
ap.add_argument("--real-out", help="dir to write per-repo real-only porcelain (with --batch)")
ns = ap.parse_args(argv)

if ns.batch:
if not ns.real_out:
ap.error("--batch requires --real-out")
return run_batch(Path(ns.batch), Path(ns.real_out), ns.config)
if not ns.name or not ns.out:
ap.error("--name and --out are required (unless using --batch)")

ts = ns.ts or datetime.datetime.now(datetime.timezone.utc).isoformat()
porcelain_lines = sys.stdin.read().splitlines()
noise_globs = load_noise_globs(ns.config)
Expand Down
7 changes: 6 additions & 1 deletion scripts/health.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,21 @@ health() {
cat <<'EOF'
health — local health/dev shell dispatcher (mirrors the Claude /health door).

Usage: health [sync|release|audit]
Usage: health [sync|release|audit] [--all]

health cross-repo git-sync dashboard (branch, behind/ahead, dirty)
health sync same as bare `health`
health --all sync dashboard with the full raw dirty listing (noise included)
health release last PyAutoBuild release-prep run dashboard
health audit structural repo-health audit (non-repo dirs, stashes, dead branches)

Release-run helpers: health-report / health-json / health-triage.
EOF
;;
-*)
# Leading-dash first arg (e.g. `health --all`) is a sync flag, not a
# subcommand — route to the default sync view with the flag intact.
_health_sync "$sub" "$@" ;;
*)
echo "health: unknown subcommand '$sub' (try: health help)" >&2
return 2
Expand Down
155 changes: 133 additions & 22 deletions scripts/health_sync.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,29 @@
# Usage (normally sourced via ~/.bashrc, run through the `health` dispatcher):
# source ~/Code/PyAutoLabs/PyAutoMind/scripts/health_sync.sh
# health # (or: health sync)
# health --all # full raw dirty-file listing (noise included)
#
# Override the repo root (e.g. for testing) via PYAUTO_STATUS_ROOT.
#
# Dirty counts are classified through Heart's real-vs-noise splitter
# (heart/noise.py + noise_globs in config/repos.yaml): MOD/UNTR count *real*
# source drift only, the NOISE column counts regenerated artifacts (*.fits,
# *data.json, *.png, …). If the classifier is unavailable (no PyAutoHeart
# checkout / python failure) the raw counts are shown with NOISE=? and the
# listing falls back to today's full form.
#
# Flag glyphs (FLAGS column):
# ↓ behind upstream
# ↑ ahead of upstream
# * dirty (modified or untracked files)
# * dirty (real modified or untracked files)
# ~ noise only (regenerated artifacts; no real drift)
# ! no upstream / fetch failed
# b current branch ≠ upstream branch (forgotten feature branch)
#
# After the main table, four optional sections may follow:
# - "Dirty files:" — per-repo `git status --porcelain` for any repo
# with mod or untr > 0.
# After the main table, optional sections may follow:
# - "Dirty files:" — per-repo listing of *real* dirty files, with
# noise collapsed to a one-line count per repo
# (`health --all` restores the raw listing).
# - "Follow-up commands:" — copy-pasteable git invocations grouped by
# category (pull / set-upstream / investigate).
# Suppressed entirely when nothing is actionable.
Expand All @@ -34,6 +44,10 @@
# ~/Code/PyAutoLabs/PyAutoBuild/test_results/*.json
# (committed by the autobuild release pipeline).
# Suppressed when no JSONs exist.
# - "Hygiene:" — nudge when the last /repo_cleanup audit stamp
# (~/.cache/pyauto/repo_cleanup_last_audit.json)
# is missing or older than PYAUTO_CLEANUP_NUDGE_DAYS
# (default 7). Suppressed when the stamp is fresh.
#
# Note: distinct from the Claude `/health status` command (the active-work
# registry dashboard). This shell command shows git *sync* state across repos;
Expand All @@ -49,6 +63,13 @@ _health_sync() {
return 1
fi

local show_all=false arg
for arg in "$@"; do
case "$arg" in
--all) show_all=true ;;
esac
done

# Discover repos. `.git` is a directory in normal checkouts and a file in
# linked worktrees, so accept both. mindepth/maxdepth 2 limits us to the
# immediate children of $root.
Expand Down Expand Up @@ -93,19 +114,16 @@ _health_sync() {
wait
) 2>/dev/null

# Header.
local fmt='%-32s %-30s %-36s %6s %5s %4s %4s %s\n'
printf "$fmt" REPO BRANCH UPSTREAM BEHIND AHEAD MOD UNTR FLAGS
printf "$fmt" "--------------------------------" \
"------------------------------" \
"------------------------------------" \
"------" "-----" "----" "----" "-----"

# Per-repo row. Porcelain is cached so the dirty-files listing below can
# reuse it without a second `git status` per repo. Action arrays collect
# actionable follow-ups for the "Follow-up commands:" section printed at
# the end.
declare -A repo_porcelain
# Pass 1 — gather per-repo state without printing. The table itself is
# printed in pass 2, after the noise classifier has run over every repo's
# porcelain in one batch (per-row python calls would blow the <10s budget).
# Porcelain is cached so the dirty-files listing below can reuse it without
# a second `git status` per repo. Action arrays collect actionable
# follow-ups for the "Follow-up commands:" section printed at the end.
local noise_tmp="$fetch_status_dir/noise"
mkdir -p "$noise_tmp/p" "$noise_tmp/real"
declare -A repo_porcelain repo_branch repo_upstream repo_behind repo_ahead \
repo_flags repo_mod_raw repo_untr_raw real_mod real_untr noise_n
local actions_pull=() actions_set_upstream=() actions_manual=()
local name branch upstream upstream_branch behind ahead mod untr flags counts porcelain branch_mismatch b_int a_int
for repo in "${repos[@]}"; do
Expand Down Expand Up @@ -149,6 +167,7 @@ _health_sync() {
else
untr="$(printf '%s\n' "$porcelain" | grep -c '^??' || true)"
mod="$(printf '%s\n' "$porcelain" | grep -cv '^??' || true)"
printf '%s\n' "$porcelain" > "$noise_tmp/p/$name"
fi

# Branch-mismatch detection. With no upstream, the heuristic is
Expand All @@ -163,10 +182,17 @@ _health_sync() {

[[ "$behind" =~ ^[0-9]+$ ]] && (( behind > 0 )) && flags+="↓"
[[ "$ahead" =~ ^[0-9]+$ ]] && (( ahead > 0 )) && flags+="↑"
(( mod + untr > 0 )) && flags+="*"
[[ "$branch_mismatch" == "true" ]] && flags+="b"

printf "$fmt" "$name" "$branch" "$upstream" "$behind" "$ahead" "$mod" "$untr" "$flags"
# Dirty flags (* / ~) depend on the noise classification and are added in
# pass 2; everything else about the row is fixed here.
repo_branch["$name"]="$branch"
repo_upstream["$name"]="$upstream"
repo_behind["$name"]="$behind"
repo_ahead["$name"]="$ahead"
repo_flags["$name"]="$flags"
repo_mod_raw["$name"]="$mod"
repo_untr_raw["$name"]="$untr"

# Categorise actionable follow-ups. Only the boring case (clean, behind,
# not ahead) becomes an auto-runnable command; everything else is
Expand All @@ -193,10 +219,64 @@ _health_sync() {
fi
done

# Noise classification — one batched heart.noise call over every dirty
# repo's porcelain. Real-only porcelain lands in $noise_tmp/real/<name>,
# counts on stdout as "<name>\t<real_mod>\t<real_untr>\t<noise>". Any
# failure (no PyAutoHeart checkout, no PyYAML, …) degrades to the raw
# behaviour: classify_ok=false, NOISE column shows "?". PYAUTO_HEART_HOME
# points at an alternative Heart checkout (e.g. a task worktree).
local heart_home="${PYAUTO_HEART_HOME:-$root/PyAutoHeart}"
local classify_ok=false
if [[ -f "$heart_home/heart/noise.py" ]]; then
if PYTHONPATH="$heart_home" python3 -m heart.noise \
--batch "$noise_tmp/p" --real-out "$noise_tmp/real" \
--config "$heart_home/config/repos.yaml" > "$noise_tmp/counts" 2>/dev/null; then
classify_ok=true
local cname crm cru cnn
while IFS=$'\t' read -r cname crm cru cnn; do
[[ -z "$cname" ]] && continue
real_mod["$cname"]="$crm"
real_untr["$cname"]="$cru"
noise_n["$cname"]="$cnn"
done < "$noise_tmp/counts"
fi
fi

# Pass 2 — print the table. MOD/UNTR are real-file counts when the
# classifier ran (raw counts otherwise); NOISE carries the regenerated-
# artifact count. `*` marks real drift, `~` noise-only dirt.
local fmt='%-32s %-30s %-36s %6s %5s %4s %4s %5s %s\n'
printf "$fmt" REPO BRANCH UPSTREAM BEHIND AHEAD MOD UNTR NOISE FLAGS
printf "$fmt" "--------------------------------" \
"------------------------------" \
"------------------------------------" \
"------" "-----" "----" "----" "-----" "-----"

local noise nreal
for repo in "${repos[@]}"; do
name="$(basename "$repo")"
flags="${repo_flags[$name]}"
if [[ "$classify_ok" == "true" ]]; then
mod="${real_mod[$name]:-0}"
untr="${real_untr[$name]:-0}"
noise="${noise_n[$name]:-0}"
(( mod + untr > 0 )) && flags+="*"
(( noise > 0 )) && flags+="~"
else
mod="${repo_mod_raw[$name]}"
untr="${repo_untr_raw[$name]}"
noise="?"
(( mod + untr > 0 )) && flags+="*"
fi
printf "$fmt" "$name" "${repo_branch[$name]}" "${repo_upstream[$name]}" \
"${repo_behind[$name]}" "${repo_ahead[$name]}" "$mod" "$untr" "$noise" "$flags"
done

# Per-repo dirty-file listing. Only repos with non-empty porcelain are
# shown — keeps the output empty when everything is clean. The `??` and
# ` M` etc. prefixes from porcelain are preserved so users can tell
# untracked from modified at a glance.
# untracked from modified at a glance. With the classifier available (and
# without --all) only *real* files are listed; noise collapses to a count.
local printed_header=false
for repo in "${repos[@]}"; do
name="$(basename "$repo")"
Expand All @@ -207,9 +287,24 @@ _health_sync() {
echo "Dirty files:"
printed_header=true
fi
echo " $name:"
printf '%s\n' "$porcelain" | sed 's/^/ /'
if [[ "$show_all" == "true" || "$classify_ok" != "true" ]]; then
echo " $name:"
printf '%s\n' "$porcelain" | sed 's/^/ /'
continue
fi
nreal=$(( ${real_mod[$name]:-0} + ${real_untr[$name]:-0} ))
noise="${noise_n[$name]:-0}"
if (( nreal == 0 )); then
echo " $name: $noise noise artifact(s) (regenerated) — \`health --all\` to list"
else
echo " $name:"
sed 's/^/ /' "$noise_tmp/real/$name" 2>/dev/null
(( noise > 0 )) && echo " (+$noise noise artifact(s) — \`health --all\` to list)"
fi
done
if [[ "$printed_header" == "true" && "$classify_ok" != "true" ]]; then
echo " (noise classification unavailable — raw listing shown)"
fi

# Follow-up commands. Suppressed entirely when nothing is actionable so
# the clean case stays quiet. The `git -C <abs-path>` form means each
Expand Down Expand Up @@ -309,4 +404,20 @@ print(f"{latest[:10]}|{num}|{len(projects)}|{total_p}|{total_f}|{total_s}")
"$njobs" "$nproj" "$pab_p" "$pab_f" "$pab_s"
fi
fi

# Hygiene nudge. /repo_cleanup writes this stamp after each audit; the
# mtime is the audit time. Missing or stale (> PYAUTO_CLEANUP_NUDGE_DAYS,
# default 7) prints a reminder — the cadence mechanism for the sweep.
local stamp="$HOME/.cache/pyauto/repo_cleanup_last_audit.json"
local nudge_days="${PYAUTO_CLEANUP_NUDGE_DAYS:-7}" age_days
if [[ ! -f "$stamp" ]]; then
echo ""
echo "Hygiene: no /repo_cleanup audit on record — consider running /repo_cleanup."
else
age_days=$(( ( $(date +%s) - $(stat -c %Y "$stamp" 2>/dev/null || echo 0) ) / 86400 ))
if (( age_days > nudge_days )); then
echo ""
echo "Hygiene: last /repo_cleanup audit ${age_days} days ago — consider running /repo_cleanup."
fi
fi
}
Loading
Loading