From 288e6b7b4021765fb5c055854c411d03415a101d Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 05:31:48 +0300 Subject: [PATCH 01/11] watchdog: test harness skeleton + local-only superpowers dir Co-Authored-By: Claude Fable 5 --- .gitignore | 2 ++ onboarding/watchdog/test/run_tests.sh | 28 ++++++++++++++++++++++ onboarding/watchdog/test/shims/curl | 2 ++ onboarding/watchdog/test/shims/notify-send | 2 ++ onboarding/watchdog/test/shims/pstree | 3 +++ onboarding/watchdog/test/shims/tmux | 19 +++++++++++++++ 6 files changed, 56 insertions(+) create mode 100755 onboarding/watchdog/test/run_tests.sh create mode 100755 onboarding/watchdog/test/shims/curl create mode 100755 onboarding/watchdog/test/shims/notify-send create mode 100755 onboarding/watchdog/test/shims/pstree create mode 100755 onboarding/watchdog/test/shims/tmux diff --git a/.gitignore b/.gitignore index 28818e0..29284af 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,5 @@ timer.dat # .vscode .vscode +# Superpowers working files (brainstorming specs, plans) — local-only, never tracked +docs/superpowers/ diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh new file mode 100755 index 0000000..189ae54 --- /dev/null +++ b/onboarding/watchdog/test/run_tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Hermetic unit tests for cc-watchdog. No real tmux/Slack needed (PATH shims). +set -uo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +PASS=0; FAIL=0 + +t() { # t + local out + if out=$("$2" 2>&1); then echo "PASS $1"; PASS=$((PASS + 1)) + else echo "FAIL $1"; echo "$out" | sed 's/^/ /'; FAIL=$((FAIL + 1)); fi +} + +setup() { + TMP=$(mktemp -d) + export CC_WATCHDOG_HOME="$TMP/wd" + export TMUX_SHIM_DIR="$TMP/tmux" + export SHIM_LOG_DIR="$TMP/shimlog" + mkdir -p "$TMUX_SHIM_DIR" "$SHIM_LOG_DIR" + export PATH="$HERE/shims:$PATH" + export SLACK_WEBHOOK_URL="http://example.invalid/hook" + unset WATCHDOG_DRY TMUX_PANE +} + +# --- tests appended by later tasks above this line --- + +echo "---" +echo "$PASS passed, $FAIL failed" +(( FAIL == 0 )) diff --git a/onboarding/watchdog/test/shims/curl b/onboarding/watchdog/test/shims/curl new file mode 100755 index 0000000..aeae7fa --- /dev/null +++ b/onboarding/watchdog/test/shims/curl @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +echo "$*" >> "$SHIM_LOG_DIR/curl.log" diff --git a/onboarding/watchdog/test/shims/notify-send b/onboarding/watchdog/test/shims/notify-send new file mode 100755 index 0000000..f98dc68 --- /dev/null +++ b/onboarding/watchdog/test/shims/notify-send @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +echo "$*" >> "$SHIM_LOG_DIR/notify-send.log" diff --git a/onboarding/watchdog/test/shims/pstree b/onboarding/watchdog/test/shims/pstree new file mode 100755 index 0000000..4db8076 --- /dev/null +++ b/onboarding/watchdog/test/shims/pstree @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# pstree -p → canned output from $TMUX_SHIM_DIR/pstree- +cat "$TMUX_SHIM_DIR/pstree-$2" 2>/dev/null diff --git a/onboarding/watchdog/test/shims/tmux b/onboarding/watchdog/test/shims/tmux new file mode 100755 index 0000000..be8daed --- /dev/null +++ b/onboarding/watchdog/test/shims/tmux @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Test shim for tmux, driven by files under $TMUX_SHIM_DIR. +cmd=$1; shift +case "$cmd" in + list-panes) + fmt="" + while (( $# > 0 )); do [[ $1 == -F ]] && fmt=$2; shift; done + if [[ "$fmt" == *pane_pid* ]]; then cat "$TMUX_SHIM_DIR/panes" 2>/dev/null + else awk '{print $1}' "$TMUX_SHIM_DIR/panes" 2>/dev/null; fi + ;; + capture-pane) + pane="" + while (( $# > 0 )); do [[ $1 == -t ]] && pane=$2; shift; done + cat "$TMUX_SHIM_DIR/content-${pane}" 2>/dev/null + ;; + display-message) echo "fake-session" ;; + send-keys) echo "$*" >> "$TMUX_SHIM_DIR/send-keys.log" ;; + *) exit 0 ;; +esac From 0a2f92ffbf1a7922b7601a1142fd5453dde2e6ac Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 05:37:44 +0300 Subject: [PATCH 02/11] watchdog: cc-deadman declaration helper Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-deadman | 88 +++++++++++++++++++++++++++ onboarding/watchdog/test/run_tests.sh | 44 ++++++++++++++ 2 files changed, 132 insertions(+) create mode 100755 onboarding/watchdog/cc-deadman diff --git a/onboarding/watchdog/cc-deadman b/onboarding/watchdog/cc-deadman new file mode 100755 index 0000000..c2e7502 --- /dev/null +++ b/onboarding/watchdog/cc-deadman @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# cc-deadman — declare when this Claude Code session expects to be alive again. +# Enforced by cc-stall-watchdog.sh (see README.md in this directory). +set -euo pipefail + +WD_HOME="${CC_WATCHDOG_HOME:-$HOME/.cc-watchdog}" +STATE_DIR="$WD_HOME/state" +mkdir -p "$STATE_DIR" + +usage() { + cat >&2 <<'EOF' +usage: cc-deadman set "" declare a deadline + cc-deadman extend push this session's deadline out + cc-deadman clear remove this session's deadline + cc-deadman list show all live deadlines +EOF + exit 2 +} + +resolve_pane() { + if [[ -n "${TMUX_PANE:-}" ]]; then printf '%s' "$TMUX_PANE"; return 0; fi + # Tool shells often lack $TMUX_PANE: find the pane whose top-level shell + # (pane_pid) is an ancestor of this process. + local pid=$$ chain=" " + while (( pid > 1 )); do + chain+="$pid " + pid=$(awk '{print $4}' "/proc/$pid/stat" 2>/dev/null) || break + done + local pane ppid + while read -r pane ppid; do + [[ "$chain" == *" $ppid "* ]] && { printf '%s' "$pane"; return 0; } + done < <(tmux list-panes -a -F '#{pane_id} #{pane_pid}' 2>/dev/null) + echo "cc-deadman: cannot resolve tmux pane (not inside tmux?)" >&2 + return 1 +} + +resolve_transcript() { + # Claude Code stores transcripts under a cwd-munged directory name. + local munged + munged=$(pwd | sed 's#[/.]#-#g') + ls -t "$HOME/.claude/projects/$munged"/*.jsonl 2>/dev/null | head -1 || true +} + +cmd=${1:-}; [[ -n "$cmd" ]] && shift || usage +case "$cmd" in + set) + (( $# == 2 )) || usage + mins=$1 reason=$2 + pane=$(resolve_pane) + f="$STATE_DIR/$pane.deadline" + now=$(date +%s) + { + echo "pane=$pane" + echo "set=$now" + echo "deadline=$(( now + mins * 60 ))" + echo "reason=$reason" + echo "transcript=$(resolve_transcript)" + } > "$f" + echo "cc-deadman: armed — $mins min for: $reason" + ;; + extend) + (( $# == 1 )) || usage + pane=$(resolve_pane) + f="$STATE_DIR/$pane.deadline" + [[ -f "$f" ]] || { echo "cc-deadman: no deadline set for $pane" >&2; exit 1; } + old=$(sed -n 's/^deadline=//p' "$f") + grep -Ev '^(deadline|notified|recovered|failed)=' "$f" > "$f.tmp" || true + echo "deadline=$(( old + $1 * 60 ))" >> "$f.tmp" + mv "$f.tmp" "$f" + echo "cc-deadman: extended by $1 min" + ;; + clear) + pane=$(resolve_pane) + rm -f "$STATE_DIR/$pane.deadline" + echo "cc-deadman: cleared" + ;; + list) + shopt -s nullglob + now=$(date +%s) + for f in "$STATE_DIR"/*.deadline; do + d=$(sed -n 's/^deadline=//p' "$f") + printf '%s due in %d min %s\n' \ + "$(sed -n 's/^pane=//p' "$f")" "$(( (d - now) / 60 ))" \ + "$(sed -n 's/^reason=//p' "$f")" + done + ;; + *) usage ;; +esac diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 189ae54..22b60d7 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -21,6 +21,50 @@ setup() { unset WATCHDOG_DRY TMUX_PANE } +test_deadman_set_writes_fields() { + setup; cd "$TMP" + TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "suite run" >/dev/null || return 1 + local f="$CC_WATCHDOG_HOME/state/%7.deadline" + [[ -f "$f" ]] || { echo "no deadline file"; return 1; } + grep -qx 'pane=%7' "$f" || { echo "bad pane"; return 1; } + grep -qx 'reason=suite run' "$f" || { echo "bad reason"; return 1; } + local s d + s=$(sed -n 's/^set=//p' "$f"); d=$(sed -n 's/^deadline=//p' "$f") + (( d - s == 1800 )) || { echo "bad delta: $((d - s))"; return 1; } +} + +test_deadman_clear_removes() { + setup; cd "$TMP" + TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "x" >/dev/null || return 1 + TMUX_PANE=%7 "$HERE/../cc-deadman" clear >/dev/null || return 1 + [[ ! -f "$CC_WATCHDOG_HOME/state/%7.deadline" ]] +} + +test_deadman_extend_pushes_and_resets_tiers() { + setup; cd "$TMP" + TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "x" >/dev/null + local f="$CC_WATCHDOG_HOME/state/%7.deadline" + echo "notified=123" >> "$f" + local before after + before=$(sed -n 's/^deadline=//p' "$f") + TMUX_PANE=%7 "$HERE/../cc-deadman" extend 10 >/dev/null + after=$(sed -n 's/^deadline=//p' "$f") + (( after - before == 600 )) || { echo "bad extend"; return 1; } + ! grep -q '^notified=' "$f" || { echo "notified not reset"; return 1; } +} + +test_deadman_pane_from_process_tree() { + setup; cd "$TMP" + echo "%9 $$" > "$TMUX_SHIM_DIR/panes" # this test shell is cc-deadman's ancestor + "$HERE/../cc-deadman" set 5 "tree" >/dev/null || return 1 + [[ -f "$CC_WATCHDOG_HOME/state/%9.deadline" ]] +} + +t deadman_set_writes_fields test_deadman_set_writes_fields +t deadman_clear_removes test_deadman_clear_removes +t deadman_extend_pushes_and_resets_tiers test_deadman_extend_pushes_and_resets_tiers +t deadman_pane_from_process_tree test_deadman_pane_from_process_tree + # --- tests appended by later tasks above this line --- echo "---" From 6d7eb16550266f186905a722c86a0d1872c7c5aa Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 05:49:19 +0300 Subject: [PATCH 03/11] =?UTF-8?q?watchdog:=20poller=20scaffolding=20?= =?UTF-8?q?=E2=80=94=20config,=20notify,=20pane=20helpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-stall-watchdog.sh | 73 ++++++++++++++++++++++++ onboarding/watchdog/test/run_tests.sh | 44 ++++++++++++++ 2 files changed, 117 insertions(+) create mode 100755 onboarding/watchdog/cc-stall-watchdog.sh diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh new file mode 100755 index 0000000..863c3d4 --- /dev/null +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# cc-stall-watchdog.sh — enforce cc-deadman declarations; catch silent die-outs. +# Driven by a systemd user timer every 5 min. NEVER by a Claude Code hook: +# hook-driven watchers only wake when a turn ends, which is exactly the blind +# spot that produced 36 h stalls. See README.md. +set -uo pipefail + +WD_HOME="${CC_WATCHDOG_HOME:-$HOME/.cc-watchdog}" +STATE_DIR="$WD_HOME/state" +ARCHIVE_DIR="$STATE_DIR/archive" +LOG_FILE="$WD_HOME/log" +CONFIG="$WD_HOME/config" + +# Defaults; override in ~/.cc-watchdog/config (sourced below). +GRACE_MIN=60 +BACKSTOP_HOURS=6 +BACKSTOP_RENOTIFY_HOURS=12 +RECOVERY_BACKOFF_HOURS=24 +BUSY_RE='esc to interrupt' +# PI-machine Phoenix state; absent (harmless) on member machines. +AGENTS_STATE="${CC_WATCHDOG_AGENTS_STATE:-$HOME/agents/state}" + +mkdir -p "$STATE_DIR" "$ARCHIVE_DIR" +# shellcheck source=/dev/null +[[ -f "$CONFIG" ]] && . "$CONFIG" + +log() { printf '%s %s\n' "$(date '+%F %T')" "$*" >> "$LOG_FILE"; } + +notify() { # $1 = message + local msg=${1//\"/\'} + log "NOTIFY: $msg" + [[ -n "${WATCHDOG_DRY:-}" ]] && return 0 + if [[ -n "${SLACK_WEBHOOK_URL:-}" ]]; then + curl -sf -X POST -H 'Content-type: application/json' \ + --data "{\"text\": \"$msg\"}" "$SLACK_WEBHOOK_URL" >/dev/null 2>&1 \ + || log "ERROR: Slack post failed" + else + log "WARNING: SLACK_WEBHOOK_URL unset — falling back to notify-send" + notify-send "cc-watchdog" "$msg" 2>/dev/null || true + fi +} + +pane_alive() { tmux list-panes -a -F '#{pane_id}' 2>/dev/null | grep -qx "$1"; } +pane_busy() { tmux capture-pane -p -t "$1" 2>/dev/null | grep -q "$BUSY_RE"; } +pane_idle() { pane_alive "$1" && ! pane_busy "$1"; } +session_name() { tmux display-message -p -t "$1" '#{session_name}' 2>/dev/null || echo '?'; } + +get_field() { sed -n "s/^$2=//p" "$1" | head -1; } +set_field() { + grep -v "^$2=" "$1" > "$1.tmp" || true + echo "$2=$3" >> "$1.tmp" + mv "$1.tmp" "$1" +} + +phoenix_owns() { # $1 = pane — true iff Phoenix is limit-waiting on this pane's session + local f sid + for f in "$AGENTS_STATE"/*.limit-wait; do + [[ -e "$f" ]] || return 1 + sid=$(basename "$f" .limit-wait) + [[ "$(cat "$AGENTS_STATE/$sid.tmux-pane" 2>/dev/null)" == "$1" ]] && return 0 + done + return 1 +} + +main() { + shopt -s nullglob + local f + for f in "$STATE_DIR"/*.deadline; do process_deadline "$f"; done + backstop_scan +} + +# Sourceable for tests; runs only when executed. +[[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@" diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 22b60d7..3f40348 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -65,6 +65,50 @@ t deadman_clear_removes test_deadman_clear_removes t deadman_extend_pushes_and_resets_tiers test_deadman_extend_pushes_and_resets_tiers t deadman_pane_from_process_tree test_deadman_pane_from_process_tree +src_watchdog() { . "$HERE/../cc-stall-watchdog.sh"; } + +test_wd_notify_posts_to_slack() { + setup; src_watchdog + notify "hello world" + grep -q "hello world" "$SHIM_LOG_DIR/curl.log" || { echo "no slack post"; return 1; } +} + +test_wd_notify_dry_is_log_only() { + setup; export WATCHDOG_DRY=1; src_watchdog + notify "quiet" + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "curl called in dry mode"; return 1; } + grep -q "NOTIFY: quiet" "$CC_WATCHDOG_HOME/log" || { echo "not logged"; return 1; } +} + +test_wd_notify_falls_back_without_webhook() { + setup; unset SLACK_WEBHOOK_URL; src_watchdog + notify "fallback" + grep -q "fallback" "$SHIM_LOG_DIR/notify-send.log" || { echo "no notify-send"; return 1; } +} + +test_wd_pane_busy_matches_footer() { + setup; src_watchdog + echo "%1 100" > "$TMUX_SHIM_DIR/panes" + printf 'stuff\n... esc to interrupt\n' > "$TMUX_SHIM_DIR/content-%1" + pane_busy %1 || { echo "should be busy"; return 1; } + printf '> \n' > "$TMUX_SHIM_DIR/content-%1" + pane_idle %1 || { echo "should be idle"; return 1; } +} + +test_wd_config_overrides_defaults() { + setup + mkdir -p "$CC_WATCHDOG_HOME" + echo "GRACE_MIN=7" > "$CC_WATCHDOG_HOME/config" + src_watchdog + (( GRACE_MIN == 7 )) || { echo "GRACE_MIN=$GRACE_MIN"; return 1; } +} + +t wd_notify_posts_to_slack test_wd_notify_posts_to_slack +t wd_notify_dry_is_log_only test_wd_notify_dry_is_log_only +t wd_notify_falls_back_without_webhook test_wd_notify_falls_back_without_webhook +t wd_pane_busy_matches_footer test_wd_pane_busy_matches_footer +t wd_config_overrides_defaults test_wd_config_overrides_defaults + # --- tests appended by later tasks above this line --- echo "---" From 46edf2c5cd6d194f3451859e219ec5e69183468e Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 05:56:38 +0300 Subject: [PATCH 04/11] =?UTF-8?q?watchdog:=20deadline=20ladder=20=E2=80=94?= =?UTF-8?q?=20self-clear,=20notify,=20recover,=20backoff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-deadman | 2 +- onboarding/watchdog/cc-stall-watchdog.sh | 70 +++++++++++++++ onboarding/watchdog/test/run_tests.sh | 105 +++++++++++++++++++++++ 3 files changed, 176 insertions(+), 1 deletion(-) diff --git a/onboarding/watchdog/cc-deadman b/onboarding/watchdog/cc-deadman index c2e7502..522833b 100755 --- a/onboarding/watchdog/cc-deadman +++ b/onboarding/watchdog/cc-deadman @@ -64,7 +64,7 @@ case "$cmd" in f="$STATE_DIR/$pane.deadline" [[ -f "$f" ]] || { echo "cc-deadman: no deadline set for $pane" >&2; exit 1; } old=$(sed -n 's/^deadline=//p' "$f") - grep -Ev '^(deadline|notified|recovered|failed)=' "$f" > "$f.tmp" || true + grep -Ev '^(deadline|notified|recovered|failed|recovery_kind)=' "$f" > "$f.tmp" || true echo "deadline=$(( old + $1 * 60 ))" >> "$f.tmp" mv "$f.tmp" "$f" echo "cc-deadman: extended by $1 min" diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh index 863c3d4..f437b6c 100755 --- a/onboarding/watchdog/cc-stall-watchdog.sh +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -62,6 +62,76 @@ phoenix_owns() { # $1 = pane — true iff Phoenix is limit-waiting on this pane' return 1 } +RECOVERY_PROMPT_TMPL='Dead-man deadline expired for: %s. Your awaited work likely died silently. Check honest status (process alive? output complete?) and continue or re-run.' + +inject_recovery() { # $1 pane, $2 reason + log "RECOVER: injecting into $1 (reason: $2)" + [[ -n "${WATCHDOG_DRY:-}" ]] && return 0 + if pane_busy "$1"; then + tmux send-keys -t "$1" Escape + sleep 5 + fi + # shellcheck disable=SC2059 + tmux send-keys -t "$1" -l "$(printf "$RECOVERY_PROMPT_TMPL" "$2")" + sleep 1 + tmux send-keys -t "$1" Enter +} + +process_deadline() { # $1 = deadline file + local f=$1 pane deadline setts reason transcript notified recovered now + pane=$(get_field "$f" pane); deadline=$(get_field "$f" deadline) + setts=$(get_field "$f" set); reason=$(get_field "$f" reason) + transcript=$(get_field "$f" transcript) + notified=$(get_field "$f" notified); recovered=$(get_field "$f" recovered) + now=$(date +%s) + + if ! pane_alive "$pane"; then + notify "cc-watchdog: pane $pane VANISHED with unmet deadline: $reason" + mv "$f" "$ARCHIVE_DIR/"; return + fi + + if phoenix_owns "$pane"; then log "DEFER: Phoenix owns $pane"; return; fi + + # Self-clear: the session did something after declaring, and is idle again. + if [[ -n "$transcript" && -f "$transcript" ]] \ + && (( $(stat -c %Y "$transcript") > setts )) && pane_idle "$pane"; then + log "SELF-CLEAR: $pane progressed and is idle ($reason)" + mv "$f" "$ARCHIVE_DIR/"; return + fi + + (( now < deadline )) && return + + if [[ -z "$notified" ]]; then + notify "cc-watchdog: [$(session_name "$pane")] deadline MISSED ($(( (now - deadline) / 60 )) min overdue): $reason" + set_field "$f" notified "$now" + return + fi + + if [[ -z "$recovered" ]]; then + (( now >= deadline + GRACE_MIN * 60 )) || return + local lr="$STATE_DIR/pane-$pane.last-recovery" + if [[ -f "$lr" ]] && (( now - $(cat "$lr") < RECOVERY_BACKOFF_HOURS * 3600 )); then + notify "cc-watchdog: [$(session_name "$pane")] stalled AGAIN within ${RECOVERY_BACKOFF_HOURS}h of a recovery — notify-only backoff: $reason" + set_field "$f" recovered "$now" + set_field "$f" recovery_kind backoff + return + fi + inject_recovery "$pane" "$reason" + set_field "$f" recovered "$now" + set_field "$f" recovery_kind inject + echo "$now" > "$lr" + return + fi + + # "RECOVERY FAILED ... after injection" is only truthful when an injection + # actually happened; backoff-mode deadlines already got their final notice. + if [[ "$(get_field "$f" recovery_kind)" == inject ]] \ + && [[ -z "$(get_field "$f" failed)" ]] && (( now >= recovered + GRACE_MIN * 60 )); then + notify "cc-watchdog: [$(session_name "$pane")] RECOVERY FAILED (no progress ${GRACE_MIN} min after injection): $reason" + set_field "$f" failed "$now" + fi +} + main() { shopt -s nullglob local f diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 3f40348..5f17637 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -109,6 +109,111 @@ t wd_notify_falls_back_without_webhook test_wd_notify_falls_back_without_webhook t wd_pane_busy_matches_footer test_wd_pane_busy_matches_footer t wd_config_overrides_defaults test_wd_config_overrides_defaults +mk_deadline() { # — echoes file path + local now f + now=$(date +%s) + f="$CC_WATCHDOG_HOME/state/$1.deadline" + mkdir -p "$CC_WATCHDOG_HOME/state" + { + echo "pane=$1" + echo "set=$(( now + $3 ))" + echo "deadline=$(( now + $2 ))" + echo "reason=$4" + echo "transcript=$TMP/transcript.jsonl" + } > "$f" + echo "$f" +} + +live_busy_pane() { # + echo "$1 100" > "$TMUX_SHIM_DIR/panes" + printf 'working... esc to interrupt\n' > "$TMUX_SHIM_DIR/content-$1" +} + +test_wd_not_due_is_noop() { + setup; src_watchdog; live_busy_pane %1 + local f; f=$(mk_deadline %1 600 -60 "later") + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + [[ -f "$f" ]] && ! grep -q '^notified=' "$f" +} + +test_wd_tier1_notifies_once() { + setup; src_watchdog; live_busy_pane %1 + local f; f=$(mk_deadline %1 -120 -1800 "suite") + process_deadline "$f" + grep -q 'deadline MISSED' "$SHIM_LOG_DIR/curl.log" || { echo "no tier1"; return 1; } + process_deadline "$f" + (( $(grep -c 'deadline MISSED' "$SHIM_LOG_DIR/curl.log") == 1 )) \ + || { echo "notified twice"; return 1; } +} + +test_wd_tier2_injects_after_grace() { + setup; src_watchdog; live_busy_pane %1 + local f; f=$(mk_deadline %1 $(( -GRACE_MIN * 60 - 60 )) -7200 "suite") + set_field "$f" notified "$(( $(date +%s) - GRACE_MIN * 60 ))" + process_deadline "$f" + grep -q 'Escape' "$TMUX_SHIM_DIR/send-keys.log" || { echo "no Escape"; return 1; } + grep -q 'Dead-man deadline expired' "$TMUX_SHIM_DIR/send-keys.log" \ + || { echo "no recovery prompt"; return 1; } + grep -q '^recovered=' "$f" || { echo "no recovered stamp"; return 1; } + [[ -f "$CC_WATCHDOG_HOME/state/pane-%1.last-recovery" ]] || { echo "no last-recovery"; return 1; } +} + +test_wd_backoff_after_recent_recovery() { + setup; src_watchdog; live_busy_pane %1 + date +%s > "$CC_WATCHDOG_HOME/state/pane-%1.last-recovery" + local f; f=$(mk_deadline %1 $(( -GRACE_MIN * 60 - 60 )) -7200 "again") + set_field "$f" notified "$(( $(date +%s) - GRACE_MIN * 60 ))" + process_deadline "$f" + [[ ! -f "$TMUX_SHIM_DIR/send-keys.log" ]] || { echo "injected during backoff"; return 1; } + grep -q 'notify-only backoff' "$SHIM_LOG_DIR/curl.log" || { echo "no backoff notice"; return 1; } + set_field "$f" recovered "$(( $(date +%s) - GRACE_MIN * 60 - 60 ))" + process_deadline "$f" || { echo "process_deadline failed on 2nd pass"; return 1; } + ! grep -q 'RECOVERY FAILED' "$SHIM_LOG_DIR/curl.log" || { echo "false RECOVERY FAILED in backoff mode"; return 1; } +} + +test_wd_self_clear_on_progress_and_idle() { + setup; src_watchdog + echo "%1 100" > "$TMUX_SHIM_DIR/panes" + printf '> \n' > "$TMUX_SHIM_DIR/content-%1" # idle + local f; f=$(mk_deadline %1 -120 -1800 "done actually") + touch "$TMP/transcript.jsonl" # mtime now > set time + process_deadline "$f" + [[ ! -f "$f" ]] || { echo "not cleared"; return 1; } + [[ -f "$CC_WATCHDOG_HOME/state/archive/%1.deadline" ]] || { echo "not archived"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "notified on self-clear"; return 1; } +} + +test_wd_vanished_pane_notifies_and_archives() { + setup; src_watchdog + : > "$TMUX_SHIM_DIR/panes" # no panes at all + local f; f=$(mk_deadline %1 -120 -1800 "gone") + process_deadline "$f" + grep -q 'VANISHED' "$SHIM_LOG_DIR/curl.log" || { echo "no vanish notice"; return 1; } + [[ ! -f "$f" ]] || { echo "not archived"; return 1; } +} + +test_wd_phoenix_defers() { + setup; src_watchdog; live_busy_pane %1 + export CC_WATCHDOG_AGENTS_STATE="$TMP/agents" + mkdir -p "$TMP/agents" + touch "$TMP/agents/abc.limit-wait" + echo "%1" > "$TMP/agents/abc.tmux-pane" + src_watchdog # re-source to pick up override + local f; f=$(mk_deadline %1 -120 -1800 "limit") + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "notified despite Phoenix"; return 1; } + [[ -f "$f" ]] || { echo "file touched"; return 1; } + grep -q 'DEFER: Phoenix owns' "$CC_WATCHDOG_HOME/log" || { echo "no DEFER log"; return 1; } +} + +t wd_not_due_is_noop test_wd_not_due_is_noop +t wd_tier1_notifies_once test_wd_tier1_notifies_once +t wd_tier2_injects_after_grace test_wd_tier2_injects_after_grace +t wd_backoff_after_recent_recovery test_wd_backoff_after_recent_recovery +t wd_self_clear_on_progress_and_idle test_wd_self_clear_on_progress_and_idle +t wd_vanished_pane_notifies_and_archives test_wd_vanished_pane_notifies_and_archives +t wd_phoenix_defers test_wd_phoenix_defers + # --- tests appended by later tasks above this line --- echo "---" From d7459a6b422bab4ffd81c9cbf657d8f3fe10e0ec Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 06:19:13 +0300 Subject: [PATCH 05/11] =?UTF-8?q?watchdog:=20multiplexer=20seam=20?= =?UTF-8?q?=E2=80=94=20Herdr=20backend=20for=20the=20deadline=20ladder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-deadman | 4 ++ onboarding/watchdog/cc-stall-watchdog.sh | 58 +++++++++++++-- onboarding/watchdog/test/run_tests.sh | 92 +++++++++++++++++++++++- onboarding/watchdog/test/shims/herdr | 9 +++ 4 files changed, 155 insertions(+), 8 deletions(-) create mode 100755 onboarding/watchdog/test/shims/herdr diff --git a/onboarding/watchdog/cc-deadman b/onboarding/watchdog/cc-deadman index 522833b..17a7fbe 100755 --- a/onboarding/watchdog/cc-deadman +++ b/onboarding/watchdog/cc-deadman @@ -18,6 +18,10 @@ EOF } resolve_pane() { + # Herdr injects its pane id into every managed pane; prefer it when present. + if [[ "${HERDR_ENV:-}" == 1 && -n "${HERDR_PANE_ID:-}" ]]; then + printf 'herdr:%s' "$HERDR_PANE_ID"; return 0 + fi if [[ -n "${TMUX_PANE:-}" ]]; then printf '%s' "$TMUX_PANE"; return 0; fi # Tool shells often lack $TMUX_PANE: find the pane whose top-level shell # (pane_pid) is an ancestor of this process. diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh index f437b6c..03b781e 100755 --- a/onboarding/watchdog/cc-stall-watchdog.sh +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -40,10 +40,46 @@ notify() { # $1 = message fi } -pane_alive() { tmux list-panes -a -F '#{pane_id}' 2>/dev/null | grep -qx "$1"; } -pane_busy() { tmux capture-pane -p -t "$1" 2>/dev/null | grep -q "$BUSY_RE"; } -pane_idle() { pane_alive "$1" && ! pane_busy "$1"; } -session_name() { tmux display-message -p -t "$1" '#{session_name}' 2>/dev/null || echo '?'; } +# --- multiplexer seam: pane keys "herdr:" route to the herdr CLI ------- +is_herdr() { [[ "$1" == herdr:* ]]; } +hid() { printf '%s' "${1#herdr:}"; } + +herdr_status() { # $1 = full pane key → agent_status, empty if unreachable + herdr pane get "$(hid "$1")" 2>/dev/null \ + | sed -n 's/.*"agent_status"[[:space:]]*:[[:space:]]*"\([a-z]*\)".*/\1/p' | head -1 +} + +pane_alive() { + if is_herdr "$1"; then + local out + # unreachable herdr => assume alive: never false-VANISH a session we can't see + out=$(herdr pane list 2>/dev/null) || return 0 + grep -qF "\"$(hid "$1")\"" <<<"$out" + else + tmux list-panes -a -F '#{pane_id}' 2>/dev/null | grep -qx "$1" + fi +} + +pane_busy() { + if is_herdr "$1"; then [[ "$(herdr_status "$1")" == working ]] + else tmux capture-pane -p -t "$1" 2>/dev/null | grep -q "$BUSY_RE"; fi +} + +pane_idle() { + if is_herdr "$1"; then + local s; s=$(herdr_status "$1") + # 'blocked' (agent waiting for input) is deliberately NOT idle: a blocked + # agent must not self-clear its deadline — tier 1 should reach the user. + [[ "$s" == idle || "$s" == done ]] + else + pane_alive "$1" && ! pane_busy "$1" + fi +} + +session_name() { + if is_herdr "$1"; then printf '%s' "$1" + else tmux display-message -p -t "$1" '#{session_name}' 2>/dev/null || echo '?'; fi +} get_field() { sed -n "s/^$2=//p" "$1" | head -1; } set_field() { @@ -52,12 +88,16 @@ set_field() { mv "$1.tmp" "$1" } -phoenix_owns() { # $1 = pane — true iff Phoenix is limit-waiting on this pane's session +phoenix_owns() { # $1 = pane key — Phoenix records herdr panes in .herdr-pane local f sid for f in "$AGENTS_STATE"/*.limit-wait; do [[ -e "$f" ]] || return 1 sid=$(basename "$f" .limit-wait) - [[ "$(cat "$AGENTS_STATE/$sid.tmux-pane" 2>/dev/null)" == "$1" ]] && return 0 + if is_herdr "$1"; then + [[ "$(cat "$AGENTS_STATE/$sid.herdr-pane" 2>/dev/null)" == "$(hid "$1")" ]] && return 0 + else + [[ "$(cat "$AGENTS_STATE/$sid.tmux-pane" 2>/dev/null)" == "$1" ]] && return 0 + fi done return 1 } @@ -67,6 +107,12 @@ RECOVERY_PROMPT_TMPL='Dead-man deadline expired for: %s. Your awaited work likel inject_recovery() { # $1 pane, $2 reason log "RECOVER: injecting into $1 (reason: $2)" [[ -n "${WATCHDOG_DRY:-}" ]] && return 0 + if is_herdr "$1"; then + # shellcheck disable=SC2059 + herdr pane run "$(hid "$1")" "$(printf "$RECOVERY_PROMPT_TMPL" "$2")" >/dev/null 2>&1 \ + || log "ERROR: herdr pane run failed for $1 (external control unavailable?)" + return 0 + fi if pane_busy "$1"; then tmux send-keys -t "$1" Escape sleep 5 diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 5f17637..7ed3f5b 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -15,10 +15,12 @@ setup() { export CC_WATCHDOG_HOME="$TMP/wd" export TMUX_SHIM_DIR="$TMP/tmux" export SHIM_LOG_DIR="$TMP/shimlog" - mkdir -p "$TMUX_SHIM_DIR" "$SHIM_LOG_DIR" + export HERDR_SHIM_DIR="$TMP/herdr" + mkdir -p "$TMUX_SHIM_DIR" "$SHIM_LOG_DIR" "$HERDR_SHIM_DIR" export PATH="$HERE/shims:$PATH" export SLACK_WEBHOOK_URL="http://example.invalid/hook" - unset WATCHDOG_DRY TMUX_PANE + export CC_WATCHDOG_AGENTS_STATE="$TMP/agents" + unset WATCHDOG_DRY TMUX_PANE HERDR_ENV HERDR_PANE_ID } test_deadman_set_writes_fields() { @@ -214,6 +216,92 @@ t wd_self_clear_on_progress_and_idle test_wd_self_clear_on_progress_and_idle t wd_vanished_pane_notifies_and_archives test_wd_vanished_pane_notifies_and_archives t wd_phoenix_defers test_wd_phoenix_defers +test_deadman_herdr_pane_key() { + setup; cd "$TMP" + HERDR_ENV=1 HERDR_PANE_ID=w1:p3 "$HERE/../cc-deadman" set 30 "melt sim" >/dev/null || return 1 + local f="$CC_WATCHDOG_HOME/state/herdr:w1:p3.deadline" + [[ -f "$f" ]] || { echo "no herdr deadline file"; return 1; } + grep -qx 'pane=herdr:w1:p3' "$f" || { echo "bad pane key"; return 1; } +} + +test_wd_herdr_vanished_detected() { + setup; src_watchdog + herdr_pane w9:p9 idle # herdr reachable, but lists a DIFFERENT pane + local f; f=$(mk_deadline herdr:w1:p3 -120 -1800 "gone") + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + grep -q 'VANISHED' "$SHIM_LOG_DIR/curl.log" || { echo "no vanish notice"; return 1; } + [[ ! -f "$f" ]] || { echo "not archived"; return 1; } +} + +herdr_pane() { # — register a live herdr pane in the shim + printf '{"panes": [{"pane_id": "%s"}]}\n' "$1" > "$HERDR_SHIM_DIR/panes.json" + printf '{"pane_id": "%s", "agent_status": "%s"}\n' "$1" "$2" > "$HERDR_SHIM_DIR/status-$1.json" +} + +test_wd_herdr_status_mapping() { + setup; src_watchdog + herdr_pane w1:p3 working + pane_busy herdr:w1:p3 || { echo "working should be busy"; return 1; } + ! pane_idle herdr:w1:p3 || { echo "working must not be idle"; return 1; } + herdr_pane w1:p3 blocked + ! pane_busy herdr:w1:p3 || { echo "blocked is not busy"; return 1; } + ! pane_idle herdr:w1:p3 || { echo "blocked must NOT be idle (no self-clear)"; return 1; } + herdr_pane w1:p3 idle + pane_idle herdr:w1:p3 || { echo "idle should be idle"; return 1; } +} + +test_wd_herdr_tier2_uses_pane_run() { + setup; src_watchdog + herdr_pane w1:p3 working + local f; f=$(mk_deadline herdr:w1:p3 $(( -GRACE_MIN * 60 - 60 )) -7200 "herdr suite") + set_field "$f" notified "$(( $(date +%s) - GRACE_MIN * 60 ))" + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + grep -q 'Dead-man deadline expired' "$HERDR_SHIM_DIR/pane-run.log" || { echo "no pane run"; return 1; } + [[ ! -f "$TMUX_SHIM_DIR/send-keys.log" ]] || { echo "tmux injection used for herdr pane"; return 1; } + grep -qx 'recovery_kind=inject' "$f" || { echo "no inject stamp"; return 1; } +} + +test_wd_herdr_unreachable_degrades_to_notify() { + setup; src_watchdog + # no panes.json / status files: every herdr call fails (external control unavailable) + local f; f=$(mk_deadline herdr:w1:p3 -120 -1800 "unreachable") + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + ! grep -q 'VANISHED' "$SHIM_LOG_DIR/curl.log" || { echo "false VANISH"; return 1; } + grep -q 'deadline MISSED' "$SHIM_LOG_DIR/curl.log" || { echo "tier1 did not fire"; return 1; } + [[ -f "$f" ]] || { echo "deadline archived while unreachable"; return 1; } +} + +test_wd_herdr_self_clear_on_idle() { + setup; src_watchdog + herdr_pane w1:p3 idle + local f; f=$(mk_deadline herdr:w1:p3 -120 -1800 "finished") + touch "$TMP/transcript.jsonl" + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + [[ ! -f "$f" ]] || { echo "not self-cleared"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "notified on self-clear"; return 1; } +} + +test_wd_herdr_phoenix_defers() { + setup; src_watchdog + herdr_pane w1:p3 working + mkdir -p "$CC_WATCHDOG_AGENTS_STATE" + touch "$CC_WATCHDOG_AGENTS_STATE/abc.limit-wait" + echo "w1:p3" > "$CC_WATCHDOG_AGENTS_STATE/abc.herdr-pane" + src_watchdog # re-source after the state dir exists + local f; f=$(mk_deadline herdr:w1:p3 -120 -1800 "limit") + process_deadline "$f" || { echo "process_deadline failed"; return 1; } + grep -q 'DEFER: Phoenix owns' "$CC_WATCHDOG_HOME/log" || { echo "no DEFER log"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "notified despite Phoenix"; return 1; } +} + +t deadman_herdr_pane_key test_deadman_herdr_pane_key +t wd_herdr_vanished_detected test_wd_herdr_vanished_detected +t wd_herdr_phoenix_defers test_wd_herdr_phoenix_defers +t wd_herdr_status_mapping test_wd_herdr_status_mapping +t wd_herdr_tier2_uses_pane_run test_wd_herdr_tier2_uses_pane_run +t wd_herdr_unreachable_degrades_to_notify test_wd_herdr_unreachable_degrades_to_notify +t wd_herdr_self_clear_on_idle test_wd_herdr_self_clear_on_idle + # --- tests appended by later tasks above this line --- echo "---" diff --git a/onboarding/watchdog/test/shims/herdr b/onboarding/watchdog/test/shims/herdr new file mode 100755 index 0000000..50fe6ec --- /dev/null +++ b/onboarding/watchdog/test/shims/herdr @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Test shim: driven by files under $HERDR_SHIM_DIR; missing file => exit 1 +# (models herdr being unreachable / refusing external control). +case "$1 $2" in + "pane list") cat "$HERDR_SHIM_DIR/panes.json" ;; + "pane get") cat "$HERDR_SHIM_DIR/status-$3.json" ;; + "pane run") id=$3; shift 3; echo "$id $*" >> "$HERDR_SHIM_DIR/pane-run.log" ;; + *) exit 0 ;; +esac From 5284438d8d77953b0fd654c50ef336485f6434e2 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 06:32:26 +0300 Subject: [PATCH 06/11] watchdog: notify-only backstop for undeclared stalled sessions Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-stall-watchdog.sh | 39 +++++++++++++++++ onboarding/watchdog/test/run_tests.sh | 54 ++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh index 03b781e..ac65bd5 100755 --- a/onboarding/watchdog/cc-stall-watchdog.sh +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -123,6 +123,45 @@ inject_recovery() { # $1 pane, $2 reason tmux send-keys -t "$1" Enter } +cc_panes() { # emit " " for panes running a claude process + local pane ppid + while read -r pane ppid; do + pstree -p "$ppid" 2>/dev/null | grep -q 'claude(' && printf '%s %s\n' "$pane" "$ppid" + done < <(tmux list-panes -a -F '#{pane_id} #{pane_pid}' 2>/dev/null) +} + +pane_transcript_mtime() { # $1 = pane_pid → epoch mtime of newest transcript, or fail + local cpid cwd munged newest + cpid=$(pstree -p "$1" 2>/dev/null | grep -o 'claude([0-9]\+)' | head -1 | tr -dc 0-9) + [[ -n "$cpid" ]] || return 1 + cwd=$(readlink "/proc/$cpid/cwd" 2>/dev/null) || return 1 + munged=$(printf '%s' "$cwd" | sed 's#[/.]#-#g') + newest=$(ls -t "$HOME/.claude/projects/$munged"/*.jsonl 2>/dev/null | head -1) + [[ -n "$newest" ]] || return 1 + stat -c %Y "$newest" +} + +backstop_scan() { + local now pane ppid bs since tmtime stamp + now=$(date +%s) + while read -r pane ppid; do + bs="$STATE_DIR/pane-$pane.busy-since" + if ! pane_busy "$pane"; then rm -f "$bs"; continue; fi + [[ -f "$bs" ]] || echo "$now" > "$bs" + [[ -f "$STATE_DIR/$pane.deadline" ]] && continue # declared: ladder owns it + since=$(cat "$bs") + (( now - since >= BACKSTOP_HOURS * 3600 )) || continue + tmtime=$(pane_transcript_mtime "$ppid") || continue # can't resolve → skip, no guess + (( now - tmtime >= BACKSTOP_HOURS * 3600 )) || continue + stamp="$STATE_DIR/pane-$pane.backstop-notified" + if [[ -f "$stamp" ]] && (( now - $(cat "$stamp") < BACKSTOP_RENOTIFY_HOURS * 3600 )); then + continue + fi + notify "cc-watchdog BACKSTOP: [$(session_name "$pane")] busy >${BACKSTOP_HOURS}h, transcript stale, no declared deadline — possible silent stall (notify-only)" + echo "$now" > "$stamp" + done < <(cc_panes) +} + process_deadline() { # $1 = deadline file local f=$1 pane deadline setts reason transcript notified recovered now pane=$(get_field "$f" pane); deadline=$(get_field "$f" deadline) diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 7ed3f5b..8e81582 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -302,6 +302,60 @@ t wd_herdr_tier2_uses_pane_run test_wd_herdr_tier2_uses_pane_run t wd_herdr_unreachable_degrades_to_notify test_wd_herdr_unreachable_degrades_to_notify t wd_herdr_self_clear_on_idle test_wd_herdr_self_clear_on_idle +mk_cc_pane() { # — busy CC pane whose transcript lives under fake $HOME + echo "$1 100" > "$TMUX_SHIM_DIR/panes" + printf 'working... esc to interrupt\n' > "$TMUX_SHIM_DIR/content-$1" + echo "shell(100)---claude($2)" > "$TMUX_SHIM_DIR/pstree-100" +} + +test_wd_backstop_trips_on_stale_busy_undeclared() { + setup + export HOME="$TMP/home" # fake HOME for transcript resolution + src_watchdog + mk_cc_pane %1 $$ # real pid → /proc/$$/cwd works + local munged proj + munged=$(readlink "/proc/$$/cwd" | sed 's#[/.]#-#g') + proj="$HOME/.claude/projects/$munged" + mkdir -p "$proj" + touch -d '7 hours ago' "$proj/session.jsonl" + echo "$(( $(date +%s) - 7 * 3600 ))" > "$CC_WATCHDOG_HOME/state/pane-%1.busy-since" + backstop_scan || { echo "backstop_scan failed"; return 1; } + grep -q 'BACKSTOP' "$SHIM_LOG_DIR/curl.log" || { echo "no backstop notify"; return 1; } + backstop_scan || { echo "backstop_scan failed on renotify pass"; return 1; } # renotify suppression + (( $(grep -c 'BACKSTOP' "$SHIM_LOG_DIR/curl.log") == 1 )) || { echo "renotified"; return 1; } +} + +test_wd_backstop_skips_declared_and_fresh() { + setup + export HOME="$TMP/home" + src_watchdog + mk_cc_pane %1 $$ + # fresh pane: the scan must enumerate it (creating busy-since — the positive + # control proving the scan actually ran) yet not trip + backstop_scan || { echo "backstop_scan failed"; return 1; } + [[ -f "$CC_WATCHDOG_HOME/state/pane-%1.busy-since" ]] || { echo "pane not enumerated"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "tripped while fresh"; return 1; } + # stale busy-since but declared deadline → A-layer owns it, no trip + echo "$(( $(date +%s) - 7 * 3600 ))" > "$CC_WATCHDOG_HOME/state/pane-%1.busy-since" + mk_deadline %1 600 -60 "declared" >/dev/null + backstop_scan || { echo "backstop_scan failed on 2nd pass"; return 1; } + [[ ! -f "$SHIM_LOG_DIR/curl.log" ]] || { echo "tripped despite declaration"; return 1; } +} + +test_wd_backstop_clears_busy_since_on_idle() { + setup; src_watchdog + echo "%1 100" > "$TMUX_SHIM_DIR/panes" + echo "shell(100)---claude(1)" > "$TMUX_SHIM_DIR/pstree-100" + printf '> \n' > "$TMUX_SHIM_DIR/content-%1" # idle + echo "123" > "$CC_WATCHDOG_HOME/state/pane-%1.busy-since" + backstop_scan || { echo "backstop_scan failed"; return 1; } + [[ ! -f "$CC_WATCHDOG_HOME/state/pane-%1.busy-since" ]] +} + +t wd_backstop_trips_on_stale_busy_undeclared test_wd_backstop_trips_on_stale_busy_undeclared +t wd_backstop_skips_declared_and_fresh test_wd_backstop_skips_declared_and_fresh +t wd_backstop_clears_busy_since_on_idle test_wd_backstop_clears_busy_since_on_idle + # --- tests appended by later tasks above this line --- echo "---" From d7ecf5fe46d0e9e96ce397b195b603a085088832 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 06:48:20 +0300 Subject: [PATCH 07/11] watchdog: systemd units, installer, README Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/README.md | 72 +++++++++++++++++++ onboarding/watchdog/cc-stall-watchdog.service | 6 ++ onboarding/watchdog/cc-stall-watchdog.timer | 9 +++ onboarding/watchdog/install.sh | 18 +++++ onboarding/watchdog/test/run_tests.sh | 15 ++++ 5 files changed, 120 insertions(+) create mode 100644 onboarding/watchdog/README.md create mode 100644 onboarding/watchdog/cc-stall-watchdog.service create mode 100644 onboarding/watchdog/cc-stall-watchdog.timer create mode 100755 onboarding/watchdog/install.sh diff --git a/onboarding/watchdog/README.md b/onboarding/watchdog/README.md new file mode 100644 index 0000000..b6d4c12 --- /dev/null +++ b/onboarding/watchdog/README.md @@ -0,0 +1,72 @@ +# cc-watchdog — dead-man's switch for Claude Code sessions + +Long autonomous Claude Code sessions can die out silently: the session waits on +background work (a test suite, a build) whose owner died, and nothing ever wakes +it — it looks exactly like "done, waiting for you". cc-watchdog catches this. + +## How it works + +1. Before a session goes quiet while waiting on something, it (or you) declares + a deadline: `cc-deadman set 40 "full test suite, ~25 min"` (expected duration + + 50% margin). +2. A systemd user timer runs `cc-stall-watchdog.sh` every 5 minutes: + - Session progressed and is idle again → deadline self-clears. + - Deadline missed → **Slack ping** (tier 1). + - Still nothing an hour later (`GRACE_MIN=60`) → the watchdog **nudges the + session itself** (tier 2): Escape if busy, then a recovery prompt asking it + to check honest status and continue. Repeat stalls within 24 h back off to + notify-only — no injection loops. + - Sessions that never declared: if a pane is mid-turn busy for over + `BACKSTOP_HOURS=6` h with an equally stale transcript, you get a + notify-only ping (max once per 12 h). Legitimate long quiet runs (a 2.75 h + suite) can never trip this. + +The watchdog is deliberately timer-driven, not Claude-Code-hook-driven: hook +watchers only wake when a turn ends, which is exactly the blind spot that +produces multi-hour stalls. + +## tmux and Herdr + +Sessions under **tmux** and under **Herdr** are both supported. Inside a Herdr +pane, `cc-deadman` keys the deadline to `$HERDR_PANE_ID` automatically; the +watchdog then reads Herdr's native `agent_status` (busy = `working`; `blocked` +counts as NOT idle, so a blocked agent still triggers your tier-1 ping) and +injects recovery via `herdr pane run`. If `herdr` refuses commands from the +timer's context, the watchdog degrades gracefully: the pane is assumed alive, +injection is skipped (logged), and the Slack alert still fires. Run the smoke +test below once from inside a Herdr pane to confirm external control works on +your setup. Limitation: the no-declaration backstop scans tmux panes only — +Herdr sessions are covered by declared deadlines. + +## Install + + ./install.sh + +Then set your Slack incoming webhook in `~/.cc-watchdog/config`: + + SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... + +Optional overrides in the same file: `GRACE_MIN`, `BACKSTOP_HOURS`, +`BACKSTOP_RENOTIFY_HOURS`, `RECOVERY_BACKOFF_HOURS`, `WATCHDOG_DRY=1` (log-only). + +## Session-side commands + + cc-deadman set "" # declare before a long wait + cc-deadman extend # work is legitimately taking longer + cc-deadman clear # first action after resuming + cc-deadman list # what's armed right now + +## Smoke test + +1. In a tmux Claude Code session: `cc-deadman set 1 "smoke test"`. +2. Within ~6 min: Slack ping (tier 1). +3. To see tier 2 quickly: `echo GRACE_MIN=2 >> ~/.cc-watchdog/config`, wait + ~3 more min, watch the recovery prompt land in the pane. Revert the config + line and `cc-deadman clear` afterwards. + +Recommended first deployment: `WATCHDOG_DRY=1` in the config for 2–3 days; +watch `~/.cc-watchdog/log` for what it *would* have done, then remove the flag. + +## Unit tests + + test/run_tests.sh # hermetic; fakes tmux/curl/pstree via PATH shims diff --git a/onboarding/watchdog/cc-stall-watchdog.service b/onboarding/watchdog/cc-stall-watchdog.service new file mode 100644 index 0000000..16d5b47 --- /dev/null +++ b/onboarding/watchdog/cc-stall-watchdog.service @@ -0,0 +1,6 @@ +[Unit] +Description=Claude Code stall watchdog (dead-man's switch tick) + +[Service] +Type=oneshot +ExecStart=%h/.local/bin/cc-stall-watchdog.sh diff --git a/onboarding/watchdog/cc-stall-watchdog.timer b/onboarding/watchdog/cc-stall-watchdog.timer new file mode 100644 index 0000000..fdfcc0c --- /dev/null +++ b/onboarding/watchdog/cc-stall-watchdog.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run cc-stall-watchdog every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min + +[Install] +WantedBy=timers.target diff --git a/onboarding/watchdog/install.sh b/onboarding/watchdog/install.sh new file mode 100755 index 0000000..0bc5115 --- /dev/null +++ b/onboarding/watchdog/install.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# install.sh — install cc-watchdog: symlinks into ~/.local/bin + systemd user timer. +# Idempotent. CC_WATCHDOG_NO_SYSTEMD=1 skips systemctl (tests/CI). +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +BIN="$HOME/.local/bin" +UNITS="$HOME/.config/systemd/user" +mkdir -p "$BIN" "$UNITS" "$HOME/.cc-watchdog/state" +ln -sf "$HERE/cc-deadman" "$BIN/cc-deadman" +ln -sf "$HERE/cc-stall-watchdog.sh" "$BIN/cc-stall-watchdog.sh" +cp "$HERE/cc-stall-watchdog.service" "$HERE/cc-stall-watchdog.timer" "$UNITS/" +touch "$HOME/.cc-watchdog/config" +if [[ -z "${CC_WATCHDOG_NO_SYSTEMD:-}" ]]; then + systemctl --user daemon-reload + systemctl --user enable --now cc-stall-watchdog.timer +fi +echo "cc-watchdog installed." +echo "Next: put SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... in ~/.cc-watchdog/config" diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 8e81582..ac6106d 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -356,6 +356,21 @@ t wd_backstop_trips_on_stale_busy_undeclared test_wd_backstop_trips_on_stale_bus t wd_backstop_skips_declared_and_fresh test_wd_backstop_skips_declared_and_fresh t wd_backstop_clears_busy_since_on_idle test_wd_backstop_clears_busy_since_on_idle +test_install_creates_links_and_units() { + setup + export HOME="$TMP/home"; mkdir -p "$HOME" + CC_WATCHDOG_NO_SYSTEMD=1 "$HERE/../install.sh" >/dev/null || return 1 + [[ -L "$HOME/.local/bin/cc-deadman" ]] || { echo "no cc-deadman link"; return 1; } + [[ -L "$HOME/.local/bin/cc-stall-watchdog.sh" ]] || { echo "no watchdog link"; return 1; } + [[ -f "$HOME/.config/systemd/user/cc-stall-watchdog.timer" ]] || { echo "no timer unit"; return 1; } + [[ -f "$HOME/.config/systemd/user/cc-stall-watchdog.service" ]] || { echo "no service unit"; return 1; } + [[ -f "$HOME/.cc-watchdog/config" ]] || { echo "no config stub"; return 1; } + # idempotent + CC_WATCHDOG_NO_SYSTEMD=1 "$HERE/../install.sh" >/dev/null || { echo "second run failed"; return 1; } +} + +t install_creates_links_and_units test_install_creates_links_and_units + # --- tests appended by later tasks above this line --- echo "---" From 92e9a1305799bf39cf529ad891d0b8408037d228 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 06:57:51 +0300 Subject: [PATCH 08/11] ci: shellcheck + unit tests for cc-watchdog Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bc869f..d4cd9e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,18 @@ jobs: # Scan the working tree (not full history): a "don't add secrets" gate, # free of false positives from large base64 blobs in old commits. run: gitleaks dir --config .gitleaks.toml --redact --verbose . + + watchdog: + name: cc-watchdog (shellcheck + unit tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: shellcheck + run: | + shellcheck onboarding/watchdog/cc-deadman \ + onboarding/watchdog/cc-stall-watchdog.sh \ + onboarding/watchdog/install.sh \ + onboarding/watchdog/test/run_tests.sh \ + onboarding/watchdog/test/shims/* + - name: unit tests + run: onboarding/watchdog/test/run_tests.sh From cbde80b4184924b4d8dfec8514e5f8c13917a537 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 07:07:21 +0300 Subject: [PATCH 09/11] docs: disseminate cc-watchdog through onboarding + doctrine Member-facing docs for cc-watchdog: ONBOARDING step 13 (install + tmux/Herdr note) with two section-C smoke checks, a MAINTAINING rationale entry, and the silent-stall-prevention doctrine block in CLAUDE.global.md. Co-Authored-By: Claude Fable 5 --- onboarding/CLAUDE.global.md | 15 +++++++++++++++ onboarding/MAINTAINING.md | 1 + onboarding/ONBOARDING.md | 38 +++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/onboarding/CLAUDE.global.md b/onboarding/CLAUDE.global.md index 9fde7a3..c6372e1 100644 --- a/onboarding/CLAUDE.global.md +++ b/onboarding/CLAUDE.global.md @@ -76,3 +76,18 @@ Before building on a premise — what a field actually contains at the read site Treat unverified premises as likely-wrong, not likely-right. Across long arcs of load-bearing checks the probes invert the premise more often than they confirm it (the moments weren't end-of-run; the "healthy" channel was dead; the MW didn't survive the data boundary; the core had zero reactions where the counts implied dozens; the "reference-state-paired" reaction wasn't, in the guard's eyes). A plan built on an unverified premise produces correct implementations of wrong designs — the most expensive failure class, because every downstream review validates against the same false premise. Corollary — don't build to pass the check. When a guard / test / tripwire refuses your construct, do not reshape the construct to slip under it; a thing built only to make a check go green validates nothing that transfers. Ask first whether the guard caught a real defect (then fix the defect) or exposed a false-positive in itself (then record it as a finding) — not how to get past it. + +# Silent-stall prevention (cc-watchdog contract) + +Long autonomous sessions must never wait indefinitely on work that could die +silently: + +- Run final gates (full test suites, builds) in the mother session as tracked + background tasks — never owned by a subagent that will stop before the work + completes. +- Before ending any turn while background/external work is still pending, + declare a dead-man deadline: `cc-deadman set + ""`. +- First action after resuming from a wait: `cc-deadman clear` (or re-arm for + the next wait). +- Wrap any command without a natural bound in `timeout`. diff --git a/onboarding/MAINTAINING.md b/onboarding/MAINTAINING.md index 13208dd..e238fb9 100644 --- a/onboarding/MAINTAINING.md +++ b/onboarding/MAINTAINING.md @@ -47,6 +47,7 @@ not by copying secrets or by being granted access to the PI's machines. | Statusline | the **simple context-% script**, not the PI's `~/agents` auto-handoff variant | Self-contained; no extra infra to stand up. | | Seeds | **sanitized**, real files under `vault-seeds/` | Easy `cp` into place; the PI's real remote-dev note (tailnet IPs, VPN endpoint, cluster/exit-node config) is excluded — a generic `Remote Dev — Pattern` replaces it. | | First pass | **Claude Code only** | Smaller surface; the deferred list below grows it later. | +| Silent-stall detection | **cc-watchdog** (`onboarding/watchdog/`): timer-driven dead-man's switch, not hook-driven heuristics | CC-hook watchers only wake at turn ends, so they share the session's blind spot (a 36 h TA die-out in 2026-07 motivated this). Deadlines declared by the session itself make false positives ~zero (a 2.75 h quiet suite run is legitimate work); a 6 h notify-only backstop covers sessions that never declared. Unlike the PI-local auto-handoff/Phoenix stack, this ships to members: standalone (tmux + coreutils + systemd user timer + one Slack webhook). | ## Deferred — "later" (and how to un-defer) diff --git a/onboarding/ONBOARDING.md b/onboarding/ONBOARDING.md index 4fcf3f8..9425f94 100644 --- a/onboarding/ONBOARDING.md +++ b/onboarding/ONBOARDING.md @@ -168,6 +168,41 @@ headroom perf # savings, once traffic has flowed > linger from step **c** — but only if no *other* systemd user service relies on it (linger keeps > all your user services alive across logout, not just Headroom's). +### 13. cc-watchdog — silent-stall guard for long agent sessions + +Long autonomous Claude Code runs can die out silently: the session waits on +background work (a test suite, a build) whose owner died, and nothing ever +wakes it — it looks exactly like "done, waiting for you". cc-watchdog is a +dead-man's switch that catches this. + +Before a session goes quiet while waiting on something, it declares a deadline +(`cc-deadman set 40 "full test suite, ~25 min"`). A systemd user timer checks +every 5 minutes; if the deadline passes you get a Slack ping, and if nothing +recovers within an hour the watchdog nudges the session itself ("your awaited +work likely died — check honest status and continue"). + +Install: +```bash +cd ~/Code/DRGScripts/onboarding/watchdog +./install.sh +``` +Then put your Slack incoming-webhook URL in `~/.cc-watchdog/config`: +```bash +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +``` +(Create one: api.slack.com/apps → your app → Incoming Webhooks → add to your +own DM or a private channel.) + +Works for agent sessions under tmux and under Herdr alike (inside a Herdr pane +the deadline keys itself to the Herdr pane id and recovery uses `herdr pane +run`; see the README's "tmux and Herdr" section and run its smoke test once +from inside Herdr). + +The session-side habit lives in the global CLAUDE.md you merged in step 8 +(silent-stall prevention rules): agents declare deadlines before long waits and +run final gates in the mother session. The watchdog is the enforcement; the +rules are the prevention. Details: `onboarding/watchdog/README.md`. + --- ## B. Laptop (Windows / macOS) — thin client + Obsidian @@ -211,6 +246,9 @@ headroom perf # savings, once traffic has flowed healthy*; in a **freshly started** Claude Code session, typing `!env | grep ANTHROPIC_BASE_URL` at the Claude Code prompt (the leading `!` tells Claude Code to run the rest as a shell command) prints `http://127.0.0.1:8787`. +- [ ] `systemctl --user status cc-stall-watchdog.timer` shows *active (waiting)*. +- [ ] In a tmux Claude Code session: `cc-deadman set 1 "smoke test"` → Slack ping within + ~6 min (then `cc-deadman clear`). --- From 31890efafd636ae49f71affadd3850962ea8d902 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 07:36:45 +0300 Subject: [PATCH 10/11] watchdog: baseline-stamped self-clear + robust Slack JSON escaping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-clear must distinguish the declaring turn's own transcript tail writes from later resumed activity: cc-deadman runs as a tool call, so bare mtime>set — and any freshness-vs-now window — archives every declared deadline at the first idle tick, leaving idle-at-prompt stalls unalertable. First idle tick now stamps baseline=; self-clear requires strictly newer activity. extend strips the baseline. notify() now escapes backslashes/quotes and flattens control chars so a hostile reason can never cost the tier-1 alert. Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/README.md | 3 ++ onboarding/watchdog/cc-deadman | 2 +- onboarding/watchdog/cc-stall-watchdog.sh | 35 +++++++++++++++++----- onboarding/watchdog/test/run_tests.sh | 37 ++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 8 deletions(-) diff --git a/onboarding/watchdog/README.md b/onboarding/watchdog/README.md index b6d4c12..75e4e5c 100644 --- a/onboarding/watchdog/README.md +++ b/onboarding/watchdog/README.md @@ -66,6 +66,9 @@ Optional overrides in the same file: `GRACE_MIN`, `BACKSTOP_HOURS`, Recommended first deployment: `WATCHDOG_DRY=1` in the config for 2–3 days; watch `~/.cc-watchdog/log` for what it *would* have done, then remove the flag. +When you remove the flag to arm for real, also clear dry-run artifacts: +`rm -f ~/.cc-watchdog/state/pane-*.last-recovery` (dry runs stamp recoveries +too, and a stale stamp would wrongly back off the first real recovery). ## Unit tests diff --git a/onboarding/watchdog/cc-deadman b/onboarding/watchdog/cc-deadman index 17a7fbe..83e07d4 100755 --- a/onboarding/watchdog/cc-deadman +++ b/onboarding/watchdog/cc-deadman @@ -68,7 +68,7 @@ case "$cmd" in f="$STATE_DIR/$pane.deadline" [[ -f "$f" ]] || { echo "cc-deadman: no deadline set for $pane" >&2; exit 1; } old=$(sed -n 's/^deadline=//p' "$f") - grep -Ev '^(deadline|notified|recovered|failed|recovery_kind)=' "$f" > "$f.tmp" || true + grep -Ev '^(deadline|notified|recovered|failed|recovery_kind|baseline)=' "$f" > "$f.tmp" || true echo "deadline=$(( old + $1 * 60 ))" >> "$f.tmp" mv "$f.tmp" "$f" echo "cc-deadman: extended by $1 min" diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh index ac65bd5..3f1a9cd 100755 --- a/onboarding/watchdog/cc-stall-watchdog.sh +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -26,8 +26,15 @@ mkdir -p "$STATE_DIR" "$ARCHIVE_DIR" log() { printf '%s %s\n' "$(date '+%F %T')" "$*" >> "$LOG_FILE"; } -notify() { # $1 = message - local msg=${1//\"/\'} +notify() { # $1 = message — escape for the JSON payload; a dropped alert is + # the one unacceptable failure mode, so backslashes/quotes/control chars + # must never invalidate the body. + local msg=$1 + msg=${msg//\\/\\\\} + msg=${msg//\"/\\\"} + msg=${msg//$'\n'/ } + msg=${msg//$'\t'/ } + msg=${msg//$'\r'/ } log "NOTIFY: $msg" [[ -n "${WATCHDOG_DRY:-}" ]] && return 0 if [[ -n "${SLACK_WEBHOOK_URL:-}" ]]; then @@ -177,11 +184,25 @@ process_deadline() { # $1 = deadline file if phoenix_owns "$pane"; then log "DEFER: Phoenix owns $pane"; return; fi - # Self-clear: the session did something after declaring, and is idle again. - if [[ -n "$transcript" && -f "$transcript" ]] \ - && (( $(stat -c %Y "$transcript") > setts )) && pane_idle "$pane"; then - log "SELF-CLEAR: $pane progressed and is idle ($reason)" - mv "$f" "$ARCHIVE_DIR/"; return + # Self-clear needs a BASELINE distinguishing the declaring turn's own tail + # writes from later resumed activity: cc-deadman runs as a tool call, so + # the transcript is appended AFTER set=, and any freshness-vs-now window + # fails because the first tick after turn-end always lands inside it + # (5-min cadence). First idle tick stamps baseline=; + # self-clear requires strictly newer activity than that baseline, while + # idle. + if [[ -n "$transcript" && -f "$transcript" ]]; then + local tm baseline + tm=$(stat -c %Y "$transcript" 2>/dev/null) || tm="" + baseline=$(get_field "$f" baseline) + if [[ -n "$tm" ]] && pane_idle "$pane"; then + if [[ -z "$baseline" ]]; then + set_field "$f" baseline "$tm" # the declaring turn's final write + elif (( tm > baseline )); then + log "SELF-CLEAR: $pane progressed past baseline and is idle ($reason)" + mv "$f" "$ARCHIVE_DIR/"; return + fi + fi fi (( now < deadline )) && return diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index ac6106d..34614c2 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -47,12 +47,14 @@ test_deadman_extend_pushes_and_resets_tiers() { TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "x" >/dev/null local f="$CC_WATCHDOG_HOME/state/%7.deadline" echo "notified=123" >> "$f" + echo "baseline=123" >> "$f" local before after before=$(sed -n 's/^deadline=//p' "$f") TMUX_PANE=%7 "$HERE/../cc-deadman" extend 10 >/dev/null after=$(sed -n 's/^deadline=//p' "$f") (( after - before == 600 )) || { echo "bad extend"; return 1; } ! grep -q '^notified=' "$f" || { echo "notified not reset"; return 1; } + ! grep -q '^baseline=' "$f" || { echo "baseline not stripped"; return 1; } } test_deadman_pane_from_process_tree() { @@ -178,6 +180,7 @@ test_wd_self_clear_on_progress_and_idle() { echo "%1 100" > "$TMUX_SHIM_DIR/panes" printf '> \n' > "$TMUX_SHIM_DIR/content-%1" # idle local f; f=$(mk_deadline %1 -120 -1800 "done actually") + set_field "$f" baseline "$(( $(date +%s) - 900 ))" touch "$TMP/transcript.jsonl" # mtime now > set time process_deadline "$f" [[ ! -f "$f" ]] || { echo "not cleared"; return 1; } @@ -275,6 +278,7 @@ test_wd_herdr_self_clear_on_idle() { setup; src_watchdog herdr_pane w1:p3 idle local f; f=$(mk_deadline herdr:w1:p3 -120 -1800 "finished") + set_field "$f" baseline "$(( $(date +%s) - 900 ))" touch "$TMP/transcript.jsonl" process_deadline "$f" || { echo "process_deadline failed"; return 1; } [[ ! -f "$f" ]] || { echo "not self-cleared"; return 1; } @@ -371,6 +375,39 @@ test_install_creates_links_and_units() { t install_creates_links_and_units test_install_creates_links_and_units +test_wd_self_clear_baseline_sequence() { + setup; src_watchdog + echo "%1 100" > "$TMUX_SHIM_DIR/panes" + printf '> \n' > "$TMUX_SHIM_DIR/content-%1" # idle at prompt + local f; f=$(mk_deadline %1 300 -60 "awaiting suite") # set 1 min ago, due in 5 + touch "$TMP/transcript.jsonl" # turn-end write, seconds after set + # tick 1 (pre-expiry, fresh transcript, idle): must NOT archive; stamps baseline + process_deadline "$f" || { echo "tick1 failed"; return 1; } + [[ -f "$f" ]] || { echo "archived at first idle tick (the production bug)"; return 1; } + grep -q '^baseline=' "$f" || { echo "no baseline stamped"; return 1; } + # deadline passes, transcript frozen (the stall): tier 1 must fire + set_field "$f" deadline "$(( $(date +%s) - 60 ))" + process_deadline "$f" || { echo "tick2 failed"; return 1; } + grep -q 'deadline MISSED' "$SHIM_LOG_DIR/curl.log" || { echo "tier1 did not fire"; return 1; } + [[ -f "$f" ]] || { echo "file gone after tier1"; return 1; } + # session resumes: write past baseline -> self-clears + sleep 1.1; touch "$TMP/transcript.jsonl" + process_deadline "$f" || { echo "tick3 failed"; return 1; } + [[ ! -f "$f" ]] || { echo "did not self-clear after resumed activity"; return 1; } +} + +test_wd_notify_json_escapes_hostile_reason() { + setup; src_watchdog + notify 'path a\b and "quotes" here' + local payload + payload=$(grep -o '{.*}' "$SHIM_LOG_DIR/curl.log" | head -1) + python3 -c 'import json,sys; d=json.loads(sys.argv[1]); assert "a\\b" in d["text"] and "quotes" in d["text"]' "$payload" \ + || { echo "payload not valid JSON with escaped content: $payload"; return 1; } +} + +t wd_self_clear_baseline_sequence test_wd_self_clear_baseline_sequence +t wd_notify_json_escapes_hostile_reason test_wd_notify_json_escapes_hostile_reason + # --- tests appended by later tasks above this line --- echo "---" From 0bb1e9009f555015102f79fb5a45dd5b0932ad61 Mon Sep 17 00:00:00 2001 From: Alon Grinberg Dana Date: Tue, 14 Jul 2026 11:10:31 +0300 Subject: [PATCH 11/11] ci: satisfy first real shellcheck run Quote herdr status keywords (SC1010), drop the dead setts read left by the baseline-stamp redesign (SC2034), guard test cd calls (SC2164), rework cc-deadman arg parsing away from A&&B||C (SC2015), and add reasoned disables where the flagged code is correct (SC2012 mtime-sort ls, SC2001 per-line sed prefix, SC1091 runtime source). Co-Authored-By: Claude Fable 5 --- onboarding/watchdog/cc-deadman | 5 ++++- onboarding/watchdog/cc-stall-watchdog.sh | 7 ++++--- onboarding/watchdog/test/run_tests.sh | 25 ++++++++++++++++-------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/onboarding/watchdog/cc-deadman b/onboarding/watchdog/cc-deadman index 83e07d4..a89ca27 100755 --- a/onboarding/watchdog/cc-deadman +++ b/onboarding/watchdog/cc-deadman @@ -42,10 +42,13 @@ resolve_transcript() { # Claude Code stores transcripts under a cwd-munged directory name. local munged munged=$(pwd | sed 's#[/.]#-#g') + # shellcheck disable=SC2012 # ls -t is the point (mtime sort); CC-generated paths contain no newlines ls -t "$HOME/.claude/projects/$munged"/*.jsonl 2>/dev/null | head -1 || true } -cmd=${1:-}; [[ -n "$cmd" ]] && shift || usage +cmd=${1:-} +[[ -n "$cmd" ]] || usage +shift case "$cmd" in set) (( $# == 2 )) || usage diff --git a/onboarding/watchdog/cc-stall-watchdog.sh b/onboarding/watchdog/cc-stall-watchdog.sh index 3f1a9cd..035431f 100755 --- a/onboarding/watchdog/cc-stall-watchdog.sh +++ b/onboarding/watchdog/cc-stall-watchdog.sh @@ -77,7 +77,7 @@ pane_idle() { local s; s=$(herdr_status "$1") # 'blocked' (agent waiting for input) is deliberately NOT idle: a blocked # agent must not self-clear its deadline — tier 1 should reach the user. - [[ "$s" == idle || "$s" == done ]] + [[ "$s" == "idle" || "$s" == "done" ]] else pane_alive "$1" && ! pane_busy "$1" fi @@ -143,6 +143,7 @@ pane_transcript_mtime() { # $1 = pane_pid → epoch mtime of newest transcript, [[ -n "$cpid" ]] || return 1 cwd=$(readlink "/proc/$cpid/cwd" 2>/dev/null) || return 1 munged=$(printf '%s' "$cwd" | sed 's#[/.]#-#g') + # shellcheck disable=SC2012 # ls -t is the point (mtime sort); CC-generated paths contain no newlines newest=$(ls -t "$HOME/.claude/projects/$munged"/*.jsonl 2>/dev/null | head -1) [[ -n "$newest" ]] || return 1 stat -c %Y "$newest" @@ -170,9 +171,9 @@ backstop_scan() { } process_deadline() { # $1 = deadline file - local f=$1 pane deadline setts reason transcript notified recovered now + local f=$1 pane deadline reason transcript notified recovered now pane=$(get_field "$f" pane); deadline=$(get_field "$f" deadline) - setts=$(get_field "$f" set); reason=$(get_field "$f" reason) + reason=$(get_field "$f" reason) transcript=$(get_field "$f" transcript) notified=$(get_field "$f" notified); recovered=$(get_field "$f" recovered) now=$(date +%s) diff --git a/onboarding/watchdog/test/run_tests.sh b/onboarding/watchdog/test/run_tests.sh index 34614c2..a9efa10 100755 --- a/onboarding/watchdog/test/run_tests.sh +++ b/onboarding/watchdog/test/run_tests.sh @@ -6,8 +6,14 @@ PASS=0; FAIL=0 t() { # t local out - if out=$("$2" 2>&1); then echo "PASS $1"; PASS=$((PASS + 1)) - else echo "FAIL $1"; echo "$out" | sed 's/^/ /'; FAIL=$((FAIL + 1)); fi + if out=$("$2" 2>&1); then + echo "PASS $1"; PASS=$((PASS + 1)) + else + echo "FAIL $1" + # shellcheck disable=SC2001 # per-line prefix of multiline output; parameter expansion cannot anchor ^ + echo "$out" | sed 's/^/ /' + FAIL=$((FAIL + 1)) + fi } setup() { @@ -24,7 +30,7 @@ setup() { } test_deadman_set_writes_fields() { - setup; cd "$TMP" + setup; cd "$TMP" || return 1 TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "suite run" >/dev/null || return 1 local f="$CC_WATCHDOG_HOME/state/%7.deadline" [[ -f "$f" ]] || { echo "no deadline file"; return 1; } @@ -36,14 +42,14 @@ test_deadman_set_writes_fields() { } test_deadman_clear_removes() { - setup; cd "$TMP" + setup; cd "$TMP" || return 1 TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "x" >/dev/null || return 1 TMUX_PANE=%7 "$HERE/../cc-deadman" clear >/dev/null || return 1 [[ ! -f "$CC_WATCHDOG_HOME/state/%7.deadline" ]] } test_deadman_extend_pushes_and_resets_tiers() { - setup; cd "$TMP" + setup; cd "$TMP" || return 1 TMUX_PANE=%7 "$HERE/../cc-deadman" set 30 "x" >/dev/null local f="$CC_WATCHDOG_HOME/state/%7.deadline" echo "notified=123" >> "$f" @@ -58,7 +64,7 @@ test_deadman_extend_pushes_and_resets_tiers() { } test_deadman_pane_from_process_tree() { - setup; cd "$TMP" + setup; cd "$TMP" || return 1 echo "%9 $$" > "$TMUX_SHIM_DIR/panes" # this test shell is cc-deadman's ancestor "$HERE/../cc-deadman" set 5 "tree" >/dev/null || return 1 [[ -f "$CC_WATCHDOG_HOME/state/%9.deadline" ]] @@ -69,7 +75,10 @@ t deadman_clear_removes test_deadman_clear_removes t deadman_extend_pushes_and_resets_tiers test_deadman_extend_pushes_and_resets_tiers t deadman_pane_from_process_tree test_deadman_pane_from_process_tree -src_watchdog() { . "$HERE/../cc-stall-watchdog.sh"; } +src_watchdog() { + # shellcheck disable=SC1091 # sourced from the repo tree at runtime; path is dynamic + . "$HERE/../cc-stall-watchdog.sh" +} test_wd_notify_posts_to_slack() { setup; src_watchdog @@ -220,7 +229,7 @@ t wd_vanished_pane_notifies_and_archives test_wd_vanished_pane_notifies_and_arch t wd_phoenix_defers test_wd_phoenix_defers test_deadman_herdr_pane_key() { - setup; cd "$TMP" + setup; cd "$TMP" || return 1 HERDR_ENV=1 HERDR_PANE_ID=w1:p3 "$HERE/../cc-deadman" set 30 "melt sim" >/dev/null || return 1 local f="$CC_WATCHDOG_HOME/state/herdr:w1:p3.deadline" [[ -f "$f" ]] || { echo "no herdr deadline file"; return 1; }