Keep Tier-0 health checks bounded on large databases

docs/operations/brainlayer-health-check.md

@@ -15,14 +15,34 @@ brainlayer health-check --json --heal
 - Active chunks missing semantic vectors are decreasing across ticks. One unchanged tick is tolerated; the second unchanged tick alarms.
 - BrainBar's served MCP socket can answer a `brain_search` canary with at least one result.
 
+The missing-vector count is exact, but it computes the ID difference through the
+covering `chunks` and `chunk_vectors_rowids` indexes before reading chunk payloads.
+This keeps the scheduled check independent of database payload size.
+
+The check has a 45-second internal deadline. A query that exhausts that budget is
+interrupted and the state file is still refreshed with `slow_check: true`, the
+stage name, and the elapsed duration.
+
 ## Self-Heal
 
 With `--heal`, the check uses `launchctl kickstart -k` for cheap recovery:
 
 - `com.brainlayer.hotlane-brainbar` when hotlane is dead, backlog is disabled, or missing vectors are climbing/stalled.
 - `com.brainlayer.brainbar-daemon` when the BrainBar MCP canary fails or returns zero results.
 
-The command always writes the latest missing-vector count to `~/.local/share/brainlayer/health-check-state.json`.
+Before a kickstart, the check reads the launchd process state. Processes in an
+uninterruptible `U`/`D` state are not kickstarted repeatedly; the check records a
+`heal_backoff` action and the existing circuit breaker escalates after repeated
+failed ticks.
+
+The command writes the latest successfully measured missing-vector count to
+`~/.local/share/brainlayer/health-check-state.json`; a timed-out tick preserves
+the previous count and marks the state as slow.
+
+The independent Tier-0 watchdog treats `slow_check: true` as unhealthy even
+when the state file was just refreshed, and treats any state as stale after 900
+seconds. Its first alert is immediate; repeat alerts for the same failure class
+are suppressed for 1,800 seconds while recovery attempts continue.
 
 ## Logs
 

scripts/launchd/com.brainlayer.tier0-watchdog.plist

@@ -15,6 +15,12 @@
 		<string>__HOME__</string>
 		<key>PATH</key>
 		<string>/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
+		<key>TIER0_ALERT_STATE_PATH</key>
+		<string>__HOME__/.local/share/brainlayer/tier0-watchdog-alert-state</string>
+		<key>TIER0_REPEAT_ALERT_SECONDS</key>
+		<string>1800</string>
+		<key>TIER0_STALE_SECONDS</key>
+		<string>900</string>
 	</dict>
 	<key>StandardOutPath</key>
 	<string>__HOME__/Library/Logs/brainlayer/tier0-watchdog.out.log</string>

scripts/tier0-watchdog.sh

@@ -12,6 +12,7 @@ TIER0_ID=${TIER0_ID:-/usr/bin/id}
 TIER0_SLEEP=${TIER0_SLEEP:-/bin/sleep}
 TIER0_DIRNAME=${TIER0_DIRNAME:-/usr/bin/dirname}
 TIER0_MKDIR=${TIER0_MKDIR:-/bin/mkdir}
+TIER0_GREP=${TIER0_GREP:-/usr/bin/grep}
 
 TIER0_LABEL=${TIER0_LABEL:-com.brainlayer.health-check}
 if [ -z "${TIER0_DOMAIN:-}" ]; then
@@ -26,8 +27,10 @@ fi
 TIER0_STATE_PATH=${TIER0_STATE_PATH:-$HOME/.local/share/brainlayer/health-check-state.json}
 TIER0_HEALTH_PLIST_PATH=${TIER0_HEALTH_PLIST_PATH:-$HOME/Library/LaunchAgents/com.brainlayer.health-check.plist}
 TIER0_LOG_PATH=${TIER0_LOG_PATH:-$HOME/.local/share/brainlayer/logs/tier0-watchdog.log}
+TIER0_ALERT_STATE_PATH=${TIER0_ALERT_STATE_PATH:-$HOME/.local/share/brainlayer/tier0-watchdog-alert-state}
 TIER0_NOTIFY_ENDPOINT=${TIER0_NOTIFY_ENDPOINT:-http://localhost:3847/notify}
-TIER0_STALE_SECONDS=${TIER0_STALE_SECONDS:-1200}
+TIER0_STALE_SECONDS=${TIER0_STALE_SECONDS:-900}
+TIER0_REPEAT_ALERT_SECONDS=${TIER0_REPEAT_ALERT_SECONDS:-1800}
 TIER0_ALERT_TIMEOUT_SECONDS=${TIER0_ALERT_TIMEOUT_SECONDS:-3}
 TIER0_NOTIFY_TIMEOUT_SECONDS=${TIER0_NOTIFY_TIMEOUT_SECONDS:-3}
 
@@ -54,6 +57,7 @@ require_epoch() {
 }
 
 require_positive_integer TIER0_STALE_SECONDS "$TIER0_STALE_SECONDS"
+require_positive_integer TIER0_REPEAT_ALERT_SECONDS "$TIER0_REPEAT_ALERT_SECONDS"
 require_positive_integer TIER0_ALERT_TIMEOUT_SECONDS "$TIER0_ALERT_TIMEOUT_SECONDS"
 require_positive_integer TIER0_NOTIFY_TIMEOUT_SECONDS "$TIER0_NOTIFY_TIMEOUT_SECONDS"
 
@@ -121,46 +125,92 @@ alert_all_channels() {
     wait_for_alerts "$log_pid" "$osascript_pid" "$curl_pid"
 }
 
+should_alert() {
+    failure_key=$1
+    if [ ! -f "$TIER0_ALERT_STATE_PATH" ]; then
+        return 0
+    fi
+
+    last_alert_epoch=
+    last_failure_key=
+    IFS="$(printf '\t')" read -r last_alert_epoch last_failure_key < "$TIER0_ALERT_STATE_PATH" || return 0
+    case "$last_alert_epoch" in
+        ''|*[!0-9]*) return 0 ;;
+    esac
+    if [ "$last_failure_key" != "$failure_key" ] || [ "$last_alert_epoch" -gt "$now_epoch" ]; then
+        return 0
+    fi
+    alert_age=$((now_epoch - last_alert_epoch))
+    [ "$alert_age" -ge "$TIER0_REPEAT_ALERT_SECONDS" ]
+}
+
+record_alert() {
+    failure_key=$1
+    alert_state_dir=$($TIER0_DIRNAME "$TIER0_ALERT_STATE_PATH" 2>/dev/null) || return 1
+    "$TIER0_MKDIR" -p "$alert_state_dir" 2>/dev/null || return 1
+    printf '%s\t%s\n' "$now_epoch" "$failure_key" > "$TIER0_ALERT_STATE_PATH"
+}
+
+reset_alert_cooldown() {
+    if [ -f "$TIER0_ALERT_STATE_PATH" ]; then
+        printf '0\tok\n' > "$TIER0_ALERT_STATE_PATH"
+    fi
+}
+
 target="$TIER0_DOMAIN/$TIER0_LABEL"
 failure_reason=
+failure_key=
 label_loaded=0
 
 if "$TIER0_LAUNCHCTL" print "$target" >/dev/null 2>&1; then
     label_loaded=1
 else
     failure_reason=label_unloaded
+    failure_key=label_unloaded
 fi
 
 if [ "$label_loaded" -eq 1 ]; then
     if [ ! -f "$TIER0_STATE_PATH" ]; then
         failure_reason=state_missing
+        failure_key=state_missing
     else
         state_mtime=$($TIER0_STAT -f %m "$TIER0_STATE_PATH" 2>/dev/null) || state_mtime=
         case "$state_mtime" in
             ''|*[!0-9]*)
                 failure_reason=state_mtime_unreadable
+                failure_key=state_mtime_unreadable
                 ;;
             *)
                 if [ "$state_mtime" -gt "$now_epoch" ]; then
                     future_offset=$((state_mtime - now_epoch))
                     failure_reason="state_mtime_future offset=${future_offset}s"
+                    failure_key=state_mtime_future
                 else
                     state_age=$((now_epoch - state_mtime))
                     if [ "$state_age" -ge "$TIER0_STALE_SECONDS" ]; then
                         failure_reason="state_stale age=${state_age}s threshold=${TIER0_STALE_SECONDS}s"
+                        failure_key=state_stale
                     fi
                 fi
                 ;;
         esac
+        if [ -z "$failure_reason" ] && "$TIER0_GREP" -Eq '"slow_check"[[:space:]]*:[[:space:]]*true' "$TIER0_STATE_PATH" 2>/dev/null; then
+            failure_reason=state_slow_check
+            failure_key=state_slow_check
+        fi
     fi
 fi
 
 if [ -z "$failure_reason" ]; then
+    reset_alert_cooldown
     exit 0
 fi
 
-# Detection and all alert attempts intentionally precede every recovery command.
-alert_all_channels "$failure_reason"
+# Detection and all due alert attempts intentionally precede every recovery command.
+if should_alert "$failure_key"; then
+    alert_all_channels "$failure_reason"
+    record_alert "$failure_key" || :
+fi
 
 if [ "$label_loaded" -eq 0 ]; then
     "$TIER0_LAUNCHCTL" bootstrap "$TIER0_DOMAIN" "$TIER0_HEALTH_PLIST_PATH" >/dev/null 2>&1 || :

src/brainlayer/doctor.py

@@ -387,7 +387,7 @@ def _check_brainbar_version_consistency(
 def run_doctor(
     config: DoctorConfig,
     *,
-    ps_output_fn: Callable[[], str] = _default_ps_output,
+    ps_output_fn: Callable[[], str | None] = _default_ps_output,
     command_runner: CommandRunner = _default_command_runner,
     now_fn: Callable[[], datetime] = lambda: datetime.now(UTC),
 ) -> DoctorResult:
@@ -500,10 +500,17 @@ def warning(code: str, message: str, **details: Any) -> None:
             command_runner=command_runner,
         )
 
-    hotlane_processes = parse_hotlane_processes(ps_output_fn())
-    result.hotlane_running = bool(hotlane_processes)
-    if not hotlane_processes:
-        fatal("hotlane_dead", "hot-lane embedding process is not running")
+    ps_output = ps_output_fn()
+    if ps_output is None:
+        fatal(
+            "process_snapshot_failed",
+            "could not read the process table; hot-lane daemon liveness was not evaluated",
+        )
+    else:
+        hotlane_processes = parse_hotlane_processes(ps_output)
+        result.hotlane_running = bool(hotlane_processes)
+        if not hotlane_processes:
+            fatal("hotlane_dead", "hot-lane embedding process is not running")
 
     if result.enrichment_backlog and result.enrichment_backlog > 0:
         warning(