fix(serve): degrade large watcher trees instead of failing startup (#443)

## Summary

- Cap recursive file watcher setup at an internal process-wide budget of 8192 directories. When the budget is exhausted or `fsnotify.Add` returns `EMFILE`/`ENOSPC`, the affected root degrades to the existing periodic polling path instead of bubbling up as a fatal startup error.
- Defer file watcher startup until after the backend listener is bound, the managed Caddy proxy is reachable, and the state file/startup lock are published. Slow recursive watch setup can no longer block the web UI from coming up or stall daemon discovery.
- Report polled roots after the existing `Watching N directories` line so the operator can tell when degradation is in effect.

Fixes #364.

Co-authored-by: Wes McKinney <wesm@users.noreply.github.com>

Author: wesm

cmd/agentsview/main.go

@@ -32,6 +32,7 @@ const (
 	periodicSyncInterval  = 15 * time.Minute
 	unwatchedPollInterval = 2 * time.Minute
 	watcherDebounce       = 500 * time.Millisecond
+	recursiveWatchBudget  = 8192
 )
 
 func main() {
@@ -142,9 +143,6 @@ func runServe(cfg config.Config) {
 			return
 		}
 
-		stopWatcher, unwatchedDirs := startFileWatcher(cfg, engine)
-		defer stopWatcher()
-
 		// Backfill runs in the background. On a large DB (e.g.
 		// after copying tens of thousands of orphaned sessions
 		// during a resync), walking every row to recompute
@@ -164,9 +162,6 @@ func runServe(cfg config.Config) {
 		}()
 
 		go startPeriodicSync(engine, database)
-		if len(unwatchedDirs) > 0 {
-			go startUnwatchedPoll(engine)
-		}
 	}
 
 	// Seed model_pricing after any resync swap so the new DB
@@ -249,6 +244,14 @@ func runServe(cfg config.Config) {
 	}
 	fmt.Printf("Database: %s\n", cfg.DBPath)
 
+	if engine != nil {
+		stopWatcher, unwatchedDirs := startFileWatcher(cfg, engine)
+		defer stopWatcher()
+		if len(unwatchedDirs) > 0 {
+			go startUnwatchedPoll(engine)
+		}
+	}
+
 	if err := waitForServerRuntime(ctx, srv, rt); err != nil {
 		fatal("%v", err)
 	}
@@ -498,6 +501,7 @@ func startFileWatcher(
 
 	var totalWatched int
 	var shallowWatched int
+	remaining := recursiveWatchBudget
 	for _, r := range roots {
 		if r.shallow {
 			if watcher.WatchShallow(r.root) {
@@ -508,14 +512,19 @@ func startFileWatcher(
 			}
 			continue
 		}
-		watched, uw, _ := watcher.WatchRecursive(r.root)
-		totalWatched += watched
-		if uw > 0 {
+		result := watcher.WatchRecursiveBudgeted(r.root, remaining)
+		totalWatched += result.Watched
+		remaining -= result.Watched
+		if result.Unwatched > 0 || result.BudgetExhausted ||
+			result.ResourceExhausted || result.Err != nil {
 			unwatchedDirs = append(unwatchedDirs, r.dir)
 			log.Printf(
 				"Couldn't watch %d directories under %s, will poll every %s",
-				uw, r.dir, unwatchedPollInterval,
+				result.Unwatched, r.dir, unwatchedPollInterval,
 			)
+			if result.Err != nil {
+				log.Printf("watching %s: %v", r.dir, result.Err)
+			}
 		}
 	}
 
@@ -530,6 +539,12 @@ func startFileWatcher(
 			totalWatched, time.Since(t).Round(time.Millisecond),
 		)
 	}
+	if len(unwatchedDirs) > 0 {
+		fmt.Printf(
+			"Polling %d roots every %s for changes\n",
+			len(unwatchedDirs), unwatchedPollInterval,
+		)
+	}
 	watcher.Start()
 	return watcher.Stop, unwatchedDirs
 }

internal/sync/watcher.go

@@ -1,19 +1,31 @@
 package sync
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
 )
 
+type RecursiveWatchResult struct {
+	Watched             int
+	Unwatched           int
+	Err                 error
+	BudgetExhausted     bool
+	ResourceExhausted   bool
+	ResourceExhaustedAt string
+}
+
 // Watcher uses fsnotify to watch session directories for changes
 // and triggers a callback with debouncing.
 type Watcher struct {
@@ -60,27 +72,58 @@ func NewWatcher(debounce time.Duration, onChange func(paths []string), excludes
 // subdirectories to the watch list. Returns the number
 // of directories watched and unwatched (failed to add).
 func (w *Watcher) WatchRecursive(root string) (watched int, unwatched int, err error) {
+	result := w.WatchRecursiveBudgeted(root, math.MaxInt)
+	return result.Watched, result.Unwatched, result.Err
+}
+
+// WatchRecursiveBudgeted walks a directory tree and adds at most
+// budget subdirectories to the watch list. The walk stops as soon
+// as the budget is exhausted or fsnotify reports resource
+// exhaustion, so the caller can degrade the rest of the tree to
+// polling without continuing to traverse it.
+func (w *Watcher) WatchRecursiveBudgeted(root string, budget int) RecursiveWatchResult {
+	var result RecursiveWatchResult
 	root = filepath.Clean(root)
 	w.addRoot(root)
-	err = filepath.WalkDir(root,
+
+	remaining := budget
+	result.Err = filepath.WalkDir(root,
 		func(path string, d fs.DirEntry, err error) error {
 			if err != nil {
 				return nil // skip inaccessible dirs
 			}
-			if d.IsDir() {
-				// Skip entire excluded subtrees, but always keep the root.
-				if path != root && w.shouldExcludeForRoot(path, root) {
-					return filepath.SkipDir
-				}
-				if addErr := w.watcher.Add(path); addErr != nil {
-					unwatched++
-				} else {
-					watched++
+			if !d.IsDir() {
+				return nil
+			}
+			// Skip entire excluded subtrees, but always keep the root.
+			if path != root && w.shouldExcludeForRoot(path, root) {
+				return filepath.SkipDir
+			}
+			if remaining <= 0 {
+				result.BudgetExhausted = true
+				return filepath.SkipAll
+			}
+			if addErr := w.watcher.Add(path); addErr != nil {
+				result.Unwatched++
+				if isWatchResourceExhaustion(addErr) {
+					result.ResourceExhausted = true
+					result.ResourceExhaustedAt = path
+					return filepath.SkipAll
 				}
+				return nil
 			}
+			remaining--
+			result.Watched++
 			return nil
 		})
-	return watched, unwatched, err
+	if errors.Is(result.Err, filepath.SkipAll) {
+		result.Err = nil
+	}
+	return result
+}
+
+func isWatchResourceExhaustion(err error) bool {
+	return errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENOSPC)
 }
 
 // WatchShallow adds only the root directory to the watch list,

internal/sync/watcher_test.go

@@ -2,10 +2,12 @@ package sync
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -351,6 +353,42 @@ func TestWatchRecursive_ExcludesDirectoryNames(t *testing.T) {
 	}
 }
 
+func TestWatchRecursiveBudget_DegradesWhenBudgetExhausted(t *testing.T) {
+	root := t.TempDir()
+	for i := range 5 {
+		if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("dir-%d", i)), 0o755); err != nil {
+			t.Fatalf("MkdirAll: %v", err)
+		}
+	}
+
+	w, err := NewWatcher(time.Second, func(_ []string) {}, nil)
+	if err != nil {
+		t.Fatalf("NewWatcher: %v", err)
+	}
+	w.Start()
+	t.Cleanup(func() { w.Stop() })
+
+	result := w.WatchRecursiveBudgeted(root, 3)
+	if result.Watched != 3 {
+		t.Fatalf("Watched = %d, want 3", result.Watched)
+	}
+	if !result.BudgetExhausted {
+		t.Fatal("BudgetExhausted = false, want true")
+	}
+}
+
+func TestIsWatchResourceExhaustion(t *testing.T) {
+	if !isWatchResourceExhaustion(syscall.EMFILE) {
+		t.Fatal("EMFILE should be resource exhaustion")
+	}
+	if !isWatchResourceExhaustion(syscall.ENOSPC) {
+		t.Fatal("ENOSPC should be resource exhaustion")
+	}
+	if isWatchResourceExhaustion(os.ErrNotExist) {
+		t.Fatal("ErrNotExist should not be resource exhaustion")
+	}
+}
+
 func TestWatcherAutoWatchesNewDirs_RespectsExcludes(t *testing.T) {
 	pathsCh := make(chan []string, 10)
 	w, err := NewWatcher(20*time.Millisecond, func(paths []string) {