From 3999e7fc2d9b35fb2516b859a6c9caec122a4c24 Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Tue, 14 Jul 2026 04:47:32 +0000 Subject: [PATCH 1/3] fix(daemon): raise proxy daemon file-descriptor limit at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy daemon is the single shared egress point for every run. It inherited the host's default RLIMIT_NOFILE soft limit (typically 256 on macOS), so a burst of concurrent connections through the one proxy — e.g. `bun install` opens up to 64 parallel connections — could exhaust its file descriptors and stall every run's traffic until the burst cleared. This surfaced as all Claude sessions inside moat freezing at once on macOS during a package install, then recovering gracefully. Raise RLIMIT_NOFILE toward 65536 (capped at the hard limit) at daemon startup, before the proxy accepts connections. Cross-platform via x/sys/unix with a macOS step-down fallback (macOS rejects a soft limit above kern.maxfilesperproc even when the hard limit is higher). Best-effort: a failure logs a warning and the daemon still runs. --- cmd/moat/cli/daemon.go | 13 +++++++ internal/daemon/filelimit.go | 57 +++++++++++++++++++++++++++ internal/daemon/filelimit_other.go | 10 +++++ internal/daemon/filelimit_test.go | 62 ++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 internal/daemon/filelimit.go create mode 100644 internal/daemon/filelimit_other.go create mode 100644 internal/daemon/filelimit_test.go diff --git a/cmd/moat/cli/daemon.go b/cmd/moat/cli/daemon.go index 96c5e71d..2d80cda8 100644 --- a/cmd/moat/cli/daemon.go +++ b/cmd/moat/cli/daemon.go @@ -44,6 +44,19 @@ func runDaemon(_ *cobra.Command, _ []string) error { daemonDir = filepath.Join(config.GlobalConfigDir(), "proxy") } + // Raise the file-descriptor limit before the proxy starts accepting. The + // daemon is the single shared egress proxy for every run, so a burst of + // concurrent connections from one container (e.g. `bun install` opens up to + // 64 parallel connections) must not exhaust its FDs and stall every other + // run's traffic. Best-effort: a failure is non-fatal. + if oldSoft, newSoft, ferr := daemon.RaiseFileLimit(); ferr != nil { + log.Warn("could not raise daemon file-descriptor limit", "error", ferr, "soft", oldSoft) + } else if newSoft > oldSoft { + log.Info("raised daemon file-descriptor limit", "from", oldSoft, "to", newSoft) + } else { + log.Debug("daemon file-descriptor limit already sufficient", "soft", oldSoft) + } + // Expose build version to daemon package so the health endpoint can // report it. This allows detecting version skew between daemon and CLI. daemon.BuildCommit = commit diff --git a/internal/daemon/filelimit.go b/internal/daemon/filelimit.go new file mode 100644 index 00000000..00fa3630 --- /dev/null +++ b/internal/daemon/filelimit.go @@ -0,0 +1,57 @@ +//go:build unix + +package daemon + +import "golang.org/x/sys/unix" + +// desiredNofile is the soft RLIMIT_NOFILE the proxy daemon targets at startup. +// +// The daemon is the single shared egress proxy for every active run, so a burst +// of concurrent connections from one container must not exhaust its file +// descriptors and stall every other run's traffic. A package install is the +// common trigger: `bun install` opens up to 64 parallel connections (Bun's +// default since v1.1.33), each consuming FDs on both the client and upstream +// sides, and several runs can install at once. The frequent 1024 soft default +// is easily exhausted by that; 65536 gives ample headroom. +const desiredNofile = 65536 + +// RaiseFileLimit raises this process's RLIMIT_NOFILE soft limit toward +// desiredNofile, capped at the hard limit. It returns the previous and new soft +// limits. It is a no-op (returns equal values) when the soft limit is already +// at or above the target. +// +// Best-effort: callers treat a returned error as non-fatal (the daemon still +// runs, just with the inherited limit). The step-down fallback handles macOS, +// which rejects a soft limit above kern.maxfilesperproc even when the hard +// limit is nominally higher. +func RaiseFileLimit() (oldSoft, newSoft uint64, err error) { + var lim unix.Rlimit + if err = unix.Getrlimit(unix.RLIMIT_NOFILE, &lim); err != nil { + return 0, 0, err + } + oldSoft = lim.Cur + + target := uint64(desiredNofile) + if lim.Max < target { // the hard limit caps the soft limit + target = lim.Max + } + if oldSoft >= target { + return oldSoft, oldSoft, nil // already sufficient + } + + lim.Cur = target + if err = unix.Setrlimit(unix.RLIMIT_NOFILE, &lim); err == nil { + return oldSoft, target, nil + } + + // Fallback: some systems (notably macOS) reject a soft limit above + // kern.maxfilesperproc even when the hard limit is higher. Step the target + // down until one sticks or we drop back to the old soft limit. + for t := target / 2; t > oldSoft; t /= 2 { + lim.Cur = t + if setErr := unix.Setrlimit(unix.RLIMIT_NOFILE, &lim); setErr == nil { + return oldSoft, t, nil + } + } + return oldSoft, oldSoft, err +} diff --git a/internal/daemon/filelimit_other.go b/internal/daemon/filelimit_other.go new file mode 100644 index 00000000..ffc4237a --- /dev/null +++ b/internal/daemon/filelimit_other.go @@ -0,0 +1,10 @@ +//go:build !unix + +package daemon + +// RaiseFileLimit is a no-op on platforms without POSIX rlimits. The proxy +// daemon runs on the host (macOS or Linux, both unix), so this stub exists only +// to keep the package building on other GOOS values. +func RaiseFileLimit() (oldSoft, newSoft uint64, err error) { + return 0, 0, nil +} diff --git a/internal/daemon/filelimit_test.go b/internal/daemon/filelimit_test.go new file mode 100644 index 00000000..2ba4098e --- /dev/null +++ b/internal/daemon/filelimit_test.go @@ -0,0 +1,62 @@ +//go:build unix + +package daemon + +import ( + "testing" + + "golang.org/x/sys/unix" +) + +func currentNofile(t *testing.T) (soft, hard uint64) { + t.Helper() + var lim unix.Rlimit + if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &lim); err != nil { + t.Fatalf("Getrlimit: %v", err) + } + return lim.Cur, lim.Max +} + +func TestRaiseFileLimit(t *testing.T) { + beforeSoft, hard := currentNofile(t) + + oldSoft, newSoft, err := RaiseFileLimit() + if err != nil { + t.Fatalf("RaiseFileLimit: %v", err) + } + if oldSoft != beforeSoft { + t.Errorf("reported oldSoft %d, want the current soft limit %d", oldSoft, beforeSoft) + } + if newSoft < oldSoft { + t.Errorf("newSoft %d must never be below oldSoft %d (the limit is only ever raised)", newSoft, oldSoft) + } + if newSoft > hard { + t.Errorf("newSoft %d must not exceed the hard limit %d", newSoft, hard) + } + // The process's actual soft limit must reflect the reported new value. + gotSoft, _ := currentNofile(t) + if gotSoft != newSoft { + t.Errorf("process soft limit is %d, want the reported newSoft %d", gotSoft, newSoft) + } +} + +// TestRaiseFileLimitNonLowering asserts a second call never reduces the limit — +// it either reaches the target once and no-ops, or (on systems that cap the soft +// limit below the hard limit, e.g. macOS) stays put. It deliberately does not +// require err==nil on the second call, since a system already at its effective +// ceiling legitimately reports that it could not climb further. +func TestRaiseFileLimitNonLowering(t *testing.T) { + _, firstNew, err := RaiseFileLimit() + if err != nil { + t.Fatalf("first RaiseFileLimit: %v", err) + } + + _, secondNew, _ := RaiseFileLimit() + if secondNew < firstNew { + t.Errorf("second call lowered the soft limit: %d -> %d", firstNew, secondNew) + } + afterSoft, _ := currentNofile(t) + if afterSoft < firstNew { + t.Errorf("process soft limit dropped after a second call: %d -> %d", firstNew, afterSoft) + } +} From 3425cdc55340d0afd472f1ae935f6a5404307616 Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Tue, 14 Jul 2026 04:48:24 +0000 Subject: [PATCH 2/3] docs(changelog): add entry for daemon FD-limit fix (#439) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e8f16b..304e29c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Adds HTTP request-body inspection to Keep policies. File- and pack-based `networ ### Fixed +- Fix all Claude sessions inside moat freezing at once on macOS during a package install — previously, the shared credential-injecting proxy daemon inherited the host's default `RLIMIT_NOFILE` soft limit (typically 256 on macOS), so a burst of concurrent connections through the single proxy (`bun install` opens up to 64 parallel connections) could exhaust its file descriptors and stall every run's `claude.ai` traffic until the burst cleared, then recover. The daemon now raises its file-descriptor soft limit toward 65536 (capped at the hard limit) at startup, before the proxy accepts connections. ([#439](https://github.com/majorcontext/moat/pull/439)) - Fix the injected agent context misrepresenting network access — previously, the "Moat Environment" instructions file (CLAUDE.md/AGENTS.md) listed grant/rule hosts under "Allowed hosts" regardless of policy, so under the default `permissive` policy (where all outbound traffic is allowed) it read as an egress allowlist restricting the agent to those few hosts. The Network Policy section is now policy-aware: `permissive` states that all outbound access is allowed (surfacing only explicit per-path rules, which still apply), and `strict` is described as the allowlist it actually is. The context also now reflects **Docker/DIND availability**, the resolved **workspace mode** (bind vs. ephemeral `volume`), and **installed tool dependencies** — all previously omitted, which could lead an agent to assume capabilities were absent. ([#431](https://github.com/majorcontext/moat/pull/431)) - Fix `moat logs -f` silently doing nothing — previously, follow mode printed a debug-log line ("not yet implemented", only visible with `--verbose`) and exited 0 as if it had streamed, so `-f` looked like it worked. moat now prints a visible notice that follow mode isn't supported yet and shows the current logs. ([#413](https://github.com/majorcontext/moat/pull/413)) - Fix non-deterministic Dockerfile generation causing spurious image rebuilds — previously, dependency `ENV` lines were emitted in random map order, so the generated Dockerfile changed between runs and missed Docker's layer cache. `ENV` keys are now sorted. ([#413](https://github.com/majorcontext/moat/pull/413)) From 508bf35e470be44df0d6856a0dd217b27e95dbb8 Mon Sep 17 00:00:00 2001 From: Dan Pupius Date: Tue, 14 Jul 2026 05:20:57 +0000 Subject: [PATCH 3/3] fix(daemon): binary-search the FD-limit fallback for elevated soft limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on #439. The step-down fallback started at target/2, so if the process already had a soft limit above half the target and the full target was rejected, the loop body never ran and no intermediate value was probed (leaving the limit unraised even when a value between oldSoft and target would have been accepted). Replace it with a binary search over (oldSoft, target] for the highest value the system accepts, and treat a limit that cannot be raised further as success rather than a spurious error — avoiding a misleading "could not raise" warning at startup when the existing limit is already adequate. --- cmd/moat/cli/daemon.go | 4 +-- internal/daemon/filelimit.go | 46 +++++++++++++++++++------------ internal/daemon/filelimit_test.go | 12 ++++---- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/cmd/moat/cli/daemon.go b/cmd/moat/cli/daemon.go index 2d80cda8..3a6a450d 100644 --- a/cmd/moat/cli/daemon.go +++ b/cmd/moat/cli/daemon.go @@ -50,11 +50,11 @@ func runDaemon(_ *cobra.Command, _ []string) error { // 64 parallel connections) must not exhaust its FDs and stall every other // run's traffic. Best-effort: a failure is non-fatal. if oldSoft, newSoft, ferr := daemon.RaiseFileLimit(); ferr != nil { - log.Warn("could not raise daemon file-descriptor limit", "error", ferr, "soft", oldSoft) + log.Warn("could not read daemon file-descriptor limit", "error", ferr) } else if newSoft > oldSoft { log.Info("raised daemon file-descriptor limit", "from", oldSoft, "to", newSoft) } else { - log.Debug("daemon file-descriptor limit already sufficient", "soft", oldSoft) + log.Debug("daemon file-descriptor limit unchanged", "soft", oldSoft) } // Expose build version to daemon package so the health endpoint can diff --git a/internal/daemon/filelimit.go b/internal/daemon/filelimit.go index 00fa3630..9417ffb6 100644 --- a/internal/daemon/filelimit.go +++ b/internal/daemon/filelimit.go @@ -12,18 +12,21 @@ import "golang.org/x/sys/unix" // common trigger: `bun install` opens up to 64 parallel connections (Bun's // default since v1.1.33), each consuming FDs on both the client and upstream // sides, and several runs can install at once. The frequent 1024 soft default -// is easily exhausted by that; 65536 gives ample headroom. +// (and macOS's 256) is easily exhausted by that; 65536 gives ample headroom. const desiredNofile = 65536 // RaiseFileLimit raises this process's RLIMIT_NOFILE soft limit toward -// desiredNofile, capped at the hard limit. It returns the previous and new soft -// limits. It is a no-op (returns equal values) when the soft limit is already -// at or above the target. +// desiredNofile, capped at the hard limit. It returns the previous soft limit +// and the soft limit in effect afterward (always >= oldSoft). It is a no-op +// (equal values) when the soft limit is already at or above the target. // -// Best-effort: callers treat a returned error as non-fatal (the daemon still -// runs, just with the inherited limit). The step-down fallback handles macOS, -// which rejects a soft limit above kern.maxfilesperproc even when the hard -// limit is nominally higher. +// A limit that cannot be raised is not an error — the process keeps its +// existing, already-in-effect limit and err stays nil. err is non-nil only when +// the current limit cannot be read. Some systems, notably macOS, reject a soft +// limit above kern.maxfilesperproc even when the hard limit is nominally +// higher; when the full target is rejected, the fallback binary-searches the +// highest value the system will accept between the old soft limit and the +// target. func RaiseFileLimit() (oldSoft, newSoft uint64, err error) { var lim unix.Rlimit if err = unix.Getrlimit(unix.RLIMIT_NOFILE, &lim); err != nil { @@ -39,19 +42,28 @@ func RaiseFileLimit() (oldSoft, newSoft uint64, err error) { return oldSoft, oldSoft, nil // already sufficient } + // Fast path: the full target is usually accepted. lim.Cur = target - if err = unix.Setrlimit(unix.RLIMIT_NOFILE, &lim); err == nil { + if unix.Setrlimit(unix.RLIMIT_NOFILE, &lim) == nil { return oldSoft, target, nil } - // Fallback: some systems (notably macOS) reject a soft limit above - // kern.maxfilesperproc even when the hard limit is higher. Step the target - // down until one sticks or we drop back to the old soft limit. - for t := target / 2; t > oldSoft; t /= 2 { - lim.Cur = t - if setErr := unix.Setrlimit(unix.RLIMIT_NOFILE, &lim); setErr == nil { - return oldSoft, t, nil + // Fallback: the target was rejected (e.g. macOS caps the soft limit at + // kern.maxfilesperproc, below the hard limit). Binary-search the highest + // value in (oldSoft, target) the system accepts. lo is always known-good + // (oldSoft is currently in effect), hi is always known-bad (a rejected + // value). Each successful Setrlimit leaves the process at that value, so + // once the search narrows, the process's soft limit equals best. + best := oldSoft + lo, hi := oldSoft, target + for hi-lo > 1 { + mid := lo + (hi-lo)/2 + lim.Cur = mid + if unix.Setrlimit(unix.RLIMIT_NOFILE, &lim) == nil { + best, lo = mid, mid + } else { + hi = mid } } - return oldSoft, oldSoft, err + return oldSoft, best, nil } diff --git a/internal/daemon/filelimit_test.go b/internal/daemon/filelimit_test.go index 2ba4098e..8a903bd2 100644 --- a/internal/daemon/filelimit_test.go +++ b/internal/daemon/filelimit_test.go @@ -41,17 +41,19 @@ func TestRaiseFileLimit(t *testing.T) { } // TestRaiseFileLimitNonLowering asserts a second call never reduces the limit — -// it either reaches the target once and no-ops, or (on systems that cap the soft -// limit below the hard limit, e.g. macOS) stays put. It deliberately does not -// require err==nil on the second call, since a system already at its effective -// ceiling legitimately reports that it could not climb further. +// it either reaches the target once and no-ops, or (on systems that cap the +// soft limit below the hard limit, e.g. macOS) stays put. A limit that cannot +// be raised further is reported as success, not an error. func TestRaiseFileLimitNonLowering(t *testing.T) { _, firstNew, err := RaiseFileLimit() if err != nil { t.Fatalf("first RaiseFileLimit: %v", err) } - _, secondNew, _ := RaiseFileLimit() + _, secondNew, err := RaiseFileLimit() + if err != nil { + t.Fatalf("second RaiseFileLimit: %v", err) + } if secondNew < firstNew { t.Errorf("second call lowered the soft limit: %d -> %d", firstNew, secondNew) }