From 0e15acafa0f4809df97e1845cec6320670c0474f Mon Sep 17 00:00:00 2001 From: mhenrixon Date: Wed, 29 Jul 2026 15:16:27 +0200 Subject: [PATCH] feat(service): hold requests for a sleeping service and wake it on demand The Service half of scale-to-zero (#19): the request gate, the options, the persisted state, and the CLI surface. The Router still does not install a container lifecycle, so nothing sleeps yet -- that is the last chunk. The flag is --sleep-after, NOT --idle-timeout. `run --idle-timeout` already exists for HTTP keep-alive (config.go, DefaultIdleTimeout = 60s), and two flags sharing one name with two unrelated meanings is an operator trap no compiler catches. --sleep-after also puts the flag, the persisted state name, and the log lines in one vocabulary. The gem needs to know this: kamal#68 assumed --idle-timeout would be the name. Gate placement is the load-bearing decision. It sits in serviceRequestWithTarget, after handlePausedAndStoppedRequests and before target selection. Every gate above it -- basic auth, IP allow list, rate limit, redirects, request deadline, the ACME HTTP handler -- therefore runs first, so a blocked, throttled, unauthenticated or redirected request can never spend a container start. Upstream #228 put this a layer up in ServeHTTP, where an anonymous client, a rate-limited flood, or an ACME HTTP-01 challenge each start containers. It is also above target selection, so the request handed on is byte-for-byte the one that arrived, body still unread. Health checks are answered, never held and never allowed to wake: an uptime monitor polling /up would pin a service awake forever, and holding the probe would make a load balancer in front evict a service that is sleeping correctly. They report unhealthy once a wake has actually failed, so monitoring stops being told everything is fine while every real request 503s. The 503 body carries none of the wake error. That error holds container references and up to four kilobytes of daemon output, and the response is reachable by anyone who can open a connection, so it is logged instead. Refs are recomputed in UpdateLoadBalancer, not just initialize. initialize runs before a load balancer exists on a first deploy, so refs were empty and a "wake" succeeded having started nothing -- caught by a test asserting the helper actually derived some. A redeploy pointing at new containers lands here too, which is exactly when the controller needs telling. Normalize only defaults a zero wake-timeout. Defaulting anything <= 0 swallowed a negative, making its validation unreachable; a negative is a typo and is now rejected rather than silently turned into 30s. UnmarshalJSON records the restored idle state but never builds a controller: the lifecycle is nil at that point, so one built there could reach StopContainer on a nil interface. SetContainerLifecycle creates it after the whole state file is decoded. A state file written before this feature has no idle_state key, parses to active, and re-marshals byte-identically since every new key is omitempty. Refs #19 --- internal/cmd/deploy.go | 3 + internal/cmd/run.go | 1 + internal/server/config.go | 9 + internal/server/idle_controller.go | 50 +++++ internal/server/service.go | 72 ++++++- internal/server/service_idle.go | 225 +++++++++++++++++++ internal/server/service_idle_test.go | 308 +++++++++++++++++++++++++++ 7 files changed, 666 insertions(+), 2 deletions(-) create mode 100644 internal/server/service_idle.go create mode 100644 internal/server/service_idle_test.go diff --git a/internal/cmd/deploy.go b/internal/cmd/deploy.go index d68f2fe..3162406 100644 --- a/internal/cmd/deploy.go +++ b/internal/cmd/deploy.go @@ -52,6 +52,9 @@ func newDeployCommand() *deployCommand { deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.TLSDomainsSource, "tls-domains-source", "", "Fetch additional TLS domains from this endpoint (path resolved against the service, or absolute URL)") deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.TLSDomainsInterval, "tls-domains-interval", 0, "Interval between domain source polls (default 5m)") deployCommand.cmd.Flags().IntVar(&deployCommand.args.ServiceOptions.TLSDomainsBatchSize, "tls-domains-batch-size", 0, "Dynamic domains to batch per certificate (default 1, max 25)") + deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.SleepAfter, "sleep-after", 0, "Stop this service's target containers after this long with no traffic, and start them again on the next request (default 0, never). Requires the proxy to run with --docker-socket. Health checks and the proxy's own TLS probes are not traffic and never wake a sleeping service") + deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.WakeTimeout, "wake-timeout", server.DefaultWakeTimeout, "Maximum time a request waits for a sleeping service's containers to start and pass a health check before failing with 503") + deployCommand.cmd.Flags().StringArrayVar(&deployCommand.args.ServiceOptions.SleepContainers, "sleep-container", nil, "Container to stop and start for --sleep-after, replacing what the proxy infers from the target address. Needed when a target names a network alias rather than a container (may be specified multiple times)") deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.CanonicalHost, "canonical-host", "", "Redirect all requests to this host (e.g., force root or www)") // StringArray rather than StringSlice: a pattern or a replacement may diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 4f7b940..4c27e2d 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -54,6 +54,7 @@ func newRunCommand() *runCommand { runCommand.cmd.Flags().DurationVar(&globalConfig.ShutdownTimeout, "shutdown-timeout", getEnvDuration("SHUTDOWN_TIMEOUT", server.DefaultShutdownTimeout), "Maximum time to wait for in-flight requests to drain on shutdown") // ACME/TLS configuration + runCommand.cmd.Flags().StringVar(&globalConfig.DockerSocketPath, "docker-socket", getEnvString("DOCKER_SOCKET", ""), "Path to the container runtime socket, enabling --sleep-after on deploy (default empty, disabled). Reaching this socket is root-equivalent on the host, so it is opt-in") runCommand.cmd.Flags().StringVar(&globalConfig.MinTLS, "min-tls", getEnvString("MIN_TLS", server.DefaultMinTLSVersion), "Lowest TLS version the HTTPS listener will negotiate: 1.2 or 1.3 (TLS 1.0 and 1.1 cannot be enabled; HTTP/3 is always 1.3)") runCommand.cmd.Flags().StringVar(&globalConfig.ACMEEmail, "acme-email", getEnvString("ACME_EMAIL", ""), "Email address for ACME account registration (required for automatic TLS)") runCommand.cmd.Flags().StringVar(&globalConfig.ACMEDirectory, "acme-directory", getEnvString("ACME_DIRECTORY", server.LetsEncryptProduction), "ACME directory URL") diff --git a/internal/server/config.go b/internal/server/config.go index 98d32ae..def3657 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -46,6 +46,15 @@ type Config struct { MetricsPort int HTTP3Enabled bool + // DockerSocketPath enables scale-to-zero by giving the proxy a container + // runtime to stop and start with. Empty (the default) leaves the feature + // unavailable, and a deploy asking for --sleep-after is refused rather than + // accepted and silently never acted on. + // + // Reaching this socket is root-equivalent on the host, which is why it is + // opt-in and off by default. + DockerSocketPath string + // MinTLS is the lowest TLS version the HTTPS listener will negotiate, // written as "1.2" or "1.3". Empty means 1.2, which is also Go's own // minimum, so this setting can only ever narrow what the listener accepts - diff --git a/internal/server/idle_controller.go b/internal/server/idle_controller.go index fd97c6a..c394561 100644 --- a/internal/server/idle_controller.go +++ b/internal/server/idle_controller.go @@ -315,6 +315,56 @@ func (c *IdleController) EndRequest() { c.signal() } +// Configure applies a redeploy's settings in place. The controller is created +// once per Service lifetime and reconfigured thereafter, so the request path can +// read s.idleController without a lock. +// +// Changing the container set means the same thing a redeploy does -- the service +// is awake and pointed somewhere new -- so this takes the same generation bump +// Reset does, invalidating any lifecycle goroutine started against the old set. +func (c *IdleController) Configure(sleepAfter, wakeTimeout time.Duration, refs []string) { + if wakeTimeout <= 0 { + wakeTimeout = DefaultWakeTimeout + } + + c.lock.Lock() + changed := !slices.Equal(c.refs, refs) + + c.sleepAfter = sleepAfter + c.wakeTimeout = wakeTimeout + c.refs = slices.Clone(refs) + + var cancel context.CancelFunc + if changed { + c.lastRequest = time.Now() + c.lastErr, c.failures, c.retryAfter = nil, 0, time.Time{} + + cancel = c.cancel + c.cancel = nil + + if c.state == IdleStateActive { + c.generation++ + } else { + c.setStateLocked(IdleStateActive) + } + } + c.lock.Unlock() + + if cancel != nil { + cancel() + } + c.signal() +} + +// LastWakeError reports why the most recent wake failed, so a health check can +// stop claiming a service is fine once it can no longer start. Nil once a wake +// has succeeded. +func (c *IdleController) LastWakeError() error { + c.lock.Lock() + defer c.lock.Unlock() + return c.lastErr +} + func (c *IdleController) WakeTimeout() time.Duration { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/server/service.go b/internal/server/service.go index ebfd254..cca5f9e 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -174,6 +174,17 @@ type ServiceOptions struct { Redirects []PathRule `json:"redirects,omitempty"` Rewrites []PathRule `json:"rewrites,omitempty"` + // SleepAfter stops this service's containers after this long with no traffic, + // starting them again on the next request. Zero (the default) never sleeps. + SleepAfter time.Duration `json:"sleep_after,omitempty"` + // WakeTimeout bounds how long a request is held while those containers start + // and pass a health check. Zero means DefaultWakeTimeout. + WakeTimeout time.Duration `json:"wake_timeout,omitempty"` + // SleepContainers names the containers to stop and start, replacing what the + // proxy infers from the target addresses. Needed when a target names a network + // alias rather than a container. + SleepContainers []string `json:"sleep_containers,omitempty"` + // Compression encodes responses on their way back to the client. A zero // value leaves them alone, which is what every state file written before // this option existed restores to. @@ -188,6 +199,14 @@ func (so *ServiceOptions) Normalize() { so.Hosts = NormalizeHosts(so.Hosts) so.PathPrefixes = NormalizePathPrefixes(so.PathPrefixes) so.Compression.Normalize() + + // The cobra default makes --help honest; this is what gives restored state + // files and direct RPC callers the same value. + // Only zero defaults. A negative is a typo, and silently turning it into 30s + // would hide it -- validateSleep rejects it instead. + if so.SleepAfter > 0 && so.WakeTimeout == 0 { + so.WakeTimeout = DefaultWakeTimeout + } } func (so ServiceOptions) Validate() error { @@ -265,6 +284,10 @@ func (so ServiceOptions) Validate() error { return err } + if err := so.validateSleep(); err != nil { + return err + } + return so.validateDynamicDomains() } @@ -318,6 +341,11 @@ type Service struct { rateLimiter *rateLimiter redirects *pathRuleSet rewrites *pathRuleSet + + lifecycle ContainerLifecycle + idleController *IdleController + restoredIdleState IdleState + statePersister func() } func NewService(name string, options ServiceOptions, targetOptions TargetOptions, sanCertManager *SANCertManager) (*Service, error) { @@ -353,7 +381,13 @@ func (s *Service) SetSANCertManager(manager *SANCertManager) { } func (s *Service) Dispose() { - s.active.Dispose() + if s.idleController != nil { + s.idleController.Close() + } + + if s.active != nil { + s.active.Dispose() + } if s.rollout != nil { s.rollout.Dispose() } @@ -381,6 +415,12 @@ func (s *Service) UpdateLoadBalancer(lb *LoadBalancer, slot TargetSlot) *LoadBal s.active = lb } + // Refs are derived from the targets, so they are only knowable once a load + // balancer is installed -- initialize runs before that on a first deploy. A + // redeploy pointing at new containers reaches here too, which is exactly when + // the controller needs to be told. + s.configureIdleController(s.options) + return replaced } @@ -418,7 +458,12 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type marshalledService struct { - Name string `json:"name"` + Name string `json:"name"` + // IdleState is written as a name rather than an enum's number: the state file + // outlives proxy versions and operators read it. Absent -- every state file + // written before scale-to-zero existed -- parses to active, which is also what + // SleepAfter == 0 produces, so the feature restores off. + IdleState string `json:"idle_state,omitempty"` Options ServiceOptions `json:"options"` TargetOptions TargetOptions `json:"target_options"` ActiveTargets []string `json:"active_targets"` @@ -453,8 +498,14 @@ func (s *Service) MarshalJSON() ([]byte, error) { rolloutReaders = s.rollout.ReadTargets().Specs() } + idleState := IdleStateActive + if s.idleController != nil { + idleState = s.idleController.State() + } + return json.Marshal(marshalledService{ Name: s.name, + IdleState: idleState.String(), ActiveTargets: s.active.WriteTargets().Specs(), ActiveReaders: s.active.ReadTargets().Specs(), RolloutTargets: rolloutTargets, @@ -512,6 +563,11 @@ func (s *Service) UnmarshalJSON(data []byte) error { s.rollout.MarkAllHealthy() } + // Recorded, not acted on: the lifecycle is still nil here, so a controller + // built now could reach StopContainer on a nil interface. SetContainerLifecycle + // is what creates it, after the router has decoded the whole state file. + s.restoredIdleState = ParseIdleState(ms.IdleState) + return s.initialize(ms.Options, ms.TargetOptions) } @@ -584,6 +640,7 @@ func (s *Service) initialize(options ServiceOptions, targetOptions TargetOptions s.rewrites = rewrites s.options = options s.targetOptions = targetOptions + s.configureIdleController(options) s.certManager = certManager s.clientCAs = clientCAs s.middleware = middleware @@ -790,6 +847,17 @@ func (s *Service) serviceRequestWithTarget(w http.ResponseWriter, r *http.Reques return } + // After every gate above, so that no blocked, throttled, unauthenticated or + // redirected request can spend a container start. Before target selection, so + // the request body is still unread when the hold begins. + handled, endIdleRequest := s.handleIdleRequest(w, r) + if endIdleRequest != nil { + defer endIdleRequest() + } + if handled { + return + } + // Last, so that everything above -- the health check exemptions, the // redirects, the allow list -- still sees the path the client asked for. r = s.rewriteRequest(r) diff --git a/internal/server/service_idle.go b/internal/server/service_idle.go new file mode 100644 index 0000000..c24863a --- /dev/null +++ b/internal/server/service_idle.go @@ -0,0 +1,225 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "slices" + "time" +) + +// validateSleep joins the chain in ServiceOptions.Validate. +func (so ServiceOptions) validateSleep() error { + if so.SleepAfter < 0 || so.WakeTimeout < 0 { + return fmt.Errorf("%w: sleep-after and wake-timeout cannot be negative", ErrServiceOptionsInvalid) + } + + if so.SleepAfter > 0 && so.TLSOnDemandURL != "" { + // An on-demand check asks the backend, at handshake time, whether a host + // may have a certificate. A sleeping backend cannot answer, and waking one + // would let any SNI on the internet start a container. + return fmt.Errorf("%w: sleep-after cannot be used with a TLS on-demand URL", ErrServiceOptionsInvalid) + } + + if so.SleepAfter <= 0 && len(so.SleepContainers) > 0 { + return fmt.Errorf("%w: sleep-container requires sleep-after", ErrServiceOptionsInvalid) + } + + return nil +} + +// SetContainerLifecycle installs the runtime that starts and stops this service's +// containers, and builds the idle controller around it. Restored services get +// theirs after the state file is decoded, which is why UnmarshalJSON never +// creates one: at that point the lifecycle is still nil, and a controller built +// there could reach StopContainer on a nil interface. +func (s *Service) SetContainerLifecycle(lifecycle ContainerLifecycle) { + s.lifecycle = lifecycle + s.configureIdleController(s.options) + + if s.idleController == nil || s.idleController.State() != IdleStateSleeping { + return + } + + // Restore assumed every target healthy. For a sleeping service that is a + // healthy pool pointing at a stopped container -- and under + // --recheck-targets-on-restore, a probe against it every second. Put it back + // the way sleeping left it. + s.suspendForSleep() +} + +// configureIdleController writes s.idleController once per Service lifetime and +// reconfigures in place thereafter, so the request path's unlocked read of it is +// safe by construction. +func (s *Service) configureIdleController(options ServiceOptions) { + if options.SleepAfter <= 0 { + if s.idleController != nil { + s.idleController.Configure(0, 0, nil) + } + return + } + + if s.idleController == nil { + if s.lifecycle == nil { + // DeployService refuses sleep-after without a lifecycle, so this is a + // restored service whose lifecycle arrives later from + // SetContainerLifecycle. + return + } + + s.idleController = NewIdleController(IdleControllerConfig{ + Name: s.name, + Lifecycle: s.lifecycle, + Suspend: s.suspendForSleep, + Resume: s.resumeFromSleep, + Persist: s.persistState, + }) + + if s.restoredIdleState == IdleStateSleeping { + s.idleController.RestoreSleeping() + } + } + + s.idleController.Configure(options.SleepAfter, options.WakeTimeout, s.containerRefs(options)) +} + +// containerRefs names the containers behind this service's write targets. Read +// targets are replicas whose lifecycle the proxy does not own, so they are never +// stopped. +func (s *Service) containerRefs(options ServiceOptions) []string { + if len(options.SleepContainers) > 0 { + return slices.Clone(options.SleepContainers) + } + + refs := []string{} + for _, lb := range []*LoadBalancer{s.active, s.rollout} { + if lb == nil { + continue + } + for _, target := range lb.WriteTargets() { + if ref, ok := target.ContainerRef(); ok && !slices.Contains(refs, ref) { + refs = append(refs, ref) + } + } + } + + return refs +} + +func (s *Service) suspendForSleep() { + s.serviceLock.RLock() + active, rollout := s.active, s.rollout + s.serviceLock.RUnlock() + + for _, lb := range []*LoadBalancer{active, rollout} { + if lb != nil { + lb.SuspendForSleep() + } + } +} + +func (s *Service) resumeFromSleep(timeout time.Duration) error { + s.serviceLock.RLock() + active, rollout := s.active, s.rollout + s.serviceLock.RUnlock() + + for _, lb := range []*LoadBalancer{active, rollout} { + if lb != nil { + lb.ResumeFromSleep() + } + } + + if active == nil { + return nil + } + + // Started is not ready: wait for the woken container to actually answer. + return active.WaitUntilHealthy(timeout) +} + +func (s *Service) persistState() { + if s.statePersister != nil { + s.statePersister() + } +} + +// handleIdleRequest holds the request until this service's containers are back. +// It reports whether the client has already been answered, and returns the +// function that releases this request's hold on the idle timer. +func (s *Service) handleIdleRequest(w http.ResponseWriter, r *http.Request) (bool, func()) { + controller := s.idleController + if controller == nil { + return false, nil + } + + // Synthesized inside the proxy -- a TLS on-demand probe -- and must never + // start a container. validateSleep refuses the combination, so this only + // guards a state file written before that rule existed. + if isInternalRequest(r) { + return false, nil + } + + // A health check must never wake a service, or an uptime monitor pins it + // awake forever. It must be answered rather than held, or a load balancer in + // front evicts a service that is sleeping exactly as intended. + if s.targetOptions.IsHealthCheckRequest(r) { + return s.answerIdleHealthCheck(w, r, controller), nil + } + + if err := controller.BeginRequest(r.Context()); err != nil { + if errors.Is(err, context.Canceled) { + // The client hung up mid-wake. Nobody left to answer. + return true, nil + } + + // Logged, not rendered: the error carries container references and up to + // four kilobytes of daemon output, and this response is reachable by + // anyone who can open a connection. + slog.Error("Rejecting request: service did not wake", + "service", s.name, "path", r.URL.Path, "error", err) + w.Header().Set("Retry-After", "1") + SetErrorResponse(w, r, http.StatusServiceUnavailable, nil) + return true, nil + } + + return false, controller.EndRequest +} + +// answerIdleHealthCheck answers for a sleeping service without waking it. It +// reports healthy while the sleep is working as intended, and stops the moment a +// wake has actually failed -- otherwise a service that can no longer start +// reports green to its monitoring forever while 503ing every real request. +func (s *Service) answerIdleHealthCheck(w http.ResponseWriter, r *http.Request, controller *IdleController) bool { + if controller.State() == IdleStateActive { + return false + } + + if err := controller.LastWakeError(); err != nil { + slog.Warn("Reporting unhealthy: last wake failed", "service", s.name, "error", err) + SetErrorResponse(w, r, http.StatusServiceUnavailable, nil) + return true + } + + w.WriteHeader(http.StatusOK) + return true +} + +// ContainerRef returns the container this target's address names: its hostname, +// which is a container short id in a Kamal deployment and a container name under +// Compose. It is a best guess -- a Compose service alias resolves over Docker's +// DNS but names no container -- so --sleep-container exists to state it +// explicitly, and the deploy preflight turns a wrong guess into an error on the +// operator's terminal rather than a 503 an hour later. +// +// An address is rejected outright: no container runtime could act on one. +func (t *Target) ContainerRef() (string, bool) { + host := t.targetURL.Hostname() + if host == "" || net.ParseIP(host) != nil { + return "", false + } + + return host, true +} diff --git a/internal/server/service_idle_test.go b/internal/server/service_idle_test.go new file mode 100644 index 0000000..9c33105 --- /dev/null +++ b/internal/server/service_idle_test.go @@ -0,0 +1,308 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceOptions_ValidateSleep(t *testing.T) { + tests := []struct { + name string + options ServiceOptions + expectedError string + }{ + { + name: "sleep disabled by default", + options: ServiceOptions{Hosts: []string{"app.example.com"}}, + }, + { + name: "sleep enabled", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: time.Minute}, + }, + { + name: "negative sleep-after", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: -time.Second}, + expectedError: "cannot be negative", + }, + { + name: "negative wake-timeout", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: time.Minute, WakeTimeout: -time.Second}, + expectedError: "cannot be negative", + }, + { + // An on-demand check asks the backend at handshake time whether a host + // may have a certificate. A sleeping backend cannot answer, and waking + // one would let any SNI on the internet start a container. + name: "sleep with a TLS on-demand URL", + options: ServiceOptions{TLSEnabled: true, TLSOnDemandURL: "/ask", SleepAfter: time.Minute}, + expectedError: "cannot be used with a TLS on-demand URL", + }, + { + name: "sleep-container without sleep-after", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepContainers: []string{"web-1"}}, + expectedError: "requires sleep-after", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.options.Validate() + + if tt.expectedError == "" { + assert.NoError(t, err) + return + } + + require.ErrorIs(t, err, ErrServiceOptionsInvalid) + assert.ErrorContains(t, err, tt.expectedError) + }) + } +} + +// The cobra default makes --help honest; Normalize is what gives restored state +// files and direct RPC callers the same value. +func TestServiceOptions_NormalizeDefaultsTheWakeTimeout(t *testing.T) { + options := ServiceOptions{SleepAfter: time.Minute} + options.Normalize() + assert.Equal(t, DefaultWakeTimeout, options.WakeTimeout) + + explicit := ServiceOptions{SleepAfter: time.Minute, WakeTimeout: 5 * time.Second} + explicit.Normalize() + assert.Equal(t, 5*time.Second, explicit.WakeTimeout) + + // Nothing is defaulted for a service that does not sleep. + off := ServiceOptions{} + off.Normalize() + assert.Zero(t, off.WakeTimeout) +} + +func testSleepingService(t *testing.T, lifecycle ContainerLifecycle, handler http.HandlerFunc) *Service { + t.Helper() + + backend := httptest.NewServer(handler) + t.Cleanup(backend.Close) + + options := defaultServiceOptions + options.Hosts = []string{"app.example.com"} + options.SleepAfter = time.Hour // long: these tests drive the controller directly + // A httptest backend listens on 127.0.0.1, which ContainerRef rightly refuses + // as an address rather than a container -- so name it explicitly, which is + // exactly what --sleep-container exists for. + options.SleepContainers = []string{"web-1"} + + service, err := NewService("sleepy", options, defaultTargetOptions, nil) + require.NoError(t, err) + t.Cleanup(service.Dispose) + + targets, err := NewTargetList([]string{backend.Listener.Addr().String()}, []string{}, defaultTargetOptions) + require.NoError(t, err) + + lb := NewLoadBalancer(targets, DefaultWriterAffinityTimeout, false) + t.Cleanup(lb.Dispose) + require.NoError(t, lb.WaitUntilHealthy(5*time.Second)) + + service.SetContainerLifecycle(lifecycle) + service.UpdateLoadBalancer(lb, TargetSlotActive) + + require.NotNil(t, service.idleController, "a service with --sleep-after gets a controller") + require.NotEmpty(t, service.containerRefs(service.options), + "refs must be derived once the load balancer is installed, or a wake starts nothing") + + return service +} + +func sendServiceRequest(service *Service, path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, "http://app.example.com"+path, nil) + recorder := httptest.NewRecorder() + service.ServeHTTP(recorder, req) + return recorder +} + +// A sleeping service wakes on a real request and forwards it, body intact. +func TestService_SleepingServiceWakesOnARequest(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + require.Equal(t, IdleStateSleeping, service.idleController.State()) + + recorder := sendServiceRequest(service, "/") + + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Equal(t, int64(1), lifecycle.starts.Load(), "the request woke the containers") + assert.Equal(t, IdleStateActive, service.idleController.State()) +} + +// An uptime monitor polling /up would otherwise pin a service awake forever. +func TestService_HealthCheckNeitherWakesNorIsHeld(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + + recorder := sendServiceRequest(service, defaultTargetOptions.HealthCheckConfig.Path) + + assert.Equal(t, http.StatusOK, recorder.Code, "a sleeping service is healthy, not down") + assert.Zero(t, lifecycle.starts.Load(), "monitoring must never start a container") + assert.Equal(t, IdleStateSleeping, service.idleController.State()) +} + +// The other half: once a wake has actually failed, monitoring must stop being +// told everything is fine while every real request 503s. +func TestService_HealthCheckReportsUnhealthyAfterAFailedWake(t *testing.T) { + lifecycle := &fakeLifecycle{ + startFunc: func(ctx context.Context, ref string) error { return errors.New("no such container") }, + } + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + + // A real request tries to wake and fails. + failed := sendServiceRequest(service, "/") + require.Equal(t, http.StatusServiceUnavailable, failed.Code) + + health := sendServiceRequest(service, defaultTargetOptions.HealthCheckConfig.Path) + assert.Equal(t, http.StatusServiceUnavailable, health.Code, + "a service that can no longer start must not report green to its monitoring") +} + +// The wake error carries container references and up to 4 KB of daemon output, +// and this response is reachable by anyone who can open a connection. +func TestService_WakeFailureDoesNotLeakDaemonDetailToTheClient(t *testing.T) { + lifecycle := &fakeLifecycle{ + startFunc: func(ctx context.Context, ref string) error { + return errors.New("secret-container-name: permission denied on /var/run/docker.sock") + }, + } + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) {}) + service.idleController.RestoreSleeping() + + recorder := sendServiceRequest(service, "/") + + require.Equal(t, http.StatusServiceUnavailable, recorder.Code) + assert.NotContains(t, recorder.Body.String(), "secret-container-name") + assert.NotContains(t, recorder.Body.String(), "docker.sock") + assert.Equal(t, "1", recorder.Header().Get("Retry-After")) +} + +// The proxy's own TLS on-demand probe is synthesized internally and must never +// spend a container start. +func TestService_InternalRequestsDoNotWake(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + service.idleController.RestoreSleeping() + + req := httptest.NewRequest(http.MethodGet, "http://app.example.com/", nil) + req = req.WithContext(markInternalRequest(req.Context())) + recorder := httptest.NewRecorder() + service.ServeHTTP(recorder, req) + + assert.Zero(t, lifecycle.starts.Load(), "an internal probe must never start a container") +} + +func TestService_ContainerRefsPreferTheExplicitOverride(t *testing.T) { + options := defaultServiceOptions + options.Hosts = []string{"app.example.com"} + options.SleepAfter = time.Hour + options.SleepContainers = []string{"explicit-1", "explicit-2"} + + service, err := NewService("refs", options, defaultTargetOptions, nil) + require.NoError(t, err) + t.Cleanup(service.Dispose) + + assert.Equal(t, []string{"explicit-1", "explicit-2"}, service.containerRefs(options), + "--sleep-container replaces inference entirely") +} + +func TestTarget_ContainerRef(t *testing.T) { + tests := []struct { + address string + expected string + ok bool + }{ + {address: "web-1", expected: "web-1", ok: true}, + {address: "web-1:3000", expected: "web-1", ok: true}, + {address: "3f2a1b9c4d5e:3000", expected: "3f2a1b9c4d5e", ok: true}, + + // An address names no container, and no runtime could ever act on it. + {address: "10.0.0.5:3000", ok: false}, + {address: "127.0.0.1:80", ok: false}, + } + + for _, tt := range tests { + t.Run(tt.address, func(t *testing.T) { + target, err := NewTarget(tt.address, defaultTargetOptions) + require.NoError(t, err) + + ref, ok := target.ContainerRef() + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.Equal(t, tt.expected, ref) + } + }) + } +} + +// Old state files predate the field and must restore with the feature off. +func TestService_IdleStateRoundTrip(t *testing.T) { + assert.Equal(t, IdleStateActive, ParseIdleState("")) + + service := testSleepingService(t, &fakeLifecycle{}, func(w http.ResponseWriter, r *http.Request) {}) + service.idleController.RestoreSleeping() + + encoded, err := service.MarshalJSON() + require.NoError(t, err) + assert.Contains(t, string(encoded), `"idle_state":"sleeping"`) + + var restored Service + require.NoError(t, restored.UnmarshalJSON(encoded)) + t.Cleanup(restored.Dispose) + + assert.Equal(t, time.Hour, restored.options.SleepAfter) + assert.Equal(t, IdleStateSleeping, restored.restoredIdleState) + assert.Nil(t, restored.idleController, + "UnmarshalJSON must not build a controller: the lifecycle is still nil here") +} + +// A state file written before this feature existed restores with sleep off and +// re-marshals byte-identically, because every new key is omitempty. +func TestService_StateFileWithoutIdleStateRestoresAwake(t *testing.T) { + targetOptions, err := json.Marshal(defaultTargetOptions) + require.NoError(t, err) + + // No idle_state key, and no sleep_after in options -- exactly what every + // state file written before this feature looks like. + legacy := fmt.Sprintf( + `{"name":"old","options":{"hosts":["app.example.com"]},"target_options":%s,"active_targets":["web-1:3000"]}`, + targetOptions) + + var restored Service + require.NoError(t, restored.UnmarshalJSON([]byte(legacy))) + t.Cleanup(restored.Dispose) + + assert.Zero(t, restored.options.SleepAfter) + assert.Equal(t, IdleStateActive, restored.restoredIdleState) + assert.Nil(t, restored.idleController) +}