diff --git a/internal/cmd/deploy.go b/internal/cmd/deploy.go index b7fa221..78c6d7a 100644 --- a/internal/cmd/deploy.go +++ b/internal/cmd/deploy.go @@ -52,6 +52,9 @@ func newDeployCommand() *deployCommand { deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.TLSDomainsSource, "tls-domains-source", "", "Fetch additional TLS domains from this endpoint (path resolved against the service, or absolute URL)") deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.TLSDomainsInterval, "tls-domains-interval", 0, "Interval between domain source polls (default 5m)") deployCommand.cmd.Flags().IntVar(&deployCommand.args.ServiceOptions.TLSDomainsBatchSize, "tls-domains-batch-size", 0, "Dynamic domains to batch per certificate (default 1, max 25)") + deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.SleepAfter, "sleep-after", 0, "Stop this service's target containers after this long with no traffic, and start them again on the next request (default 0, never). Requires the proxy to run with --docker-socket. Health checks and the proxy's own TLS probes are not traffic and never wake a sleeping service") + deployCommand.cmd.Flags().DurationVar(&deployCommand.args.ServiceOptions.WakeTimeout, "wake-timeout", server.DefaultWakeTimeout, "Maximum time a request waits for a sleeping service's containers to start and pass a health check before failing with 503") + deployCommand.cmd.Flags().StringArrayVar(&deployCommand.args.ServiceOptions.SleepContainers, "sleep-container", nil, "Container to stop and start for --sleep-after, replacing what the proxy infers from the target address. Needed when a target names a network alias rather than a container (may be specified multiple times)") deployCommand.cmd.Flags().StringVar(&deployCommand.args.ServiceOptions.CanonicalHost, "canonical-host", "", "Redirect all requests to this host (e.g., force root or www)") // StringArray rather than StringSlice: a pattern or a replacement may diff --git a/internal/cmd/run.go b/internal/cmd/run.go index bfa245e..0029e8a 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -60,6 +60,7 @@ func newRunCommand() *runCommand { runCommand.cmd.Flags().DurationVar(&globalConfig.ShutdownTimeout, "shutdown-timeout", getEnvDuration("SHUTDOWN_TIMEOUT", server.DefaultShutdownTimeout), "Maximum time to wait for in-flight requests to drain on shutdown") // ACME/TLS configuration + runCommand.cmd.Flags().StringVar(&globalConfig.DockerSocketPath, "docker-socket", getEnvString("DOCKER_SOCKET", ""), "Path to the container runtime socket, enabling --sleep-after on deploy (default empty, disabled). Reaching this socket is root-equivalent on the host, so it is opt-in") runCommand.cmd.Flags().StringVar(&globalConfig.MinTLS, "min-tls", getEnvString("MIN_TLS", server.DefaultMinTLSVersion), "Lowest TLS version the HTTPS listener will negotiate: 1.2 or 1.3 (TLS 1.0 and 1.1 cannot be enabled; HTTP/3 is always 1.3)") runCommand.cmd.Flags().StringVar(&globalConfig.ACMEEmail, "acme-email", getEnvString("ACME_EMAIL", ""), "Email address for ACME account registration (required for automatic TLS)") runCommand.cmd.Flags().StringVar(&globalConfig.ACMEDirectory, "acme-directory", getEnvString("ACME_DIRECTORY", server.LetsEncryptProduction), "ACME directory URL") diff --git a/internal/server/config.go b/internal/server/config.go index e8a99e5..a52b5c3 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -46,6 +46,15 @@ type Config struct { MetricsPort int HTTP3Enabled bool + // DockerSocketPath enables scale-to-zero by giving the proxy a container + // runtime to stop and start with. Empty (the default) leaves the feature + // unavailable, and a deploy asking for --sleep-after is refused rather than + // accepted and silently never acted on. + // + // Reaching this socket is root-equivalent on the host, which is why it is + // opt-in and off by default. + DockerSocketPath string + // MinTLS is the lowest TLS version the HTTPS listener will negotiate, // written as "1.2" or "1.3". Empty means 1.2, which is also Go's own // minimum, so this setting can only ever narrow what the listener accepts - diff --git a/internal/server/idle_controller.go b/internal/server/idle_controller.go index fd97c6a..c394561 100644 --- a/internal/server/idle_controller.go +++ b/internal/server/idle_controller.go @@ -315,6 +315,56 @@ func (c *IdleController) EndRequest() { c.signal() } +// Configure applies a redeploy's settings in place. The controller is created +// once per Service lifetime and reconfigured thereafter, so the request path can +// read s.idleController without a lock. +// +// Changing the container set means the same thing a redeploy does -- the service +// is awake and pointed somewhere new -- so this takes the same generation bump +// Reset does, invalidating any lifecycle goroutine started against the old set. +func (c *IdleController) Configure(sleepAfter, wakeTimeout time.Duration, refs []string) { + if wakeTimeout <= 0 { + wakeTimeout = DefaultWakeTimeout + } + + c.lock.Lock() + changed := !slices.Equal(c.refs, refs) + + c.sleepAfter = sleepAfter + c.wakeTimeout = wakeTimeout + c.refs = slices.Clone(refs) + + var cancel context.CancelFunc + if changed { + c.lastRequest = time.Now() + c.lastErr, c.failures, c.retryAfter = nil, 0, time.Time{} + + cancel = c.cancel + c.cancel = nil + + if c.state == IdleStateActive { + c.generation++ + } else { + c.setStateLocked(IdleStateActive) + } + } + c.lock.Unlock() + + if cancel != nil { + cancel() + } + c.signal() +} + +// LastWakeError reports why the most recent wake failed, so a health check can +// stop claiming a service is fine once it can no longer start. Nil once a wake +// has succeeded. +func (c *IdleController) LastWakeError() error { + c.lock.Lock() + defer c.lock.Unlock() + return c.lastErr +} + func (c *IdleController) WakeTimeout() time.Duration { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/server/service.go b/internal/server/service.go index 5d39a84..3316231 100644 --- a/internal/server/service.go +++ b/internal/server/service.go @@ -174,6 +174,17 @@ type ServiceOptions struct { Redirects []PathRule `json:"redirects,omitempty"` Rewrites []PathRule `json:"rewrites,omitempty"` + // SleepAfter stops this service's containers after this long with no traffic, + // starting them again on the next request. Zero (the default) never sleeps. + SleepAfter time.Duration `json:"sleep_after,omitempty"` + // WakeTimeout bounds how long a request is held while those containers start + // and pass a health check. Zero means DefaultWakeTimeout. + WakeTimeout time.Duration `json:"wake_timeout,omitempty"` + // SleepContainers names the containers to stop and start, replacing what the + // proxy infers from the target addresses. Needed when a target names a network + // alias rather than a container. + SleepContainers []string `json:"sleep_containers,omitempty"` + // Compression encodes responses on their way back to the client. A zero // value leaves them alone, which is what every state file written before // this option existed restores to. @@ -194,6 +205,14 @@ func (so *ServiceOptions) Normalize() { so.PathPrefixes = NormalizePathPrefixes(so.PathPrefixes) so.Compression.Normalize() so.Cache.Normalize() + + // The cobra default makes --help honest; this is what gives restored state + // files and direct RPC callers the same value. Only zero defaults: a negative + // is a typo, and silently turning it into 30s would hide it -- validateSleep + // rejects it instead. + if so.SleepAfter > 0 && so.WakeTimeout == 0 { + so.WakeTimeout = DefaultWakeTimeout + } } func (so ServiceOptions) Validate() error { @@ -275,6 +294,10 @@ func (so ServiceOptions) Validate() error { return err } + if err := so.validateSleep(); err != nil { + return err + } + return so.validateDynamicDomains() } @@ -334,6 +357,11 @@ type Service struct { // entry into it, sitting between the checks above and the load balancer. cacheStore CacheStore cacheHandler http.Handler + + lifecycle ContainerLifecycle + idleController *IdleController + restoredIdleState IdleState + statePersister func() } func NewService(name string, options ServiceOptions, targetOptions TargetOptions, sanCertManager *SANCertManager) (*Service, error) { @@ -369,7 +397,13 @@ func (s *Service) SetSANCertManager(manager *SANCertManager) { } func (s *Service) Dispose() { - s.active.Dispose() + if s.idleController != nil { + s.idleController.Close() + } + + if s.active != nil { + s.active.Dispose() + } if s.rollout != nil { s.rollout.Dispose() } @@ -397,6 +431,12 @@ func (s *Service) UpdateLoadBalancer(lb *LoadBalancer, slot TargetSlot) *LoadBal s.active = lb } + // Refs are derived from the targets, so they are only knowable once a load + // balancer is installed -- initialize runs before that on a first deploy. A + // redeploy pointing at new containers reaches here too, which is exactly when + // the controller needs to be told. + s.configureIdleController(s.options) + return replaced } @@ -434,7 +474,12 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type marshalledService struct { - Name string `json:"name"` + Name string `json:"name"` + // IdleState is written as a name rather than an enum's number: the state file + // outlives proxy versions and operators read it. Absent -- every state file + // written before scale-to-zero existed -- parses to active, which is also what + // SleepAfter == 0 produces, so the feature restores off. + IdleState string `json:"idle_state,omitempty"` Options ServiceOptions `json:"options"` TargetOptions TargetOptions `json:"target_options"` ActiveTargets []string `json:"active_targets"` @@ -469,8 +514,14 @@ func (s *Service) MarshalJSON() ([]byte, error) { rolloutReaders = s.rollout.ReadTargets().Specs() } + idleState := IdleStateActive + if s.idleController != nil { + idleState = s.idleController.State() + } + return json.Marshal(marshalledService{ Name: s.name, + IdleState: idleState.String(), ActiveTargets: s.active.WriteTargets().Specs(), ActiveReaders: s.active.ReadTargets().Specs(), RolloutTargets: rolloutTargets, @@ -528,6 +579,11 @@ func (s *Service) UnmarshalJSON(data []byte) error { s.rollout.MarkAllHealthy() } + // Recorded, not acted on: the lifecycle is still nil here, so a controller + // built now could reach StopContainer on a nil interface. SetContainerLifecycle + // is what creates it, after the router has decoded the whole state file. + s.restoredIdleState = ParseIdleState(ms.IdleState) + return s.initialize(ms.Options, ms.TargetOptions) } @@ -601,6 +657,7 @@ func (s *Service) initialize(options ServiceOptions, targetOptions TargetOptions s.cacheHandler = s.createCacheHandler(options) s.options = options s.targetOptions = targetOptions + s.configureIdleController(options) s.certManager = certManager s.clientCAs = clientCAs s.middleware = middleware @@ -807,6 +864,17 @@ func (s *Service) serviceRequestWithTarget(w http.ResponseWriter, r *http.Reques return } + // After every gate above, so that no blocked, throttled, unauthenticated or + // redirected request can spend a container start. Before target selection, so + // the request body is still unread when the hold begins. + handled, endIdleRequest := s.handleIdleRequest(w, r) + if endIdleRequest != nil { + defer endIdleRequest() + } + if handled { + return + } + // Last, so that everything above -- the health check exemptions, the // redirects, the allow list -- still sees the path the client asked for. r = s.rewriteRequest(r) diff --git a/internal/server/service_idle.go b/internal/server/service_idle.go new file mode 100644 index 0000000..c24863a --- /dev/null +++ b/internal/server/service_idle.go @@ -0,0 +1,225 @@ +package server + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "slices" + "time" +) + +// validateSleep joins the chain in ServiceOptions.Validate. +func (so ServiceOptions) validateSleep() error { + if so.SleepAfter < 0 || so.WakeTimeout < 0 { + return fmt.Errorf("%w: sleep-after and wake-timeout cannot be negative", ErrServiceOptionsInvalid) + } + + if so.SleepAfter > 0 && so.TLSOnDemandURL != "" { + // An on-demand check asks the backend, at handshake time, whether a host + // may have a certificate. A sleeping backend cannot answer, and waking one + // would let any SNI on the internet start a container. + return fmt.Errorf("%w: sleep-after cannot be used with a TLS on-demand URL", ErrServiceOptionsInvalid) + } + + if so.SleepAfter <= 0 && len(so.SleepContainers) > 0 { + return fmt.Errorf("%w: sleep-container requires sleep-after", ErrServiceOptionsInvalid) + } + + return nil +} + +// SetContainerLifecycle installs the runtime that starts and stops this service's +// containers, and builds the idle controller around it. Restored services get +// theirs after the state file is decoded, which is why UnmarshalJSON never +// creates one: at that point the lifecycle is still nil, and a controller built +// there could reach StopContainer on a nil interface. +func (s *Service) SetContainerLifecycle(lifecycle ContainerLifecycle) { + s.lifecycle = lifecycle + s.configureIdleController(s.options) + + if s.idleController == nil || s.idleController.State() != IdleStateSleeping { + return + } + + // Restore assumed every target healthy. For a sleeping service that is a + // healthy pool pointing at a stopped container -- and under + // --recheck-targets-on-restore, a probe against it every second. Put it back + // the way sleeping left it. + s.suspendForSleep() +} + +// configureIdleController writes s.idleController once per Service lifetime and +// reconfigures in place thereafter, so the request path's unlocked read of it is +// safe by construction. +func (s *Service) configureIdleController(options ServiceOptions) { + if options.SleepAfter <= 0 { + if s.idleController != nil { + s.idleController.Configure(0, 0, nil) + } + return + } + + if s.idleController == nil { + if s.lifecycle == nil { + // DeployService refuses sleep-after without a lifecycle, so this is a + // restored service whose lifecycle arrives later from + // SetContainerLifecycle. + return + } + + s.idleController = NewIdleController(IdleControllerConfig{ + Name: s.name, + Lifecycle: s.lifecycle, + Suspend: s.suspendForSleep, + Resume: s.resumeFromSleep, + Persist: s.persistState, + }) + + if s.restoredIdleState == IdleStateSleeping { + s.idleController.RestoreSleeping() + } + } + + s.idleController.Configure(options.SleepAfter, options.WakeTimeout, s.containerRefs(options)) +} + +// containerRefs names the containers behind this service's write targets. Read +// targets are replicas whose lifecycle the proxy does not own, so they are never +// stopped. +func (s *Service) containerRefs(options ServiceOptions) []string { + if len(options.SleepContainers) > 0 { + return slices.Clone(options.SleepContainers) + } + + refs := []string{} + for _, lb := range []*LoadBalancer{s.active, s.rollout} { + if lb == nil { + continue + } + for _, target := range lb.WriteTargets() { + if ref, ok := target.ContainerRef(); ok && !slices.Contains(refs, ref) { + refs = append(refs, ref) + } + } + } + + return refs +} + +func (s *Service) suspendForSleep() { + s.serviceLock.RLock() + active, rollout := s.active, s.rollout + s.serviceLock.RUnlock() + + for _, lb := range []*LoadBalancer{active, rollout} { + if lb != nil { + lb.SuspendForSleep() + } + } +} + +func (s *Service) resumeFromSleep(timeout time.Duration) error { + s.serviceLock.RLock() + active, rollout := s.active, s.rollout + s.serviceLock.RUnlock() + + for _, lb := range []*LoadBalancer{active, rollout} { + if lb != nil { + lb.ResumeFromSleep() + } + } + + if active == nil { + return nil + } + + // Started is not ready: wait for the woken container to actually answer. + return active.WaitUntilHealthy(timeout) +} + +func (s *Service) persistState() { + if s.statePersister != nil { + s.statePersister() + } +} + +// handleIdleRequest holds the request until this service's containers are back. +// It reports whether the client has already been answered, and returns the +// function that releases this request's hold on the idle timer. +func (s *Service) handleIdleRequest(w http.ResponseWriter, r *http.Request) (bool, func()) { + controller := s.idleController + if controller == nil { + return false, nil + } + + // Synthesized inside the proxy -- a TLS on-demand probe -- and must never + // start a container. validateSleep refuses the combination, so this only + // guards a state file written before that rule existed. + if isInternalRequest(r) { + return false, nil + } + + // A health check must never wake a service, or an uptime monitor pins it + // awake forever. It must be answered rather than held, or a load balancer in + // front evicts a service that is sleeping exactly as intended. + if s.targetOptions.IsHealthCheckRequest(r) { + return s.answerIdleHealthCheck(w, r, controller), nil + } + + if err := controller.BeginRequest(r.Context()); err != nil { + if errors.Is(err, context.Canceled) { + // The client hung up mid-wake. Nobody left to answer. + return true, nil + } + + // Logged, not rendered: the error carries container references and up to + // four kilobytes of daemon output, and this response is reachable by + // anyone who can open a connection. + slog.Error("Rejecting request: service did not wake", + "service", s.name, "path", r.URL.Path, "error", err) + w.Header().Set("Retry-After", "1") + SetErrorResponse(w, r, http.StatusServiceUnavailable, nil) + return true, nil + } + + return false, controller.EndRequest +} + +// answerIdleHealthCheck answers for a sleeping service without waking it. It +// reports healthy while the sleep is working as intended, and stops the moment a +// wake has actually failed -- otherwise a service that can no longer start +// reports green to its monitoring forever while 503ing every real request. +func (s *Service) answerIdleHealthCheck(w http.ResponseWriter, r *http.Request, controller *IdleController) bool { + if controller.State() == IdleStateActive { + return false + } + + if err := controller.LastWakeError(); err != nil { + slog.Warn("Reporting unhealthy: last wake failed", "service", s.name, "error", err) + SetErrorResponse(w, r, http.StatusServiceUnavailable, nil) + return true + } + + w.WriteHeader(http.StatusOK) + return true +} + +// ContainerRef returns the container this target's address names: its hostname, +// which is a container short id in a Kamal deployment and a container name under +// Compose. It is a best guess -- a Compose service alias resolves over Docker's +// DNS but names no container -- so --sleep-container exists to state it +// explicitly, and the deploy preflight turns a wrong guess into an error on the +// operator's terminal rather than a 503 an hour later. +// +// An address is rejected outright: no container runtime could act on one. +func (t *Target) ContainerRef() (string, bool) { + host := t.targetURL.Hostname() + if host == "" || net.ParseIP(host) != nil { + return "", false + } + + return host, true +} diff --git a/internal/server/service_idle_test.go b/internal/server/service_idle_test.go new file mode 100644 index 0000000..9c33105 --- /dev/null +++ b/internal/server/service_idle_test.go @@ -0,0 +1,308 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceOptions_ValidateSleep(t *testing.T) { + tests := []struct { + name string + options ServiceOptions + expectedError string + }{ + { + name: "sleep disabled by default", + options: ServiceOptions{Hosts: []string{"app.example.com"}}, + }, + { + name: "sleep enabled", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: time.Minute}, + }, + { + name: "negative sleep-after", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: -time.Second}, + expectedError: "cannot be negative", + }, + { + name: "negative wake-timeout", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepAfter: time.Minute, WakeTimeout: -time.Second}, + expectedError: "cannot be negative", + }, + { + // An on-demand check asks the backend at handshake time whether a host + // may have a certificate. A sleeping backend cannot answer, and waking + // one would let any SNI on the internet start a container. + name: "sleep with a TLS on-demand URL", + options: ServiceOptions{TLSEnabled: true, TLSOnDemandURL: "/ask", SleepAfter: time.Minute}, + expectedError: "cannot be used with a TLS on-demand URL", + }, + { + name: "sleep-container without sleep-after", + options: ServiceOptions{Hosts: []string{"app.example.com"}, SleepContainers: []string{"web-1"}}, + expectedError: "requires sleep-after", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.options.Validate() + + if tt.expectedError == "" { + assert.NoError(t, err) + return + } + + require.ErrorIs(t, err, ErrServiceOptionsInvalid) + assert.ErrorContains(t, err, tt.expectedError) + }) + } +} + +// The cobra default makes --help honest; Normalize is what gives restored state +// files and direct RPC callers the same value. +func TestServiceOptions_NormalizeDefaultsTheWakeTimeout(t *testing.T) { + options := ServiceOptions{SleepAfter: time.Minute} + options.Normalize() + assert.Equal(t, DefaultWakeTimeout, options.WakeTimeout) + + explicit := ServiceOptions{SleepAfter: time.Minute, WakeTimeout: 5 * time.Second} + explicit.Normalize() + assert.Equal(t, 5*time.Second, explicit.WakeTimeout) + + // Nothing is defaulted for a service that does not sleep. + off := ServiceOptions{} + off.Normalize() + assert.Zero(t, off.WakeTimeout) +} + +func testSleepingService(t *testing.T, lifecycle ContainerLifecycle, handler http.HandlerFunc) *Service { + t.Helper() + + backend := httptest.NewServer(handler) + t.Cleanup(backend.Close) + + options := defaultServiceOptions + options.Hosts = []string{"app.example.com"} + options.SleepAfter = time.Hour // long: these tests drive the controller directly + // A httptest backend listens on 127.0.0.1, which ContainerRef rightly refuses + // as an address rather than a container -- so name it explicitly, which is + // exactly what --sleep-container exists for. + options.SleepContainers = []string{"web-1"} + + service, err := NewService("sleepy", options, defaultTargetOptions, nil) + require.NoError(t, err) + t.Cleanup(service.Dispose) + + targets, err := NewTargetList([]string{backend.Listener.Addr().String()}, []string{}, defaultTargetOptions) + require.NoError(t, err) + + lb := NewLoadBalancer(targets, DefaultWriterAffinityTimeout, false) + t.Cleanup(lb.Dispose) + require.NoError(t, lb.WaitUntilHealthy(5*time.Second)) + + service.SetContainerLifecycle(lifecycle) + service.UpdateLoadBalancer(lb, TargetSlotActive) + + require.NotNil(t, service.idleController, "a service with --sleep-after gets a controller") + require.NotEmpty(t, service.containerRefs(service.options), + "refs must be derived once the load balancer is installed, or a wake starts nothing") + + return service +} + +func sendServiceRequest(service *Service, path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, "http://app.example.com"+path, nil) + recorder := httptest.NewRecorder() + service.ServeHTTP(recorder, req) + return recorder +} + +// A sleeping service wakes on a real request and forwards it, body intact. +func TestService_SleepingServiceWakesOnARequest(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + require.Equal(t, IdleStateSleeping, service.idleController.State()) + + recorder := sendServiceRequest(service, "/") + + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Equal(t, int64(1), lifecycle.starts.Load(), "the request woke the containers") + assert.Equal(t, IdleStateActive, service.idleController.State()) +} + +// An uptime monitor polling /up would otherwise pin a service awake forever. +func TestService_HealthCheckNeitherWakesNorIsHeld(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + + recorder := sendServiceRequest(service, defaultTargetOptions.HealthCheckConfig.Path) + + assert.Equal(t, http.StatusOK, recorder.Code, "a sleeping service is healthy, not down") + assert.Zero(t, lifecycle.starts.Load(), "monitoring must never start a container") + assert.Equal(t, IdleStateSleeping, service.idleController.State()) +} + +// The other half: once a wake has actually failed, monitoring must stop being +// told everything is fine while every real request 503s. +func TestService_HealthCheckReportsUnhealthyAfterAFailedWake(t *testing.T) { + lifecycle := &fakeLifecycle{ + startFunc: func(ctx context.Context, ref string) error { return errors.New("no such container") }, + } + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + service.idleController.RestoreSleeping() + + // A real request tries to wake and fails. + failed := sendServiceRequest(service, "/") + require.Equal(t, http.StatusServiceUnavailable, failed.Code) + + health := sendServiceRequest(service, defaultTargetOptions.HealthCheckConfig.Path) + assert.Equal(t, http.StatusServiceUnavailable, health.Code, + "a service that can no longer start must not report green to its monitoring") +} + +// The wake error carries container references and up to 4 KB of daemon output, +// and this response is reachable by anyone who can open a connection. +func TestService_WakeFailureDoesNotLeakDaemonDetailToTheClient(t *testing.T) { + lifecycle := &fakeLifecycle{ + startFunc: func(ctx context.Context, ref string) error { + return errors.New("secret-container-name: permission denied on /var/run/docker.sock") + }, + } + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) {}) + service.idleController.RestoreSleeping() + + recorder := sendServiceRequest(service, "/") + + require.Equal(t, http.StatusServiceUnavailable, recorder.Code) + assert.NotContains(t, recorder.Body.String(), "secret-container-name") + assert.NotContains(t, recorder.Body.String(), "docker.sock") + assert.Equal(t, "1", recorder.Header().Get("Retry-After")) +} + +// The proxy's own TLS on-demand probe is synthesized internally and must never +// spend a container start. +func TestService_InternalRequestsDoNotWake(t *testing.T) { + lifecycle := &fakeLifecycle{} + + service := testSleepingService(t, lifecycle, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + service.idleController.RestoreSleeping() + + req := httptest.NewRequest(http.MethodGet, "http://app.example.com/", nil) + req = req.WithContext(markInternalRequest(req.Context())) + recorder := httptest.NewRecorder() + service.ServeHTTP(recorder, req) + + assert.Zero(t, lifecycle.starts.Load(), "an internal probe must never start a container") +} + +func TestService_ContainerRefsPreferTheExplicitOverride(t *testing.T) { + options := defaultServiceOptions + options.Hosts = []string{"app.example.com"} + options.SleepAfter = time.Hour + options.SleepContainers = []string{"explicit-1", "explicit-2"} + + service, err := NewService("refs", options, defaultTargetOptions, nil) + require.NoError(t, err) + t.Cleanup(service.Dispose) + + assert.Equal(t, []string{"explicit-1", "explicit-2"}, service.containerRefs(options), + "--sleep-container replaces inference entirely") +} + +func TestTarget_ContainerRef(t *testing.T) { + tests := []struct { + address string + expected string + ok bool + }{ + {address: "web-1", expected: "web-1", ok: true}, + {address: "web-1:3000", expected: "web-1", ok: true}, + {address: "3f2a1b9c4d5e:3000", expected: "3f2a1b9c4d5e", ok: true}, + + // An address names no container, and no runtime could ever act on it. + {address: "10.0.0.5:3000", ok: false}, + {address: "127.0.0.1:80", ok: false}, + } + + for _, tt := range tests { + t.Run(tt.address, func(t *testing.T) { + target, err := NewTarget(tt.address, defaultTargetOptions) + require.NoError(t, err) + + ref, ok := target.ContainerRef() + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.Equal(t, tt.expected, ref) + } + }) + } +} + +// Old state files predate the field and must restore with the feature off. +func TestService_IdleStateRoundTrip(t *testing.T) { + assert.Equal(t, IdleStateActive, ParseIdleState("")) + + service := testSleepingService(t, &fakeLifecycle{}, func(w http.ResponseWriter, r *http.Request) {}) + service.idleController.RestoreSleeping() + + encoded, err := service.MarshalJSON() + require.NoError(t, err) + assert.Contains(t, string(encoded), `"idle_state":"sleeping"`) + + var restored Service + require.NoError(t, restored.UnmarshalJSON(encoded)) + t.Cleanup(restored.Dispose) + + assert.Equal(t, time.Hour, restored.options.SleepAfter) + assert.Equal(t, IdleStateSleeping, restored.restoredIdleState) + assert.Nil(t, restored.idleController, + "UnmarshalJSON must not build a controller: the lifecycle is still nil here") +} + +// A state file written before this feature existed restores with sleep off and +// re-marshals byte-identically, because every new key is omitempty. +func TestService_StateFileWithoutIdleStateRestoresAwake(t *testing.T) { + targetOptions, err := json.Marshal(defaultTargetOptions) + require.NoError(t, err) + + // No idle_state key, and no sleep_after in options -- exactly what every + // state file written before this feature looks like. + legacy := fmt.Sprintf( + `{"name":"old","options":{"hosts":["app.example.com"]},"target_options":%s,"active_targets":["web-1:3000"]}`, + targetOptions) + + var restored Service + require.NoError(t, restored.UnmarshalJSON([]byte(legacy))) + t.Cleanup(restored.Dispose) + + assert.Zero(t, restored.options.SleepAfter) + assert.Equal(t, IdleStateActive, restored.restoredIdleState) + assert.Nil(t, restored.idleController) +}