diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index 5461c72d..c28e88fa 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "sync" + "sync/atomic" "time" "techulus/cloud-agent/internal/build" @@ -50,35 +51,44 @@ type ActualState struct { } type Agent struct { - state AgentState - stateMutex sync.RWMutex - reconcileRequested chan struct{} - statusReportRequested chan string - refreshMutex sync.Mutex - pendingExpectedStateRefresh bool - workMutex sync.Mutex - activeWorkItem *agenthttp.WorkQueueItem - pendingWorkResults []agenthttp.CompletedWorkItem - deploymentErrorMutex sync.Mutex - pendingDeploymentErrors []agenthttp.DeploymentError - Client *agenthttp.Client - Reconciler *reconcile.Reconciler - Config *Config - PublicIP string - PrivateIP string - DataDir string - expectedState *agenthttp.ExpectedState - processingStart time.Time - LogCollector *logs.Collector - TraefikLogCollector *logs.TraefikCollector - MetricsSender MetricsSender - Builder *build.Builder - isBuilding bool - buildMutex sync.Mutex - currentBuildID string - IsProxy bool - dnsInSync bool - DisableDNS bool + state AgentState + stateMutex sync.RWMutex + reconcileRequested chan struct{} + statusReportRequested chan string + refreshMutex sync.Mutex + pendingExpectedStateRefresh bool + workMutex sync.Mutex + activeWorkItem *agenthttp.WorkQueueItem + pendingWorkResults []agenthttp.CompletedWorkItem + deploymentErrorMutex sync.Mutex + pendingDeploymentErrors []agenthttp.DeploymentError + deployLockMutex sync.Mutex + deploymentDeployLocks map[string]*sync.Mutex + serverlessMutex sync.Mutex + pendingServerlessTransitions []agenthttp.ServerlessTransition + pendingServerlessSleep map[string]struct{} + pendingServerlessWake map[string]struct{} + expectedStateMutex sync.RWMutex + latestExpectedState *agenthttp.ExpectedState + Client *agenthttp.Client + Reconciler *reconcile.Reconciler + Config *Config + PublicIP string + PrivateIP string + DataDir string + expectedState *agenthttp.ExpectedState + processingStart time.Time + LogCollector *logs.Collector + TraefikLogCollector *logs.TraefikCollector + MetricsSender MetricsSender + Builder *build.Builder + isBuilding bool + buildMutex sync.Mutex + currentBuildID string + IsProxy bool + serverlessGatewayRunning atomic.Bool + dnsInSync bool + DisableDNS bool } func NewAgent( @@ -94,21 +104,24 @@ func NewAgent( disableDNS bool, ) *Agent { return &Agent{ - state: StateIdle, - reconcileRequested: make(chan struct{}, 1), - statusReportRequested: make(chan string, 1), - Client: client, - Reconciler: reconciler, - Config: config, - PublicIP: publicIP, - PrivateIP: privateIP, - DataDir: dataDir, - LogCollector: logCollector, - TraefikLogCollector: traefikLogCollector, - MetricsSender: metricsSender, - Builder: builder, - IsProxy: isProxy, - DisableDNS: disableDNS, + state: StateIdle, + reconcileRequested: make(chan struct{}, 1), + statusReportRequested: make(chan string, 1), + Client: client, + Reconciler: reconciler, + Config: config, + PublicIP: publicIP, + PrivateIP: privateIP, + DataDir: dataDir, + LogCollector: logCollector, + TraefikLogCollector: traefikLogCollector, + MetricsSender: metricsSender, + Builder: builder, + IsProxy: isProxy, + DisableDNS: disableDNS, + deploymentDeployLocks: map[string]*sync.Mutex{}, + pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{}, } } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 4ab0fc78..6d083884 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -23,6 +23,7 @@ const ( actionStopUnexpectedContainer reconcileActionKind = "stop_unexpected_container" actionRemoveUnexpectedContainer reconcileActionKind = "remove_unexpected_container" actionDeployMissingContainer reconcileActionKind = "deploy_missing_container" + actionStopExpectedContainer reconcileActionKind = "stop_expected_container" actionStartContainer reconcileActionKind = "start_container" actionRedeployContainer reconcileActionKind = "redeploy_container" actionUpdateDNS reconcileActionKind = "update_dns" @@ -102,6 +103,8 @@ func (a *Agent) handleIdle() { if fromCache { log.Printf("[idle] using cached state (CP unreachable)") } + a.SetLatestExpectedState(expected) + a.ReconcilePendingServerlessTransitionsWithExpected(expected, fromCache) actual, err := a.getActualState() if err != nil { @@ -250,6 +253,9 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS for id, exp := range expectedMap { if _, exists := actualMap[id]; !exists { + if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) || a.HasPendingServerlessWake(id) { + continue + } expectedContainer := exp actions = append(actions, reconcileAction{ Kind: actionDeployMissingContainer, @@ -264,26 +270,43 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS if act, exists := actualMap[id]; exists { expectedContainer := exp actualContainer := act - if act.State == "created" || act.State == "exited" { + + if a.HasPendingServerlessWake(id) { + continue + } + if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) { + if shouldStopDesiredStoppedContainer(act.State) { + actions = append(actions, reconcileAction{ + Kind: actionStopExpectedContainer, + Description: fmt.Sprintf("STOP %s (desired state: stopped)", exp.Name), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) + } + continue + } + + if normalizeImage(exp.Image) != normalizeImage(act.Image) { actions = append(actions, reconcileAction{ - Kind: actionStartContainer, - Description: fmt.Sprintf("START %s (state: %s)", exp.Name, act.State), + Kind: actionRedeployContainer, + Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, }) - } else if act.State != "running" { + } else if act.State == "created" || act.State == "exited" { actions = append(actions, reconcileAction{ - Kind: actionRedeployContainer, - Description: fmt.Sprintf("REDEPLOY %s (state: %s)", exp.Name, act.State), + Kind: actionStartContainer, + Description: fmt.Sprintf("START %s (state: %s)", exp.Name, act.State), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, }) - } else if normalizeImage(exp.Image) != normalizeImage(act.Image) { + } else if act.State != "running" { actions = append(actions, reconcileAction{ Kind: actionRedeployContainer, - Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), + Description: fmt.Sprintf("REDEPLOY %s (state: %s)", exp.Name, act.State), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, @@ -389,16 +412,32 @@ func normalizeImage(image string) string { return image + digest } +func desiredContainerState(container agenthttp.ExpectedContainer) string { + if container.DesiredState == "stopped" { + return "stopped" + } + return "running" +} + +func shouldStopDesiredStoppedContainer(state string) bool { + switch state { + case "created", "exited", "stopped": + return false + default: + return true + } +} + func (a *Agent) applyReconcileAction(action reconcileAction) error { log.Printf("[reconcile] %s", action.Description) switch action.Kind { - case actionStopOrphanNoDeploymentID, actionStopUnexpectedContainer: + case actionStopOrphanNoDeploymentID, actionStopUnexpectedContainer, actionStopExpectedContainer: if action.Actual == nil { return fmt.Errorf("missing actual container for %s", action.Kind) } if err := container.Stop(action.Actual.ID); err != nil { - return fmt.Errorf("failed to stop orphan container: %w", err) + return fmt.Errorf("failed to stop container: %w", err) } return nil @@ -433,7 +472,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if action.Expected == nil { return fmt.Errorf("missing expected container for %s", action.Kind) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to deploy container: %w", err) } return nil @@ -447,7 +486,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if err := container.Stop(action.Actual.ID); err != nil { log.Printf("[reconcile] warning: failed to stop old container: %v", err) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } } @@ -460,7 +499,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if err := container.Stop(action.Actual.ID); err != nil { log.Printf("[reconcile] warning: failed to stop old container: %v", err) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } return nil diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index e7459e0d..98c5169b 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -18,6 +18,8 @@ import ( var Version = "dev" +const serverlessGatewayCapability = "serverless_gateway" + var ( agentStartTime = time.Now() lastHealthCollect time.Time @@ -51,8 +53,9 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport report.NetworkHealth = health.CollectNetworkHealth("wg0") report.ContainerHealth = health.CollectContainerHealth() report.AgentHealth = &agenthttp.AgentHealth{ - Version: Version, - UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Version: Version, + UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Capabilities: a.agentCapabilities(), } lastHealthCollect = time.Now() log.Printf("[health] collected: cpu=%.1f%%, mem=%.1f%%, disk=%.1f%%, network=%v, containers=%d running", @@ -68,6 +71,9 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport if c.DeploymentID == "" { continue } + if a.ShouldSuppressServerlessContainerReport(c.DeploymentID) { + continue + } status := "stopped" if c.State == "running" { @@ -95,6 +101,13 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport return report } +func (a *Agent) agentCapabilities() []string { + if !a.IsProxy || !a.serverlessGatewayRunning.Load() { + return nil + } + return []string{serverlessGatewayCapability} +} + func (a *Agent) RecordDeploymentError(deploymentID string, err error) { if deploymentID == "" || err == nil { return diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index cbeb2d62..58ead8ae 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -6,6 +6,7 @@ import ( "time" "techulus/cloud-agent/internal/container" + "techulus/cloud-agent/internal/serverless" ) func (a *Agent) Run(ctx context.Context) { @@ -32,6 +33,15 @@ func (a *Agent) Run(ctx context.Context) { a.TraefikLogCollector.Start() } + if a.IsProxy { + gateway := serverless.NewGateway(a) + if err := gateway.Start(ctx); err != nil { + log.Printf("[serverless-gateway] failed to start: %v", err) + } else { + a.serverlessGatewayRunning.Store(true) + } + } + var cleanupTickerC <-chan time.Time if a.Builder != nil { cleanupTicker := time.NewTicker(BuildCleanupInterval) @@ -109,12 +119,14 @@ func (a *Agent) reportStatus(reason string) { report := a.BuildStatusReport(true) reportedDeploymentErrorCount := len(report.DeploymentErrors) completed, active := a.SnapshotWorkStatus() - response, err := a.Client.ReportStatus(report, completed, active) + serverlessTransitions := a.SnapshotServerlessTransitions() + response, err := a.Client.ReportStatus(report, completed, active, serverlessTransitions) if err != nil { log.Printf("[status] failed to report (%s): %v", reason, err) return } a.ClearReportedDeploymentErrors(reportedDeploymentErrorCount) + a.ClearReportedServerlessTransitions(len(serverlessTransitions)) a.AcknowledgeWorkResults(response.AcceptedWorkItemResults, response.RejectedWorkItemResults) a.LogRejectedActiveWorkItems(response.RejectedActiveWorkItems) a.AcceptLeasedWorkItems(response.WorkItems) diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go new file mode 100644 index 00000000..9b033cc5 --- /dev/null +++ b/agent/internal/agent/serverless.go @@ -0,0 +1,243 @@ +package agent + +import ( + "log" + "sync" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +func (a *Agent) SetLatestExpectedState(state *agenthttp.ExpectedState) { + a.expectedStateMutex.Lock() + defer a.expectedStateMutex.Unlock() + a.latestExpectedState = state +} + +func (a *Agent) ExpectedState() *agenthttp.ExpectedState { + a.expectedStateMutex.RLock() + defer a.expectedStateMutex.RUnlock() + return a.latestExpectedState +} + +func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { + return a.withDeploymentDeployLock(expected.DeploymentID, func() error { + containers, err := container.List() + if err == nil { + for _, actual := range containers { + if actual.DeploymentID != expected.DeploymentID { + continue + } + if normalizeImage(actual.Image) != normalizeImage(expected.Image) { + log.Printf( + "[serverless] recreate deployment %s because image changed (%s -> %s)", + Truncate(expected.DeploymentID, 8), + actual.Image, + expected.Image, + ) + return a.Reconciler.Deploy(expected) + } + if actual.State == "running" { + return nil + } + log.Printf( + "[serverless] starting stopped container %s for deployment %s", + Truncate(actual.ID, 12), + Truncate(expected.DeploymentID, 8), + ) + if err := container.Start(actual.ID); err != nil { + log.Printf( + "[serverless] start failed for deployment %s, recreating: %v", + Truncate(expected.DeploymentID, 8), + err, + ) + return a.Reconciler.Deploy(expected) + } + return nil + } + } + return a.Reconciler.Deploy(expected) + }) +} + +func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) error { + return a.withDeploymentDeployLock(expected.DeploymentID, func() error { + containers, err := container.List() + if err == nil { + for _, actual := range containers { + if actual.DeploymentID != expected.DeploymentID || actual.State != "running" { + continue + } + if normalizeImage(actual.Image) == normalizeImage(expected.Image) { + return nil + } + } + } + return a.Reconciler.Deploy(expected) + }) +} + +func (a *Agent) withDeploymentDeployLock(deploymentID string, fn func() error) error { + a.deployLockMutex.Lock() + if a.deploymentDeployLocks == nil { + a.deploymentDeployLocks = map[string]*sync.Mutex{} + } + lock, ok := a.deploymentDeployLocks[deploymentID] + if !ok { + lock = &sync.Mutex{} + a.deploymentDeployLocks[deploymentID] = lock + } + a.deployLockMutex.Unlock() + + lock.Lock() + defer lock.Unlock() + return fn() +} + +func (a *Agent) StopServerlessContainer(containerID string) error { + return container.Stop(containerID) +} + +func (a *Agent) ListServerlessContainers() ([]container.Container, error) { + return container.List() +} + +func (a *Agent) GetServerlessContainerHealth(containerID string) string { + return container.GetHealthStatus(containerID) +} + +func (a *Agent) QueueServerlessTransition(transition agenthttp.ServerlessTransition) { + if transition.Type == "" || transition.DeploymentID == "" { + return + } + + a.serverlessMutex.Lock() + if a.pendingServerlessSleep == nil { + a.pendingServerlessSleep = map[string]struct{}{} + } + if a.pendingServerlessWake == nil { + a.pendingServerlessWake = map[string]struct{}{} + } + switch transition.Type { + case "sleep": + a.pendingServerlessSleep[transition.DeploymentID] = struct{}{} + delete(a.pendingServerlessWake, transition.DeploymentID) + case "wake_started": + delete(a.pendingServerlessSleep, transition.DeploymentID) + a.pendingServerlessWake[transition.DeploymentID] = struct{}{} + case "wake_failed": + delete(a.pendingServerlessWake, transition.DeploymentID) + } + a.pendingServerlessTransitions = append( + a.pendingServerlessTransitions, + transition, + ) + a.serverlessMutex.Unlock() + + a.RequestStatusReport("serverless " + transition.Type) +} + +func (a *Agent) SnapshotServerlessTransitions() []agenthttp.ServerlessTransition { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + return append([]agenthttp.ServerlessTransition(nil), a.pendingServerlessTransitions...) +} + +func (a *Agent) ClearReportedServerlessTransitions(count int) { + if count <= 0 { + return + } + + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + if count > len(a.pendingServerlessTransitions) { + count = len(a.pendingServerlessTransitions) + } + a.pendingServerlessTransitions = a.pendingServerlessTransitions[count:] +} + +func (a *Agent) HasPendingServerlessSleep(deploymentID string) bool { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + _, ok := a.pendingServerlessSleep[deploymentID] + return ok +} + +func (a *Agent) HasPendingServerlessWake(deploymentID string) bool { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + _, ok := a.pendingServerlessWake[deploymentID] + return ok +} + +func (a *Agent) ShouldSuppressServerlessContainerReport(deploymentID string) bool { + a.serverlessMutex.Lock() + _, pendingSleep := a.pendingServerlessSleep[deploymentID] + _, pendingWake := a.pendingServerlessWake[deploymentID] + a.serverlessMutex.Unlock() + + if pendingWake { + return false + } + if pendingSleep { + return true + } + + a.expectedStateMutex.RLock() + defer a.expectedStateMutex.RUnlock() + if a.latestExpectedState == nil { + return false + } + for _, expected := range a.latestExpectedState.Containers { + if expected.DeploymentID == deploymentID { + return expected.DesiredState == "stopped" + } + } + return false +} + +func (a *Agent) ReconcilePendingServerlessTransitionsWithExpected(state *agenthttp.ExpectedState, fromCache bool) { + if state == nil || fromCache { + return + } + + desiredByDeploymentID := map[string]string{} + for _, expected := range state.Containers { + desiredByDeploymentID[expected.DeploymentID] = expected.DesiredState + } + + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + pendingSleepTransitions := map[string]struct{}{} + pendingWakeTransitions := map[string]struct{}{} + for _, transition := range a.pendingServerlessTransitions { + switch transition.Type { + case "sleep": + pendingSleepTransitions[transition.DeploymentID] = struct{}{} + case "wake_started": + pendingWakeTransitions[transition.DeploymentID] = struct{}{} + } + } + + for deploymentID := range a.pendingServerlessSleep { + if _, stillReporting := pendingSleepTransitions[deploymentID]; stillReporting { + continue + } + delete(a.pendingServerlessSleep, deploymentID) + desiredState, ok := desiredByDeploymentID[deploymentID] + if ok && desiredState == "running" { + log.Printf("[serverless] sleep transition for deployment %s is not reflected in expected state; allowing reconcile", Truncate(deploymentID, 8)) + } + } + + for deploymentID := range a.pendingServerlessWake { + if _, stillReporting := pendingWakeTransitions[deploymentID]; stillReporting { + continue + } + delete(a.pendingServerlessWake, deploymentID) + desiredState, ok := desiredByDeploymentID[deploymentID] + if !ok || desiredState != "running" { + log.Printf("[serverless] wake transition for deployment %s is not reflected in expected state; allowing reconcile", Truncate(deploymentID, 8)) + } + } +} diff --git a/agent/internal/agent/serverless_test.go b/agent/internal/agent/serverless_test.go new file mode 100644 index 00000000..683570ec --- /dev/null +++ b/agent/internal/agent/serverless_test.go @@ -0,0 +1,86 @@ +package agent + +import ( + "slices" + "testing" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +func TestPendingServerlessWakeDoesNotStopStaleStoppedExpectedContainer(t *testing.T) { + agent := &Agent{ + DisableDNS: true, + pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{ + "dep_serverless": {}, + }, + } + expected := &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_serverless", + ServiceID: "svc_1", + Name: "svc_1-dep_serverless", + DesiredState: "stopped", + Image: "nginx", + }, + }, + } + actual := &ActualState{ + Containers: []container.Container{ + { + ID: "ctr_serverless", + Name: "svc_1-dep_serverless", + Image: "nginx", + State: "running", + DeploymentID: "dep_serverless", + }, + }, + } + + for _, action := range agent.planReconcile(expected, actual) { + if action.DeploymentID == "dep_serverless" { + t.Fatalf("planReconcile returned action for pending wake: %+v", action) + } + } +} + +func TestServerlessGatewayCapabilityRequiresStartedGateway(t *testing.T) { + agent := &Agent{IsProxy: true} + + if slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("proxy reported serverless gateway capability before gateway start") + } + + agent.serverlessGatewayRunning.Store(true) + if !slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("proxy did not report serverless gateway capability after gateway start") + } + + agent.IsProxy = false + if slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("worker reported serverless gateway capability") + } +} + +func TestPendingServerlessWakeDoesNotSuppressContainerReport(t *testing.T) { + agent := &Agent{ + pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{ + "dep_serverless": {}, + }, + latestExpectedState: &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_serverless", + DesiredState: "stopped", + }, + }, + }, + } + + if agent.ShouldSuppressServerlessContainerReport("dep_serverless") { + t.Fatal("container report was suppressed while wake transition is pending") + } +} diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 7189f791..ec529e9a 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -73,6 +73,7 @@ type ExpectedContainer struct { ServiceID string `json:"serviceId"` ServiceName string `json:"serviceName"` Name string `json:"name"` + DesiredState string `json:"desiredState"` Image string `json:"image"` IPAddress string `json:"ipAddress"` Ports []PortMapping `json:"ports"` @@ -94,6 +95,25 @@ type Upstream struct { Weight int `json:"weight"` } +type ServerlessUpstream struct { + DeploymentID string `json:"deploymentId"` + ServerID string `json:"serverId"` + Url string `json:"url"` + Local bool `json:"local"` + AlwaysOn bool `json:"alwaysOn"` +} + +type ServerlessRoute struct { + ServiceID string `json:"serviceId"` + Domain string `json:"domain"` + Port int `json:"port"` + SleepAfterSeconds int `json:"sleepAfterSeconds"` + WakeTimeoutSeconds int `json:"wakeTimeoutSeconds"` + MinReadyReplicas int `json:"minReadyReplicas"` + LocalDeploymentIDs []string `json:"localDeploymentIds"` + Upstreams []ServerlessUpstream `json:"upstreams"` +} + type TraefikRoute struct { ID string `json:"id"` Domain string `json:"domain"` @@ -138,6 +158,9 @@ type ExpectedState struct { Dns struct { Records []DnsRecord `json:"records"` } `json:"dns"` + Serverless struct { + Routes []ServerlessRoute `json:"routes"` + } `json:"serverless"` Traefik struct { HttpRoutes []TraefikRoute `json:"httpRoutes"` TCPRoutes []TraefikTCPRoute `json:"tcpRoutes"` @@ -245,8 +268,9 @@ type Resources struct { } type AgentHealth struct { - Version string `json:"version"` - UptimeSecs int64 `json:"uptimeSecs"` + Version string `json:"version"` + UptimeSecs int64 `json:"uptimeSecs"` + Capabilities []string `json:"capabilities,omitempty"` } type StatusReport struct { @@ -274,6 +298,13 @@ type ActiveWorkItem struct { Attempt int `json:"attempt"` } +type ServerlessTransition struct { + Type string `json:"type"` + DeploymentID string `json:"deploymentId"` + ContainerID string `json:"containerId,omitempty"` + Error string `json:"error,omitempty"` +} + type BuildDetails struct { Build struct { ID string `json:"id"` @@ -377,7 +408,7 @@ type StatusResponse struct { WorkItems []WorkQueueItem `json:"workItems"` } -func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkItem, active []ActiveWorkItem) (*StatusResponse, error) { +func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkItem, active []ActiveWorkItem, serverlessTransitions []ServerlessTransition) (*StatusResponse, error) { payload := map[string]interface{}{ "statusReport": report, } @@ -387,6 +418,9 @@ func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkIte if len(active) > 0 { payload["activeWorkItems"] = active } + if len(serverlessTransitions) > 0 { + payload["serverlessTransitions"] = serverlessTransitions + } body, err := json.Marshal(payload) if err != nil { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go new file mode 100644 index 00000000..163ddbcc --- /dev/null +++ b/agent/internal/serverless/gateway.go @@ -0,0 +1,988 @@ +package serverless + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +const ( + GatewayPort = 18080 + upstreamCacheTTL = 10 * time.Second +) + +var ( + wakePollInterval = 500 * time.Millisecond + idleTimerSeedInterval = 15 * time.Second +) + +type Gateway struct { + runtime Runtime + counter uint64 + server *http.Server + mu sync.Mutex + activityMu sync.Mutex + + upstreamCache map[string]cachedUpstreams + wakeCalls map[string]*wakeCall + activities map[string]*activityState +} + +type Runtime interface { + ExpectedState() *agenthttp.ExpectedState + DeployServerlessContainer(agenthttp.ExpectedContainer) error + StopServerlessContainer(containerID string) error + ListServerlessContainers() ([]container.Container, error) + GetServerlessContainerHealth(containerID string) string + QueueServerlessTransition(agenthttp.ServerlessTransition) +} + +type cachedUpstreams struct { + upstreams []agenthttp.ServerlessUpstream + expiresAt time.Time +} + +type wakeCall struct { + done chan struct{} + upstreams []agenthttp.ServerlessUpstream + err error +} + +type activityState struct { + mu sync.Mutex + activeRequests int + sleepTimer *time.Timer +} + +func NewGateway(runtime Runtime) *Gateway { + return &Gateway{ + runtime: runtime, + upstreamCache: map[string]cachedUpstreams{}, + wakeCalls: map[string]*wakeCall{}, + activities: map[string]*activityState{}, + } +} + +func (g *Gateway) Start(ctx context.Context) error { + addr := fmt.Sprintf("127.0.0.1:%d", GatewayPort) + listener, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", addr, err) + } + + g.server = &http.Server{ + Handler: g, + ReadHeaderTimeout: 30 * time.Second, + } + + go func() { + <-ctx.Done() + g.stopAllActivities() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := g.server.Shutdown(shutdownCtx); err != nil { + log.Printf("[serverless-gateway] shutdown error: %v", err) + } + }() + + go func() { + log.Printf("[serverless-gateway] listening on %s", addr) + if err := g.server.Serve(listener); err != nil && err != http.ErrServerClosed { + log.Printf("[serverless-gateway] server error: %v", err) + } + }() + + go func() { + g.seedIdleTimers() + ticker := time.NewTicker(idleTimerSeedInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + g.seedIdleTimers() + } + } + }() + + return nil +} + +func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { + host := normalizeHost(r.Host) + if host == "" { + http.Error(w, "missing host", http.StatusBadRequest) + return + } + + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + log.Printf("[serverless-gateway] no route metadata for host %s", host) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + activityKey := serviceActivityKey(route.ServiceID) + g.beginActivity(activityKey) + defer g.endActivity(activityKey, route.ServiceID, route.SleepAfterSeconds) + + upstreams, err := g.getUpstreams(r.Context(), host) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + log.Printf("[serverless-gateway] wake failed for host %s: %v", host, err) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + + if len(upstreams) == 0 { + log.Printf("[serverless-gateway] no upstreams for host %s", host) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + + upstream := upstreams[g.nextIndex(len(upstreams))] + target, err := url.Parse("http://" + upstream.Url) + if err != nil { + log.Printf("[serverless-gateway] invalid upstream %q for host %s: %v", upstream.Url, host, err) + http.Error(w, "bad upstream", http.StatusBadGateway) + return + } + + proxy := httputil.NewSingleHostReverseProxy(target) + originalDirector := proxy.Director + originalHost := r.Host + proxy.Director = func(req *http.Request) { + originalDirector(req) + req.Host = originalHost + req.Header.Set("X-Forwarded-Host", originalHost) + req.Header.Set("X-Forwarded-Proto", forwardedProto(r)) + } + proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) { + log.Printf("[serverless-gateway] proxy error for host %s to %s: %v", host, upstream.Url, err) + g.evictUpstreams(host) + http.Error(w, "bad gateway", http.StatusBadGateway) + } + proxy.ServeHTTP(w, r) +} + +func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.ServerlessUpstream, error) { + if upstreams, ok := g.cachedUpstreams(host); ok { + return upstreams, nil + } + + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + return nil, fmt.Errorf("no serverless route metadata for host %s", host) + } + wakeKey := routeWakeKey(route) + call, owner := g.beginWake(wakeKey) + if owner { + go func() { + upstreams, err := g.resolveUpstreams(host) + g.finishWake(wakeKey, call, upstreams, err) + }() + } + + select { + case <-call.done: + case <-ctx.Done(): + return nil, ctx.Err() + } + + if call.err != nil { + return nil, call.err + } + upstreams := cloneUpstreams(call.upstreams) + if len(upstreams) > 0 { + g.cacheUpstreams(host, upstreams) + } + return upstreams, nil +} + +func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, error) { + state := g.runtime.ExpectedState() + route := findRoute(state, host) + if route == nil { + return nil, fmt.Errorf("no serverless route metadata for host %s", host) + } + + ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + if err != nil { + return nil, err + } + if len(sleepingLocalIDs) == 0 { + if len(ready) > 0 { + return ready, nil + } + return nil, fmt.Errorf("no ready upstreams for host %s", host) + } + + wakeStartedAt := time.Now() + if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && hasAlwaysOnUpstream(ready)) { + log.Printf( + "[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=background", + host, + len(sleepingLocalIDs), + len(ready), + ) + go func() { + startedIDs := g.wakeLocalDeployments(route, state, sleepingLocalIDs) + if len(startedIDs) > 0 { + g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, wakeStartedAt, startedIDs) + } + }() + return ready, nil + } + + log.Printf( + "[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=blocking timeout=%s", + host, + len(sleepingLocalIDs), + len(ready), + wakeTimeout(route.WakeTimeoutSeconds), + ) + startedIDs := g.wakeLocalDeployments(route, state, sleepingLocalIDs) + if len(startedIDs) == 0 { + return nil, fmt.Errorf("failed to start local serverless wake") + } + return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds, wakeStartedAt, startedIDs) +} + +func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) ([]agenthttp.ServerlessUpstream, []string, error) { + actualContainers, err := g.runtime.ListServerlessContainers() + if err != nil { + return nil, nil, fmt.Errorf("failed to list local containers: %w", err) + } + + expectedByDeploymentID := expectedContainersByDeploymentID(state) + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + localIDs := map[string]struct{}{} + for _, deploymentID := range route.LocalDeploymentIDs { + localIDs[deploymentID] = struct{}{} + } + + upstreamsByURL := map[string]agenthttp.ServerlessUpstream{} + for _, upstream := range route.Upstreams { + if upstream.Local { + continue + } + upstreamsByURL[upstream.Url] = upstream + } + + sleepingLocalIDs := []string{} + for _, deploymentID := range route.LocalDeploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + continue + } + actual, isRunning := actualByDeploymentID[deploymentID] + if isRunning && g.isContainerReady(actual, expected) { + if upstream, ok := localUpstream(route, expected); ok { + upstreamsByURL[upstream.Url] = upstream + } + continue + } + if expected.DesiredState == "stopped" { + sleepingLocalIDs = append(sleepingLocalIDs, deploymentID) + } + } + + upstreams := make([]agenthttp.ServerlessUpstream, 0, len(upstreamsByURL)) + for _, upstream := range upstreamsByURL { + if upstream.Local { + if _, ok := localIDs[upstream.DeploymentID]; !ok { + continue + } + } + upstreams = append(upstreams, upstream) + } + sortUpstreams(upstreams) + return upstreams, sleepingLocalIDs, nil +} + +func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState, deploymentIDs []string) []string { + expectedByDeploymentID := expectedContainersByDeploymentID(state) + startedIDs := []string{} + for _, deploymentID := range deploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + log.Printf( + "[serverless-gateway] wake skipped host=%s deployment=%s reason=missing_expected_state", + route.Domain, + deploymentID, + ) + continue + } + deployStartedAt := time.Now() + log.Printf( + "[serverless-gateway] wake starting host=%s deployment=%s service=%s container=%s", + route.Domain, + deploymentID, + expected.ServiceID, + expected.Name, + ) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_started", + DeploymentID: deploymentID, + }) + if err := g.runtime.DeployServerlessContainer(expected); err != nil { + log.Printf( + "[serverless-gateway] wake failed host=%s deployment=%s service=%s latency=%s error=%v", + route.Domain, + deploymentID, + expected.ServiceID, + roundDuration(time.Since(deployStartedAt)), + err, + ) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_failed", + DeploymentID: deploymentID, + Error: err.Error(), + }) + continue + } + startedIDs = append(startedIDs, deploymentID) + log.Printf( + "[serverless-gateway] wake container started host=%s deployment=%s service=%s latency=%s", + route.Domain, + deploymentID, + expected.ServiceID, + roundDuration(time.Since(deployStartedAt)), + ) + } + g.evictUpstreams(route.Domain) + return startedIDs +} + +func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time, wokenDeploymentIDs []string) ([]agenthttp.ServerlessUpstream, error) { + timeout := wakeTimeout(wakeTimeoutSeconds) + deadline := startedAt.Add(timeout) + + for { + state := g.runtime.ExpectedState() + ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + if err != nil { + return nil, err + } + if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) { + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + if len(pendingIDs) > 0 { + go g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, startedAt, pendingIDs) + } + log.Printf( + "[serverless-gateway] wake ready host=%s upstreams=%d latency=%s", + route.Domain, + len(ready), + roundDuration(time.Since(startedAt)), + ) + return ready, nil + } + if time.Now().After(deadline) { + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + g.queueWakeTimeouts(route, pendingIDs, startedAt) + if len(ready) > 0 { + log.Printf( + "[serverless-gateway] wake partially ready host=%s upstreams=%d latency=%s", + route.Domain, + len(ready), + roundDuration(time.Since(startedAt)), + ) + return ready, nil + } + log.Printf( + "[serverless-gateway] wake timed out host=%s latency=%s", + route.Domain, + roundDuration(time.Since(startedAt)), + ) + return nil, fmt.Errorf("timed out waiting for local serverless wake") + } + time.Sleep(wakePollInterval) + } +} + +func (g *Gateway) waitForWokenDeployments(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time, wokenDeploymentIDs []string) { + timeout := wakeTimeout(wakeTimeoutSeconds) + deadline := startedAt.Add(timeout) + + for { + state := g.runtime.ExpectedState() + ready, _, err := g.readyUpstreams(route, state) + if err != nil { + log.Printf("[serverless-gateway] wake monitor failed host=%s error=%v", route.Domain, err) + return + } + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + if len(pendingIDs) == 0 { + log.Printf( + "[serverless-gateway] wake ready host=%s deployments=%d latency=%s", + route.Domain, + len(wokenDeploymentIDs), + roundDuration(time.Since(startedAt)), + ) + return + } + if time.Now().After(deadline) { + g.queueWakeTimeouts(route, pendingIDs, startedAt) + return + } + time.Sleep(wakePollInterval) + } +} + +func pendingWakeDeploymentIDs(wokenDeploymentIDs []string, ready []agenthttp.ServerlessUpstream) []string { + readyIDs := map[string]struct{}{} + for _, upstream := range ready { + if upstream.Local { + readyIDs[upstream.DeploymentID] = struct{}{} + } + } + + pendingIDs := []string{} + for _, deploymentID := range wokenDeploymentIDs { + if _, ok := readyIDs[deploymentID]; !ok { + pendingIDs = append(pendingIDs, deploymentID) + } + } + return pendingIDs +} + +func (g *Gateway) queueWakeTimeouts(route *agenthttp.ServerlessRoute, deploymentIDs []string, startedAt time.Time) { + for _, deploymentID := range deploymentIDs { + errMessage := fmt.Sprintf("timed out waiting %s for local serverless wake", roundDuration(time.Since(startedAt))) + log.Printf( + "[serverless-gateway] wake timed out host=%s deployment=%s service=%s latency=%s", + route.Domain, + deploymentID, + route.ServiceID, + roundDuration(time.Since(startedAt)), + ) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_failed", + DeploymentID: deploymentID, + Error: errMessage, + }) + } +} + +func (g *Gateway) isContainerReady(actual container.Container, expected agenthttp.ExpectedContainer) bool { + if actual.State != "running" { + return false + } + if expected.HealthCheck == nil || expected.HealthCheck.Cmd == "" { + return true + } + health := g.runtime.GetServerlessContainerHealth(actual.ID) + return health == "healthy" || health == "none" +} + +func (g *Gateway) cachedUpstreams(host string) ([]agenthttp.ServerlessUpstream, bool) { + g.mu.Lock() + defer g.mu.Unlock() + + cached, ok := g.upstreamCache[host] + if !ok || time.Now().After(cached.expiresAt) { + if ok { + delete(g.upstreamCache, host) + } + return nil, false + } + + return cloneUpstreams(cached.upstreams), true +} + +func (g *Gateway) beginWake(wakeKey string) (*wakeCall, bool) { + g.mu.Lock() + defer g.mu.Unlock() + + if call, ok := g.wakeCalls[wakeKey]; ok { + return call, false + } + + call := &wakeCall{done: make(chan struct{})} + g.wakeCalls[wakeKey] = call + return call, true +} + +func (g *Gateway) finishWake(wakeKey string, call *wakeCall, upstreams []agenthttp.ServerlessUpstream, err error) { + g.mu.Lock() + defer g.mu.Unlock() + + call.upstreams = cloneUpstreams(upstreams) + call.err = err + delete(g.wakeCalls, wakeKey) + close(call.done) +} + +func (g *Gateway) cacheUpstreams(host string, upstreams []agenthttp.ServerlessUpstream) { + g.mu.Lock() + defer g.mu.Unlock() + g.upstreamCache[host] = cachedUpstreams{ + upstreams: cloneUpstreams(upstreams), + expiresAt: time.Now().Add(upstreamCacheTTL), + } +} + +func (g *Gateway) evictUpstreams(host string) { + g.mu.Lock() + defer g.mu.Unlock() + delete(g.upstreamCache, host) +} + +func (g *Gateway) evictServiceUpstreams(serviceID string) { + state := g.runtime.ExpectedState() + g.mu.Lock() + defer g.mu.Unlock() + if state == nil { + g.upstreamCache = map[string]cachedUpstreams{} + return + } + for _, route := range state.Serverless.Routes { + if route.ServiceID == serviceID { + delete(g.upstreamCache, normalizeHost(route.Domain)) + } + } +} + +func (g *Gateway) seedIdleTimers() { + state := g.runtime.ExpectedState() + if state == nil { + return + } + actualContainers, err := g.runtime.ListServerlessContainers() + if err != nil { + log.Printf("[serverless-gateway] failed to seed idle timers: %v", err) + return + } + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + seenServices := map[string]struct{}{} + for _, route := range state.Serverless.Routes { + if _, seen := seenServices[route.ServiceID]; seen { + continue + } + seenServices[route.ServiceID] = struct{}{} + if !hasRunningLocalDeployment(route.LocalDeploymentIDs, actualByDeploymentID) { + continue + } + g.scheduleSleepTimer(serviceActivityKey(route.ServiceID), route.ServiceID, route.SleepAfterSeconds) + } +} + +func (g *Gateway) scheduleSleepTimer(key string, serviceID string, sleepAfterSeconds int) { + activity := g.activity(key) + activity.mu.Lock() + defer activity.mu.Unlock() + if activity.activeRequests > 0 || activity.sleepTimer != nil { + return + } + delay := sleepDelay(sleepAfterSeconds) + activity.sleepTimer = time.AfterFunc(delay, func() { + g.sleepService(serviceID) + }) +} + +func (g *Gateway) activity(key string) *activityState { + g.activityMu.Lock() + defer g.activityMu.Unlock() + + activity, ok := g.activities[key] + if !ok { + activity = &activityState{} + g.activities[key] = activity + } + return activity +} + +func (g *Gateway) beginActivity(key string) { + activity := g.activity(key) + activity.mu.Lock() + defer activity.mu.Unlock() + + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + activity.sleepTimer = nil + } + activity.activeRequests += 1 +} + +func (g *Gateway) endActivity(key string, serviceID string, sleepAfterSeconds int) { + activity := g.activity(key) + activity.mu.Lock() + defer activity.mu.Unlock() + + if activity.activeRequests > 0 { + activity.activeRequests -= 1 + } + if activity.activeRequests > 0 { + return + } + + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + } + delay := sleepDelay(sleepAfterSeconds) + activity.sleepTimer = time.AfterFunc(delay, func() { + g.sleepService(serviceID) + }) +} + +func (g *Gateway) sleepHost(host string) { + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + log.Printf("[serverless-gateway] sleep skipped host=%s reason=missing_route_metadata", host) + return + } + g.sleepService(route.ServiceID) +} + +func (g *Gateway) sleepService(serviceID string) { + activityKey := serviceActivityKey(serviceID) + activity := g.activity(activityKey) + activity.mu.Lock() + if activity.activeRequests > 0 { + activeRequests := activity.activeRequests + activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep skipped service=%s reason=active_requests active=%d", + serviceID, + activeRequests, + ) + return + } + activity.sleepTimer = nil + activity.mu.Unlock() + + state := g.runtime.ExpectedState() + route := findRouteByServiceID(state, serviceID) + if route == nil { + log.Printf("[serverless-gateway] sleep skipped service=%s reason=missing_route_metadata", serviceID) + return + } + localDeploymentIDs := localDeploymentIDsForService(state, serviceID) + log.Printf( + "[serverless-gateway] sleep timer fired service=%s deployments=%d", + serviceID, + len(localDeploymentIDs), + ) + + actualContainers, err := g.runtime.ListServerlessContainers() + if err != nil { + log.Printf("[serverless-gateway] failed to list containers before sleep for service %s: %v", serviceID, err) + return + } + + expectedByDeploymentID := expectedContainersByDeploymentID(state) + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + for _, deploymentID := range localDeploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + log.Printf( + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=missing_expected_state", + serviceID, + deploymentID, + ) + continue + } + actual, ok := actualByDeploymentID[deploymentID] + if !ok || actual.State != "running" { + reason := "no_local_container" + if ok { + reason = "container_not_running" + } + log.Printf( + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=%s", + serviceID, + deploymentID, + reason, + ) + continue + } + if expected.DesiredState != "stopped" && !g.isContainerReady(actual, expected) { + log.Printf( + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=container_not_ready", + serviceID, + deploymentID, + ) + continue + } + + activity.mu.Lock() + if activity.activeRequests > 0 { + activeRequests := activity.activeRequests + activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=active_requests active=%d", + serviceID, + deploymentID, + activeRequests, + ) + return + } + sleepStartedAt := time.Now() + if expected.DesiredState != "stopped" { + log.Printf( + "[serverless-gateway] sleep starting service=%s deployment=%s container=%s", + serviceID, + deploymentID, + actual.ID, + ) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "sleep", + DeploymentID: deploymentID, + ContainerID: actual.ID, + }) + } else { + log.Printf( + "[serverless-gateway] sleep stop starting service=%s deployment=%s container=%s reason=already_expected_stopped", + serviceID, + deploymentID, + actual.ID, + ) + } + if err := g.runtime.StopServerlessContainer(actual.ID); err != nil { + activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep failed service=%s deployment=%s container=%s latency=%s error=%v", + serviceID, + deploymentID, + actual.ID, + roundDuration(time.Since(sleepStartedAt)), + err, + ) + continue + } + g.evictServiceUpstreams(serviceID) + activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep complete service=%s deployment=%s container=%s latency=%s", + serviceID, + deploymentID, + actual.ID, + roundDuration(time.Since(sleepStartedAt)), + ) + } + g.evictServiceUpstreams(serviceID) +} + +func (g *Gateway) stopAllActivities() { + g.activityMu.Lock() + activities := make([]*activityState, 0, len(g.activities)) + for _, activity := range g.activities { + activities = append(activities, activity) + } + g.activityMu.Unlock() + + for _, activity := range activities { + activity.mu.Lock() + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + activity.sleepTimer = nil + } + activity.mu.Unlock() + } +} + +func findRoute(state *agenthttp.ExpectedState, host string) *agenthttp.ServerlessRoute { + if state == nil { + return nil + } + normalizedHost := normalizeHost(host) + for i := range state.Serverless.Routes { + if normalizeHost(state.Serverless.Routes[i].Domain) == normalizedHost { + return &state.Serverless.Routes[i] + } + } + return nil +} + +func findRouteByServiceID(state *agenthttp.ExpectedState, serviceID string) *agenthttp.ServerlessRoute { + if state == nil { + return nil + } + for i := range state.Serverless.Routes { + if state.Serverless.Routes[i].ServiceID == serviceID { + return &state.Serverless.Routes[i] + } + } + return nil +} + +func localDeploymentIDsForService(state *agenthttp.ExpectedState, serviceID string) []string { + if state == nil { + return nil + } + ids := map[string]struct{}{} + for _, route := range state.Serverless.Routes { + if route.ServiceID != serviceID { + continue + } + for _, deploymentID := range route.LocalDeploymentIDs { + ids[deploymentID] = struct{}{} + } + } + deploymentIDs := make([]string, 0, len(ids)) + for deploymentID := range ids { + deploymentIDs = append(deploymentIDs, deploymentID) + } + sort.Strings(deploymentIDs) + return deploymentIDs +} + +func routeWakeKey(route *agenthttp.ServerlessRoute) string { + deploymentIDs := append([]string(nil), route.LocalDeploymentIDs...) + sort.Strings(deploymentIDs) + return fmt.Sprintf( + "%s:%d:%s", + route.ServiceID, + route.Port, + strings.Join(deploymentIDs, ","), + ) +} + +func expectedContainersByDeploymentID(state *agenthttp.ExpectedState) map[string]agenthttp.ExpectedContainer { + containersByDeploymentID := map[string]agenthttp.ExpectedContainer{} + if state == nil { + return containersByDeploymentID + } + for _, expected := range state.Containers { + containersByDeploymentID[expected.DeploymentID] = expected + } + return containersByDeploymentID +} + +func actualContainersByDeploymentID(containers []container.Container) map[string]container.Container { + containersByDeploymentID := map[string]container.Container{} + for _, actual := range containers { + if actual.DeploymentID != "" { + containersByDeploymentID[actual.DeploymentID] = actual + } + } + return containersByDeploymentID +} + +func localUpstream(route *agenthttp.ServerlessRoute, expected agenthttp.ExpectedContainer) (agenthttp.ServerlessUpstream, bool) { + if expected.IPAddress != "" { + return agenthttp.ServerlessUpstream{ + DeploymentID: expected.DeploymentID, + Url: fmt.Sprintf("%s:%d", expected.IPAddress, route.Port), + Local: true, + }, true + } + + for _, port := range expected.Ports { + if port.ContainerPort == route.Port && port.HostPort > 0 { + return agenthttp.ServerlessUpstream{ + DeploymentID: expected.DeploymentID, + Url: fmt.Sprintf("127.0.0.1:%d", port.HostPort), + Local: true, + }, true + } + } + return agenthttp.ServerlessUpstream{}, false +} + +func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool { + for _, upstream := range upstreams { + if upstream.AlwaysOn { + return true + } + } + return false +} + +func hasRunningLocalDeployment(deploymentIDs []string, actualByDeploymentID map[string]container.Container) bool { + for _, deploymentID := range deploymentIDs { + if actual, ok := actualByDeploymentID[deploymentID]; ok && actual.State == "running" { + return true + } + } + return false +} + +func sortUpstreams(upstreams []agenthttp.ServerlessUpstream) { + sort.Slice(upstreams, func(i, j int) bool { + return compareUpstream(upstreams[i], upstreams[j]) < 0 + }) +} + +func compareUpstream(a, b agenthttp.ServerlessUpstream) int { + if a.AlwaysOn != b.AlwaysOn { + if a.AlwaysOn { + return -1 + } + return 1 + } + if a.Local != b.Local { + if a.Local { + return -1 + } + return 1 + } + return strings.Compare(a.Url, b.Url) +} + +func wakeTimeout(wakeTimeoutSeconds int) time.Duration { + timeout := time.Duration(wakeTimeoutSeconds) * time.Second + if timeout <= 0 { + return 5 * time.Minute + } + return timeout +} + +func sleepDelay(sleepAfterSeconds int) time.Duration { + delay := time.Duration(sleepAfterSeconds) * time.Second + if delay <= 0 { + return 5 * time.Minute + } + return delay +} + +func serviceActivityKey(serviceID string) string { + return "service:" + serviceID +} + +func roundDuration(duration time.Duration) time.Duration { + return duration.Round(time.Millisecond) +} + +func cloneUpstreams(upstreams []agenthttp.ServerlessUpstream) []agenthttp.ServerlessUpstream { + return append([]agenthttp.ServerlessUpstream(nil), upstreams...) +} + +func (g *Gateway) nextIndex(length int) int { + if length <= 1 { + return 0 + } + return int(atomic.AddUint64(&g.counter, 1)-1) % length +} + +func normalizeHost(host string) string { + if h, _, err := net.SplitHostPort(host); err == nil { + return strings.ToLower(strings.TrimSpace(h)) + } + return strings.ToLower(strings.TrimSpace(host)) +} + +func forwardedProto(r *http.Request) string { + if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { + return proto + } + if r.TLS != nil { + return "https" + } + return "http" +} diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go new file mode 100644 index 00000000..5bf165b2 --- /dev/null +++ b/agent/internal/serverless/gateway_test.go @@ -0,0 +1,412 @@ +package serverless + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +type fakeRuntime struct { + mu sync.Mutex + state *agenthttp.ExpectedState + containers []container.Container + transitions []agenthttp.ServerlessTransition + stopped []string + deployCalls int + deployErr error + afterList func() + healthStatus string + deployStarted chan struct{} + deployStartedOnce sync.Once + allowDeploy chan struct{} +} + +func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { + f.mu.Lock() + defer f.mu.Unlock() + return f.state +} + +func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { + if f.deployStarted != nil { + f.deployStartedOnce.Do(func() { + close(f.deployStarted) + }) + } + if f.allowDeploy != nil { + <-f.allowDeploy + } + f.mu.Lock() + defer f.mu.Unlock() + f.deployCalls += 1 + if f.deployErr != nil { + return f.deployErr + } + for i, actual := range f.containers { + if actual.DeploymentID == expected.DeploymentID { + f.containers[i].State = "running" + return nil + } + } + f.containers = append(f.containers, container.Container{ + ID: "ctr-" + expected.DeploymentID, + State: "running", + DeploymentID: expected.DeploymentID, + ServiceID: expected.ServiceID, + }) + return nil +} + +func (f *fakeRuntime) StopServerlessContainer(containerID string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.stopped = append(f.stopped, containerID) + for i, actual := range f.containers { + if actual.ID == containerID { + f.containers[i].State = "exited" + } + } + return nil +} + +func (f *fakeRuntime) ListServerlessContainers() ([]container.Container, error) { + f.mu.Lock() + containers := append([]container.Container(nil), f.containers...) + afterList := f.afterList + f.mu.Unlock() + if afterList != nil { + afterList() + } + return containers, nil +} + +func (f *fakeRuntime) GetServerlessContainerHealth(containerID string) string { + if f.healthStatus != "" { + return f.healthStatus + } + return "healthy" +} + +func (f *fakeRuntime) QueueServerlessTransition(transition agenthttp.ServerlessTransition) { + f.mu.Lock() + defer f.mu.Unlock() + f.transitions = append(f.transitions, transition) +} + +func (f *fakeRuntime) snapshot() ([]agenthttp.ServerlessTransition, []string, int) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]agenthttp.ServerlessTransition(nil), f.transitions...), append([]string(nil), f.stopped...), f.deployCalls +} + +func (f *fakeRuntime) snapshotContainers() []container.Container { + f.mu.Lock() + defer f.mu.Unlock() + return append([]container.Container(nil), f.containers...) +} + +func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { + state := testExpectedState("stopped") + g := NewGateway(&fakeRuntime{state: state}) + g.wakeCalls[routeWakeKey(&state.Serverless.Routes[0])] = &wakeCall{done: make(chan struct{})} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := g.getUpstreams(ctx, "app.example.com") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestConcurrentDomainsShareOneWakeAttempt(t *testing.T) { + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + secondRoute := state.Serverless.Routes[0] + secondRoute.Domain = "api.example.com" + state.Serverless.Routes = append(state.Serverless.Routes, secondRoute) + runtime := &fakeRuntime{ + state: state, + deployStarted: make(chan struct{}), + allowDeploy: make(chan struct{}), + } + gateway := NewGateway(runtime) + + errs := make(chan error, 2) + go func() { + _, err := gateway.getUpstreams(context.Background(), "app.example.com") + errs <- err + }() + select { + case <-runtime.deployStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first wake to start") + } + go func() { + _, err := gateway.getUpstreams(context.Background(), "api.example.com") + errs <- err + }() + time.Sleep(10 * time.Millisecond) + close(runtime.allowDeploy) + + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + } + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 1 || transitions[0].Type != "wake_started" { + t.Fatalf("transitions = %+v, want one wake_started", transitions) + } +} + +func TestRequestCanUseAlwaysOnWorkerWhileLocalDeploymentWakes(t *testing.T) { + runtime := &fakeRuntime{state: testExpectedState("stopped")} + gateway := NewGateway(runtime) + + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.20:3000" { + t.Fatalf("upstreams = %+v, want worker upstream", upstreams) + } + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + transitions, _, deployCalls := runtime.snapshot() + if deployCalls == 1 && len(transitions) == 1 && transitions[0].Type == "wake_started" { + return + } + time.Sleep(5 * time.Millisecond) + } + transitions, _, deployCalls := runtime.snapshot() + t.Fatalf("deployCalls=%d transitions=%+v, want one background wake", deployCalls, transitions) +} + +func TestSleepingLocalDeploymentWakesAndReturnsLocalUpstream(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + runtime := &fakeRuntime{state: state} + gateway := NewGateway(runtime) + + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.10:3000" { + t.Fatalf("upstreams = %+v, want local upstream", upstreams) + } + + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 1 || transitions[0].Type != "wake_started" { + t.Fatalf("transitions = %+v, want wake_started", transitions) + } +} + +func TestSleepingLocalDeploymentStartsExistingStoppedContainer(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + { + ID: "ctr-existing", + State: "exited", + DeploymentID: "dep_local", + ServiceID: "svc_1", + }, + }, + } + gateway := NewGateway(runtime) + + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.10:3000" { + t.Fatalf("upstreams = %+v, want local upstream", upstreams) + } + + containers := runtime.snapshotContainers() + if len(containers) != 1 { + t.Fatalf("containers = %+v, want existing container only", containers) + } + if containers[0].ID != "ctr-existing" || containers[0].State != "running" { + t.Fatalf("container = %+v, want existing container running", containers[0]) + } +} + +func TestSleepHostStopsLocalContainerAndReportsSleep(t *testing.T) { + state := testExpectedState("running") + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + + gateway.sleepHost("app.example.com") + + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 1 || stopped[0] != "ctr-local" { + t.Fatalf("stopped = %+v, want ctr-local", stopped) + } + if len(transitions) != 1 { + t.Fatalf("transitions = %+v, want one sleep transition", transitions) + } + if transitions[0].Type != "sleep" || transitions[0].DeploymentID != "dep_local" || transitions[0].ContainerID != "ctr-local" { + t.Fatalf("transition = %+v, want sleep for dep_local/ctr-local", transitions[0]) + } +} + +func TestSleepHostRechecksActivityBeforeStoppingContainer(t *testing.T) { + state := testExpectedState("running") + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + runtime.afterList = func() { + gateway.beginActivity(serviceActivityKey("svc_1")) + } + + gateway.sleepHost("app.example.com") + + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 0 { + t.Fatalf("stopped = %+v, want no stops while request is active", stopped) + } + if len(transitions) != 0 { + t.Fatalf("transitions = %+v, want no sleep transition while request is active", transitions) + } +} + +func TestSleepHostUsesServiceActivityAcrossDomains(t *testing.T) { + state := testExpectedState("running") + secondRoute := state.Serverless.Routes[0] + secondRoute.Domain = "api.example.com" + state.Serverless.Routes = append(state.Serverless.Routes, secondRoute) + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + gateway.beginActivity(serviceActivityKey("svc_1")) + + gateway.sleepHost("api.example.com") + + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 0 { + t.Fatalf("stopped = %+v, want no stops while sibling domain is active", stopped) + } + if len(transitions) != 0 { + t.Fatalf("transitions = %+v, want no sleep transition while sibling domain is active", transitions) + } +} + +func TestWakeTimeoutQueuesWakeFailedTransition(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Containers[0].HealthCheck = &agenthttp.HealthCheck{ + Cmd: "curl http://localhost:3000/health", + Interval: 1, + Timeout: 1, + Retries: 1, + } + state.Serverless.Routes[0].Upstreams = nil + state.Serverless.Routes[0].WakeTimeoutSeconds = 1 + runtime := &fakeRuntime{ + state: state, + healthStatus: "unhealthy", + } + gateway := NewGateway(runtime) + + _, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err == nil { + t.Fatal("getUpstreams succeeded, want timeout error") + } + + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 2 { + t.Fatalf("transitions = %+v, want wake_started and wake_failed", transitions) + } + if transitions[0].Type != "wake_started" || transitions[1].Type != "wake_failed" { + t.Fatalf("transitions = %+v, want wake_started then wake_failed", transitions) + } + if transitions[1].DeploymentID != "dep_local" || transitions[1].Error == "" { + t.Fatalf("wake_failed transition = %+v, want dep_local with error", transitions[1]) + } +} + +func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { + state := &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_local", + ServiceID: "svc_1", + ServiceName: "api", + Name: "svc_1-dep_local", + DesiredState: localDesiredState, + Image: "nginx", + IPAddress: "10.0.0.10", + }, + }, + } + state.Serverless.Routes = []agenthttp.ServerlessRoute{ + { + ServiceID: "svc_1", + Domain: "app.example.com", + Port: 3000, + SleepAfterSeconds: 300, + WakeTimeoutSeconds: 5, + MinReadyReplicas: 1, + LocalDeploymentIDs: []string{"dep_local"}, + Upstreams: []agenthttp.ServerlessUpstream{ + { + DeploymentID: "dep_worker", + ServerID: "server_worker", + Url: "10.0.0.20:3000", + AlwaysOn: true, + }, + }, + }, + } + return state +} diff --git a/docs/agents/architecture.mdx b/docs/agents/architecture.mdx index 16e466bd..e9eef776 100644 --- a/docs/agents/architecture.mdx +++ b/docs/agents/architecture.mdx @@ -75,3 +75,37 @@ reports, the command can be retried up to the fixed attempt limit. | `force_cleanup` | Force remove containers for a service | | `cleanup_volumes` | Remove volume directories for a service | | `deploy` | Handled through expected-state reconciliation | + +## Serverless Gateway + +Proxy agents run a local HTTP wake gateway on `127.0.0.1:18080`. The control +plane emits Traefik routes to this gateway only on proxy nodes that host a local +proxy deployment for that serverless service. Proxies without a local +proxy-hosted deployment do not emit a public HTTP route for that service. +Worker-only serverless services keep normal direct routes because worker +deployments do not sleep. + +This means public DNS or external load balancers must route a serverless +service's traffic only to proxy nodes that own a local proxy replica for that +service. + +For serverless service traffic, Traefik forwards the public request to the local +gateway. The gateway resolves the host from expected-state serverless metadata, +queues `wake_started` as a status transition, starts local proxy-hosted +containers from expected-state config, waits for ready upstreams, then proxies +the original request to the selected container over the WireGuard mesh. + +The gateway keeps a short upstream cache and collapses concurrent wake requests +for the same host. Request cancellation is respected while waiting for a shared +wake result. + +The gateway tracks request activity in memory. When a host stays idle for the +service's configured sleep timeout, the proxy agent removes only its local +proxy-hosted containers and queues a `sleep` status transition. The control plane +records the transition and updates dashboard/routing state; it does not run the +idle timer. + +If a status report fails, pending serverless transitions stay queued in memory +and are retried. A locally slept deployment is also guarded from immediate +reconcile until a fresh expected-state fetch confirms the control plane recorded +the sleep or rejects it by continuing to advertise `desiredState: "running"`. diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 6272152d..fa9729cb 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -203,6 +203,72 @@ Proxy nodes receive routes and certificates from the control plane: Worker nodes do not run Traefik. +### Serverless Containers + +Public HTTP services can be configured as serverless. Serverless scale-to-zero is +proxy-local: deployments placed on proxy nodes may sleep after an idle period, +then wake on the next public request handled by that proxy. Deployments placed on +worker nodes stay always on and remain routable. + +Serverless uses the same declarative expected-state model as normal +deployments, but proxy agents own the local lifecycle decision: + +- A sleeping deployment keeps `trafficState: "active"` but records + `runtimeDesiredState: "stopped"` and `observedPhase: "sleeping"`. +- Expected state advertises proxy-hosted sleeping containers with + `desiredState: "stopped"` so normal reconciliation does not restart them. +- Expected state advertises worker-hosted deployments with + `desiredState: "running"` even when the service is serverless-enabled. +- The proxy gateway wakes local sleeping deployments from expected-state + metadata and reports `wake_started` through the next status report. +- The proxy agent reports local sleep with a `sleep` status transition after it + stops the local container. + +```mermaid +sequenceDiagram + participant User + participant Traefik + participant Gateway as Agent wake gateway + participant Agent + participant CP as Control plane + participant Container + + User->>Traefik: Request service domain + Traefik->>Gateway: Route to 127.0.0.1:18080 + Gateway->>Gateway: Resolve host from expected-state metadata + Gateway->>Agent: Queue wake_started status transition + Agent->>Container: Start local Podman container + Agent->>CP: Report wake_started and container status + CP->>CP: Record waking/healthy state + Gateway->>Container: Proxy original request +``` + +Proxy Traefik routes point to the local wake gateway only on proxy nodes that +host a local proxy deployment for that serverless service. Non-owner proxies do +not emit a public HTTP route for that service, even when always-on worker +upstreams exist. Worker-only serverless services use normal direct routes because +there is nothing local to wake. + +Public DNS or external load balancers must therefore send serverless traffic only +to proxy nodes that own a local proxy replica for that service. Cross-proxy wake +coordination is intentionally out of scope. + +The wake gateway keeps the incoming request open while it starts local proxy +replicas, unless an always-on worker upstream is already ready. `Min Ready` +controls how many ready upstreams are preferred before releasing queued requests. +If fewer replicas become ready and no local wake is still in progress, the +gateway serves any available ready upstream instead of returning a 503. + +Sleep is driven by the proxy gateway's in-memory activity timer. The control +plane does not scan request activity and does not enqueue sleep work. It accepts +validated `sleep`, `wake_started`, and `wake_failed` transitions through the +existing agent status endpoint. + +Wake failures are bounded. Transient failures return a deployment to `sleeping` +so a later request can retry. After repeated failures, the deployment is parked +as `failed` with a visible failed stage until a redeploy creates fresh +deployments. + ### Multiple Proxy Nodes The platform supports geographically distributed proxy nodes with proximity steering: diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 9e647b9a..46aa96ec 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -9,6 +9,41 @@ Each service can run multiple replicas across your cluster. Configure how many r Replica count ranges from 1 to 10 per service. +## Serverless scaling + +Public HTTP services can be configured to sleep when idle on proxy nodes. A +sleeping deployment keeps active traffic intent, records stopped runtime intent, +and stops the local container. The next public HTTP request wakes local +proxy-hosted deployments and is held until an upstream is ready or the wake +timeout is reached. + +Worker-hosted deployments do not sleep. If a serverless-enabled service has only +worker replicas, it stays always on and routes directly. If a service has both +proxy and worker replicas, proxy replicas may sleep while worker replicas remain +routable. + +For services with proxy-hosted serverless replicas, public traffic must be sent +only to proxy nodes that host a local proxy replica for that service. Non-owner +proxies do not emit a public HTTP route for the service, so DNS or the external +load balancer must avoid those proxies for that service's domains. + +Serverless settings are configured per service: + +| Setting | Default | Description | +| --- | --- | --- | +| Enable serverless | Off | Allows proxy-hosted HTTP deployments to sleep and wake on request | +| Sleep after | `300s` | Idle period before running containers are stopped | +| Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | +| Min Ready | `1` | Preferred number of ready deployments before queued requests resume | + +`Min Ready` does not cap how many local proxy replicas are started. A cold wake +starts the sleeping local proxy replicas for that host. `Min Ready` only controls +when held requests can resume. If a worker upstream is already ready, the gateway +can serve it immediately while local proxy replicas wake in the background. + +Serverless services require at least one configured replica. `Min Ready` must be +between `1` and `10`, and cannot exceed the configured replica count. + ## Placement Services use manual placement. Select the target servers and replica counts explicitly before deploying. @@ -27,3 +62,7 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services are always pinned to their locked server. - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. +- Serverless scaling requires a public HTTP service domain. +- Sleep and wake are proxy-local. Worker replicas are intentionally always on. +- Serverless traffic must be routed only to proxy nodes that own a local proxy replica for that service. +- Proxy agents report sleep and wake transitions through normal status reports. diff --git a/docs/services/volumes.mdx b/docs/services/volumes.mdx index ea13e4a5..d516e02e 100644 --- a/docs/services/volumes.mdx +++ b/docs/services/volumes.mdx @@ -18,6 +18,8 @@ Each volume has a name and a container path: When you add a volume, the service automatically becomes **stateful**. Stateful services are locked to a single server and limited to 1 replica so the container always mounts the same local data path. When the last volume is removed, the service reverts to stateless. +Stateful services can use serverless scaling when they have a public HTTP domain. Because volumes are local to one server, only a proxy-hosted stateful replica can sleep and wake on request. A stateful replica placed on a worker node stays always on. + ## Volume Backups Volumes can be backed up to S3-compatible storage on a schedule or on demand. diff --git a/web/actions/projects.ts b/web/actions/projects.ts index a9ef3af2..3e3865c8 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import cronstrue from "cronstrue"; -import { and, desc, eq, inArray, isNotNull } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { ZodError, z } from "zod"; import { db } from "@/db"; @@ -31,10 +31,16 @@ import { import { requireDeveloperRole } from "@/lib/auth"; import { DEFAULT_RESOURCE_LIMITS } from "@/lib/constants"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { + isObservedReady, + markDeploymentRemoved, + runtimeExpectedStates, +} from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; +import { blocksProjectDeletion } from "@/lib/project-deletion"; import { containerPathSchema, githubRepoUrlSchema, @@ -262,24 +268,13 @@ export async function deleteProject(id: string) { .from(services) .where(eq(services.projectId, id)); - const activeStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - "stopping", - ]; - for (const service of projectServices) { const activeDeployments = await db .select() .from(deployments) .where(eq(deployments.serviceId, service.id)); - const hasActiveDeployments = activeDeployments.some((d) => - activeStatuses.includes(d.status), - ); + const hasActiveDeployments = activeDeployments.some(blocksProjectDeletion); if (hasActiveDeployments) { throw new Error( @@ -536,15 +531,7 @@ async function hardDeleteService(serviceId: string) { .where(eq(deployments.serviceId, serviceId)); for (const dep of allDeployments) { - if ( - (dep.status === "running" || dep.status === "healthy") && - dep.containerId - ) { - await db - .update(deployments) - .set(markDeploymentUndesired("stopping")) - .where(eq(deployments.id, dep.id)); - + if (isObservedReady(dep.observedPhase) && dep.containerId) { await enqueueWork(dep.serverId, "stop", { deploymentId: dep.id, containerId: dep.containerId, @@ -627,7 +614,7 @@ export async function deleteService(serviceId: string) { .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), + inArray(deployments.observedPhase, ["running", "healthy"]), ), ) .then((r) => r[0]); @@ -1013,6 +1000,91 @@ export async function updateServiceSchedule( return { success: true }; } +const serverlessSettingsSchema = z.object({ + enabled: z.boolean(), + sleepAfterSeconds: z.number().int().min(60).max(86_400), + wakeTimeoutSeconds: z.number().int().min(10).max(900), + minReadyReplicas: z.number().int().min(1).max(10), +}); + +export async function updateServiceServerlessSettings( + serviceId: string, + settings: { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; + }, +) { + await requireDeveloperRole(); + const validated = serverlessSettingsSchema.parse(settings); + + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + + const [service] = await tx + .select() + .from(services) + .where(eq(services.id, serviceId)) + .limit(1); + + if (!service || service.deletedAt) { + throw new Error("Service not found"); + } + + if (validated.enabled) { + const publicHttpPorts = await tx + .select({ id: servicePorts.id }) + .from(servicePorts) + .where( + and( + eq(servicePorts.serviceId, serviceId), + eq(servicePorts.isPublic, true), + eq(servicePorts.protocol, "http"), + isNotNull(servicePorts.domain), + ), + ) + .limit(1); + + if (publicHttpPorts.length === 0) { + throw new Error( + "Serverless services require a public HTTP port with a domain", + ); + } + + const configuredReplicas = await tx + .select({ count: serviceReplicas.count }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const totalConfiguredReplicas = configuredReplicas.reduce( + (total, replica) => total + replica.count, + 0, + ); + + if (totalConfiguredReplicas < 1) { + throw new Error("Serverless services require at least one replica"); + } + if (validated.minReadyReplicas > totalConfiguredReplicas) { + throw new Error( + "Minimum ready replicas cannot exceed configured replicas", + ); + } + } + + await tx + .update(services) + .set({ + serverlessEnabled: validated.enabled, + serverlessSleepAfterSeconds: validated.sleepAfterSeconds, + serverlessWakeTimeoutSeconds: validated.wakeTimeoutSeconds, + serverlessMinReadyReplicas: validated.minReadyReplicas, + }) + .where(eq(services.id, serviceId)); + }); + + return { success: true }; +} + export type ServiceConfigUpdate = { source?: { type: "image"; image: string }; healthCheck?: ServiceHealthCheckConfig | null; @@ -1147,8 +1219,10 @@ export async function updateServiceConfig( .delete(serviceReplicas) .where(eq(serviceReplicas.serviceId, serviceId)); + let totalReplicas = 0; for (const replica of config.replicas) { if (replica.count > 0) { + totalReplicas += replica.count; await db.insert(serviceReplicas).values({ id: randomUUID(), serviceId, @@ -1157,6 +1231,15 @@ export async function updateServiceConfig( }); } } + + await db + .update(services) + .set({ + serverlessMinReadyReplicas: sql`LEAST(${services.serverlessMinReadyReplicas}, ${Math.max(1, totalReplicas)})`, + }) + .where( + and(eq(services.id, serviceId), eq(services.serverlessEnabled, true)), + ); } return { success: true }; @@ -1164,31 +1247,32 @@ export async function updateServiceConfig( export async function stopService(serviceId: string) { await requireDeveloperRole(); - const runningDeployments = await db + const desiredDeployments = await db .select() .from(deployments) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), ); - for (const dep of runningDeployments) { - if (!dep.containerId) continue; - + for (const dep of desiredDeployments) { + // User stop is teardown; runtimeDesiredState "stopped" is reserved for + // serverless sleep where the deployment must remain wakeable. await db .update(deployments) - .set(markDeploymentUndesired("stopping")) + .set(markDeploymentRemoved()) .where(eq(deployments.id, dep.id)); + if (!dep.containerId) continue; await enqueueWork(dep.serverId, "stop", { deploymentId: dep.id, containerId: dep.containerId, }); } - return { success: true, count: runningDeployments.length }; + return { success: true, count: desiredDeployments.length }; } export async function restartService(serviceId: string) { @@ -1204,7 +1288,7 @@ export async function restartService(serviceId: string) { .where(eq(deployments.serviceId, serviceId)); const deploymentsToRestart = runningDeployments.filter( - (d) => d.status === "running" && d.containerId, + (d) => isObservedReady(d.observedPhase) && d.containerId, ); if (deploymentsToRestart.length === 0) { @@ -1256,15 +1340,7 @@ export async function abortRollout(serviceId: string) { ); } - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - ), - ); + await restoreDrainingDeploymentsForRollback(serviceId); const rolloutDeployments = activeRolloutIds.length > 0 @@ -1429,16 +1505,7 @@ export async function removeServiceVolume(volumeId: string) { .from(deployments) .where(eq(deployments.serviceId, volume[0].serviceId)); - const runningStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - ]; - const hasRunning = activeDeployments.some((d) => - runningStatuses.includes(d.status), - ); + const hasRunning = activeDeployments.some(blocksProjectDeletion); if (hasRunning) { throw new Error("Stop the service before removing volumes"); } diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx index c014b426..77700a12 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx @@ -12,6 +12,7 @@ import { ReplicasSection } from "@/components/service/details/replicas-section"; import { ResourceLimitsSection } from "@/components/service/details/resource-limits-section"; import { ScheduleSection } from "@/components/service/details/schedule-section"; import { SecretsSection } from "@/components/service/details/secrets-section"; +import { ServerlessSection } from "@/components/service/details/serverless-section"; import { SourceSection } from "@/components/service/details/source-section"; import { StartCommandSection } from "@/components/service/details/start-command-section"; import { TCPProxySection } from "@/components/service/details/tcp-proxy-section"; @@ -46,7 +47,7 @@ export default function ConfigurationPage() { const hasActiveDeploymentForBackup = service.deployments.some( (deployment) => ACTIVE_DELETE_BACKUP_STATUSES.includes( - deployment.status as (typeof ACTIVE_DELETE_BACKUP_STATUSES)[number], + deployment.observedPhase as (typeof ACTIVE_DELETE_BACKUP_STATUSES)[number], ) && !!deployment.containerId, ); const hasVolumes = (service.volumes?.length ?? 0) > 0; @@ -98,6 +99,8 @@ export default function ConfigurationPage() { + + diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx index 2357c275..278395c1 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx @@ -46,6 +46,7 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { getServiceDeploymentActionState } from "@/lib/service-deployment-actions"; type ConfirmAction = "redeploy" | "stop" | "delete" | null; @@ -121,20 +122,7 @@ export default function DeploymentsPage() { }, }; - const hasRunningDeployments = service.deployments.some( - (d) => d.status === "running", - ); - const hasStoppedOrFailedDeployments = - !hasRunningDeployments && - service.deployments.some( - (d) => - d.status === "stopped" || - d.status === "failed" || - d.status === "rolled_back", - ); - const canStartAll = - hasStoppedOrFailedDeployments && - (service.configuredReplicas || []).length > 0; + const deploymentActions = getServiceDeploymentActionState(service); return (
@@ -163,8 +151,8 @@ export default function DeploymentsPage() { projectSlug={projectSlug} envName={envName} actions={ - service.deployments.length > 0 ? ( - hasRunningDeployments ? ( + deploymentActions.hasDeployments ? ( + deploymentActions.hasExpectedDeployments ? ( + )} +
+ + ); +}); diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index a876dafc..9f6f33b7 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -709,7 +709,10 @@ function buildOverviewData( total: 0, }; summary.total++; - if (deployment.status === "running") { + if ( + deployment.observedPhase === "running" || + deployment.observedPhase === "healthy" + ) { runningDeployments++; summary.running++; } @@ -788,7 +791,7 @@ function getServiceStatus( if (runningDeployments > 0) return { label: "Live", tone: "live" }; for (const deployment of service.deployments || []) { - if (deployment.status === "failed" || deployment.status === "rolled_back") { + if (deployment.observedPhase === "failed") { return { label: "Needs attention", tone: "warning" }; } } diff --git a/web/components/service/service-canvas.tsx b/web/components/service/service-canvas.tsx index 63221d28..1e825499 100644 --- a/web/components/service/service-canvas.tsx +++ b/web/components/service/service-canvas.tsx @@ -307,10 +307,10 @@ function ServiceCard({ p.externalPort, ); const hasInternalDns = service.deployments.some( - (d) => d.status === "running", + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", ); const runningCount = service.deployments.filter( - (d) => d.status === "running", + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", ).length; const hasEndpoints = diff --git a/web/components/service/service-layout-client.tsx b/web/components/service/service-layout-client.tsx index e81946e6..67fafb1c 100644 --- a/web/components/service/service-layout-client.tsx +++ b/web/components/service/service-layout-client.tsx @@ -66,7 +66,7 @@ export function ServiceLayoutClient({ svc.rollouts?.[0]?.status === "in_progress" || !!svc.migrationStatus || svc.deployments.some((d) => - IN_PROGRESS_DEPLOY_STATUSES.includes(d.status), + IN_PROGRESS_DEPLOY_STATUSES.includes(d.observedPhase), ); setHasActiveActivity(isActive); }, @@ -87,6 +87,8 @@ export function ServiceLayoutClient({ port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol, + tlsPassthrough: p.tlsPassthrough, })); const current = buildCurrentConfig( service, diff --git a/web/components/ui/canvas-wrapper.tsx b/web/components/ui/canvas-wrapper.tsx index 37702625..e863f06f 100644 --- a/web/components/ui/canvas-wrapper.tsx +++ b/web/components/ui/canvas-wrapper.tsx @@ -66,15 +66,26 @@ export function getStatusColor(status: string): StatusColors { } export function getStatusColorFromDeployments( - deployments: { status: string }[], + deployments: { observedPhase: string; runtimeDesiredState?: string }[], ): StatusColors { - const hasRunning = deployments.some((d) => d.status === "running"); + const hasRunning = deployments.some( + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", + ); const hasPending = deployments.some( - (d) => d.status === "pending" || d.status === "pulling", + (d) => + d.observedPhase === "pending" || + d.observedPhase === "pulling" || + d.observedPhase === "starting" || + d.observedPhase === "waking", + ); + const hasFailed = deployments.some((d) => d.observedPhase === "failed"); + const hasStopped = deployments.some( + (d) => + d.observedPhase === "stopped" || + d.observedPhase === "sleeping" || + d.runtimeDesiredState === "removed", ); - const hasFailed = deployments.some((d) => d.status === "failed"); - const hasStopped = deployments.some((d) => d.status === "stopped"); - const hasUnknown = deployments.some((d) => d.status === "unknown"); + const hasUnknown = deployments.some((d) => d.observedPhase === "unknown"); if (hasRunning) return statusColorMap.running; if (hasPending) return statusColorMap.pending; diff --git a/web/db/queries.ts b/web/db/queries.ts index 800895a4..7bf34803 100644 --- a/web/db/queries.ts +++ b/web/db/queries.ts @@ -228,7 +228,7 @@ export async function getServerServices(serverId: string) { const results = await db .selectDistinctOn([services.id], { deploymentId: deployments.id, - deploymentStatus: deployments.status, + deploymentStatus: deployments.observedPhase, serviceId: services.id, serviceName: services.name, serviceImage: services.image, diff --git a/web/db/schema.ts b/web/db/schema.ts index 6a4aee88..23883407 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -272,6 +272,7 @@ export type ContainerHealth = { export type AgentHealth = { version: string; uptimeSecs: number; + capabilities?: string[]; }; export type AgentUpgradeStatus = @@ -379,6 +380,16 @@ export const services = pgTable("services", { startCommand: text("start_command"), resourceCpuLimit: real("resource_cpu_limit").default(2), resourceMemoryLimitMb: integer("resource_memory_limit_mb").default(1024), + serverlessEnabled: boolean("serverless_enabled").notNull().default(false), + serverlessSleepAfterSeconds: integer("serverless_sleep_after_seconds") + .notNull() + .default(300), + serverlessWakeTimeoutSeconds: integer("serverless_wake_timeout_seconds") + .notNull() + .default(300), + serverlessMinReadyReplicas: integer("serverless_min_ready_replicas") + .notNull() + .default(1), deployedConfig: text("deployed_config"), deploymentSchedule: text("deployment_schedule"), lastScheduledDeploymentRunAt: timestamp("last_scheduled_deployment_run_at", { @@ -521,24 +532,32 @@ export const deployments = pgTable( .references(() => servers.id, { onDelete: "cascade" }), containerId: text("container_id"), ipAddress: text("ip_address"), - status: text("status", { + runtimeDesiredState: text("runtime_desired_state", { + enum: ["running", "stopped", "removed"], + }) + .notNull() + .default("running"), + trafficState: text("traffic_state", { + enum: ["candidate", "active", "draining", "inactive"], + }) + .notNull() + .default("candidate"), + observedPhase: text("observed_phase", { enum: [ "pending", "pulling", "starting", + "waking", "healthy", "running", - "draining", - "stopping", + "sleeping", "stopped", "failed", - "rolled_back", "unknown", ], }) .notNull() .default("pending"), - desired: boolean("desired").notNull().default(true), healthStatus: text("health_status", { enum: ["none", "starting", "healthy", "unhealthy"], }), @@ -554,6 +573,9 @@ export const deployments = pgTable( rolloutId: text("rollout_id"), previousDeploymentId: text("previous_deployment_id"), failedStage: text("failed_stage"), + serverlessWakeFailureCount: integer("serverless_wake_failure_count") + .notNull() + .default(0), createdAt: timestamp("created_at", { withTimezone: true }) .defaultNow() .notNull(), @@ -563,7 +585,11 @@ export const deployments = pgTable( index("deployments_rollout_id_idx").on(table.rolloutId), index("deployments_service_id_idx").on(table.serviceId), index("deployments_server_id_idx").on(table.serverId), - index("deployments_status_idx").on(table.status), + index("deployments_runtime_desired_state_idx").on( + table.runtimeDesiredState, + ), + index("deployments_traffic_state_idx").on(table.trafficState), + index("deployments_observed_phase_idx").on(table.observedPhase), ], ); diff --git a/web/db/types.ts b/web/db/types.ts index 04239040..e5674209 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -40,7 +40,7 @@ export type MemberInvitation = typeof memberInvitations.$inferSelect; export type MemberRole = User["role"]; export type InvitableMemberRole = MemberInvitation["role"]; -export type DeploymentStatus = NonNullable; +export type DeploymentStatus = NonNullable; export type HealthStatus = Deployment["healthStatus"]; export type RolloutStatus = NonNullable; export type BuildStatus = NonNullable; diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index a6d49beb..d2a4a969 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -18,9 +18,18 @@ import { getStartingHealthCheckFailureUpdate, getSteadyStateRecreateDecision, } from "@/lib/autoheal-policy"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { + type ObservedPhase, + isObservedActiveContainer, + isObservedReady, + markDeploymentFailedRemoved, + observedStartingPhases, + runtimeExpectedStates, +} from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; +import { getDeployedServerlessConfig } from "@/lib/service-config"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueWork } from "@/lib/work-queue"; @@ -36,6 +45,34 @@ type DeploymentError = { message: string; }; +export type ServerlessTransition = + | { type: "sleep"; deploymentId: string; containerId: string } + | { type: "wake_started"; deploymentId: string } + | { type: "wake_failed"; deploymentId: string; error: string }; + +export function shouldAttachReportedContainer(observedPhase: ObservedPhase) { + return (observedStartingPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function getStoppedContainerReportUpdate(deployment: { + runtimeDesiredState: string; +}) { + if (deployment.runtimeDesiredState === "stopped") { + return { + containerId: null, + observedPhase: "sleeping" as const, + healthStatus: null, + }; + } + + return { + observedPhase: "stopped" as const, + healthStatus: "none" as const, + }; +} + function isMigrationTargetStarting(status: string | null | undefined) { return status === "deploying_target" || status === "starting"; } @@ -79,12 +116,21 @@ async function applyDeploymentErrors( id: deployments.id, serviceId: deployments.serviceId, rolloutId: deployments.rolloutId, + observedPhase: deployments.observedPhase, + serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, + serverlessEnabled: services.serverlessEnabled, + serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, + serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, + serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, + stateful: services.stateful, + deployedConfig: services.deployedConfig, rolloutStatus: rollouts.status, serverName: servers.name, }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) - .innerJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) + .innerJoin(services, eq(deployments.serviceId, services.id)) + .leftJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) .where( and( eq(deployments.id, error.deploymentId), @@ -93,27 +139,64 @@ async function applyDeploymentErrors( ) .then((rows) => rows[0]); - if ( - !deployment || - !deployment.rolloutId || - deployment.rolloutStatus !== "in_progress" - ) { + if (!deployment) { + continue; + } + + const isServerlessWakeDeployment = deployment.observedPhase === "waking"; + const isActiveRolloutDeployment = + !isServerlessWakeDeployment && + deployment.rolloutId && + deployment.rolloutStatus === "in_progress"; + + if (!isActiveRolloutDeployment && !isServerlessWakeDeployment) { continue; } const updated = await db .update(deployments) - .set({ status: "failed", failedStage: "deploying" }) + .set( + isServerlessWakeDeployment + ? getServerlessWakeFailureUpdate({ + serverlessEnabled: + getDeployedServerlessConfig(deployment).enabled, + currentFailureCount: deployment.serverlessWakeFailureCount, + failedStage: "serverless_wake", + }) + : markDeploymentFailedRemoved("deploying"), + ) .where( and( eq(deployments.id, deployment.id), - inArray(deployments.status, ["pending", "pulling", "starting"]), + inArray( + deployments.observedPhase, + isServerlessWakeDeployment + ? ["waking"] + : ["pending", "pulling", "starting"], + ), ), ) .returning({ id: deployments.id }); if (updated.length === 0) continue; + if (isServerlessWakeDeployment) { + console.log( + `[serverless:wake] deployment ${deployment.id} failed on server ${deployment.serverName}: ${formatDeploymentError(error.message)}`, + ); + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "service", + parentId: deployment.serviceId, + }), + ); + continue; + } + + if (!deployment.rolloutId) continue; + await ingestRolloutLog( deployment.rolloutId, deployment.serviceId, @@ -132,6 +215,246 @@ async function applyDeploymentErrors( } } +async function applyServerlessTransitions( + serverId: string, + transitions: ServerlessTransition[], +) { + for (const transition of transitions) { + if (!isValidServerlessTransition(transition)) { + console.log(`[serverless:status] rejected malformed transition`); + continue; + } + + const deployment = await db + .select({ + id: deployments.id, + serviceId: deployments.serviceId, + serverId: deployments.serverId, + containerId: deployments.containerId, + runtimeDesiredState: deployments.runtimeDesiredState, + trafficState: deployments.trafficState, + observedPhase: deployments.observedPhase, + serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, + serverlessEnabled: services.serverlessEnabled, + serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, + serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, + serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, + stateful: services.stateful, + deployedConfig: services.deployedConfig, + serverIsProxy: servers.isProxy, + serverName: servers.name, + }) + .from(deployments) + .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where(eq(deployments.id, transition.deploymentId)) + .then((rows) => rows[0]); + + const invalidReason = getInvalidServerlessTransitionReason({ + serverId, + transition, + deployment, + }); + if (invalidReason || !deployment) { + console.log( + `[serverless:status] rejected ${transition.type} for deployment ${transition.deploymentId}: ${invalidReason}`, + ); + continue; + } + + if (transition.type === "sleep") { + const updated = await db + .update(deployments) + .set({ + runtimeDesiredState: "stopped", + observedPhase: "sleeping", + containerId: null, + healthStatus: null, + failedStage: null, + }) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + eq(deployments.containerId, transition.containerId), + eq(deployments.runtimeDesiredState, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} slept on proxy ${deployment.serverName}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + continue; + } + + if (transition.type === "wake_started") { + const updated = await db + .update(deployments) + .set({ + runtimeDesiredState: "running", + observedPhase: "waking", + containerId: null, + healthStatus: null, + failedStage: null, + }) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + eq(deployments.runtimeDesiredState, "stopped"), + eq(deployments.observedPhase, "sleeping"), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} wake started on proxy ${deployment.serverName}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + continue; + } + + const updated = await db + .update(deployments) + .set( + getServerlessWakeFailureUpdate({ + serverlessEnabled: getDeployedServerlessConfig(deployment).enabled, + currentFailureCount: deployment.serverlessWakeFailureCount, + failedStage: "serverless_wake", + }), + ) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + inArray(deployments.observedPhase, ["sleeping", "waking"]), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} wake failed on proxy ${deployment.serverName}: ${formatDeploymentError(transition.error)}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + } +} + +function isValidServerlessTransition( + transition: unknown, +): transition is ServerlessTransition { + if (!transition || typeof transition !== "object") return false; + const candidate = transition as ServerlessTransition; + if (typeof candidate.deploymentId !== "string" || !candidate.deploymentId) { + return false; + } + if (candidate.type === "sleep") { + return typeof candidate.containerId === "string" && !!candidate.containerId; + } + if (candidate.type === "wake_started") { + return true; + } + if (candidate.type === "wake_failed") { + return typeof candidate.error === "string" && !!candidate.error.trim(); + } + return false; +} + +function getInvalidServerlessTransitionReason({ + serverId, + transition, + deployment, +}: { + serverId: string; + transition: ServerlessTransition; + deployment: + | { + serverId: string; + containerId: string | null; + runtimeDesiredState: string; + trafficState: string; + observedPhase: string; + serverlessEnabled: boolean; + serverlessSleepAfterSeconds: number; + serverlessWakeTimeoutSeconds: number; + serverlessMinReadyReplicas: number; + deployedConfig: string | null; + serverIsProxy: boolean; + } + | undefined; +}) { + if (!deployment) return "deployment not found"; + if (deployment.serverId !== serverId) + return "deployment belongs to another server"; + if (!deployment.serverIsProxy) return "server is not a proxy"; + if (!getDeployedServerlessConfig(deployment).enabled) { + return "service is not serverless"; + } + if (deployment.runtimeDesiredState === "removed") { + return "deployment is removed"; + } + + if (transition.type === "sleep") { + if (deployment.runtimeDesiredState !== "running") { + return `deployment is not expected running (${deployment.runtimeDesiredState})`; + } + if (!["healthy", "running"].includes(deployment.observedPhase)) { + return `deployment is not sleepable from ${deployment.observedPhase}`; + } + if (deployment.containerId !== transition.containerId) { + return "stale containerId"; + } + } + + if ( + transition.type === "wake_started" && + deployment.observedPhase !== "sleeping" + ) { + return `deployment is not sleeping (${deployment.observedPhase})`; + } + + if ( + transition.type === "wake_failed" && + !["sleeping", "waking"].includes(deployment.observedPhase) + ) { + return `deployment is not waking or sleeping (${deployment.observedPhase})`; + } + + return null; +} + +async function emitDeploymentStatusChanged( + deploymentId: string, + serviceId: string, +) { + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deploymentId, + parentType: "service", + parentId: serviceId, + }), + ); +} + function formatDeploymentError(message: string): string { return message.trim().replace(/\s+/g, " ").slice(0, 1000); } @@ -156,6 +479,7 @@ export type StatusReport = { export async function applyStatusReport( serverId: string, report: StatusReport, + serverlessTransitions: ServerlessTransition[] = [], ) { const updateData: Record = { lastHeartbeat: new Date(), @@ -235,50 +559,44 @@ export async function applyStatusReport( }; await applyDeploymentErrors(serverId, report.deploymentErrors || []); + await applyServerlessTransitions(serverId, serverlessTransitions); const reportedDeploymentIds = report.containers .map((c) => c.deploymentId) .filter((id) => id !== ""); - const activeStatuses = [ - "starting", - "healthy", - "running", - "stopping", - ] as const; - const activeDeployments = await db .select({ id: deployments.id, containerId: deployments.containerId, - status: deployments.status, + runtimeDesiredState: deployments.runtimeDesiredState, + observedPhase: deployments.observedPhase, }) .from(deployments) .where( and( eq(deployments.serverId, serverId), isNotNull(deployments.containerId), - inArray(deployments.status, activeStatuses), ), ); for (const dep of activeDeployments) { if (!reportedDeploymentIds.includes(dep.id)) { - if (dep.status === "stopping") { + if (dep.runtimeDesiredState === "removed") { console.log( - `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} was stopping and container gone, deleting`, + `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} was removed and container gone, deleting`, ); await db .delete(deploymentPorts) .where(eq(deploymentPorts.deploymentId, dep.id)); await db.delete(deployments).where(eq(deployments.id, dep.id)); - } else { + } else if (isObservedActiveContainer(dep.observedPhase)) { console.log( `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} NOT reported, marking UNKNOWN`, ); await db .update(deployments) - .set({ status: "unknown", healthStatus: null }) + .set({ observedPhase: "unknown", healthStatus: null }) .where(eq(deployments.id, dep.id)); } } @@ -302,7 +620,6 @@ export async function applyStatusReport( } if (!deployment) { - const stuckStatuses = ["pending", "pulling"] as const; const [stuckDeployment] = await db .select() .from(deployments) @@ -310,7 +627,7 @@ export async function applyStatusReport( and( eq(deployments.serverId, serverId), isNull(deployments.containerId), - inArray(deployments.status, stuckStatuses), + inArray(deployments.observedPhase, observedStartingPhases), ), ); @@ -332,14 +649,15 @@ export async function applyStatusReport( .update(deployments) .set({ containerId: container.containerId, - status: newStatus, + observedPhase: newStatus, healthStatus: hasHealthCheck ? "starting" : "none", + serverlessWakeFailureCount: 0, }) .where(eq(deployments.id, stuckDeployment.id)); deployment = { ...stuckDeployment, - status: newStatus, + observedPhase: newStatus, containerId: container.containerId, healthStatus: hasHealthCheck ? "starting" : "none", }; @@ -379,7 +697,37 @@ export async function applyStatusReport( updateFields.containerId = container.containerId; } - if (deployment.status === "pending" || deployment.status === "pulling") { + if (container.status === "stopped") { + Object.assign(updateFields, getStoppedContainerReportUpdate(deployment)); + await db + .update(deployments) + .set(updateFields) + .where(eq(deployments.id, deployment.id)); + continue; + } + + if (container.status === "failed") { + updateFields.observedPhase = "failed"; + updateFields.healthStatus = null; + updateFields.failedStage ??= "container_failed"; + await db + .update(deployments) + .set(updateFields) + .where(eq(deployments.id, deployment.id)); + if (deployment.rolloutId) { + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "rollout", + parentId: deployment.rolloutId, + }), + ); + } + continue; + } + + if (shouldAttachReportedContainer(deployment.observedPhase)) { if (container.status !== "running") { continue; } @@ -392,12 +740,15 @@ export async function applyStatusReport( const hasHealthCheck = service?.healthCheckCmd != null; const newStatus = hasHealthCheck ? "starting" : "healthy"; - updateFields.status = newStatus; + updateFields.observedPhase = newStatus; + if (deployment.observedPhase === "waking") { + updateFields.serverlessWakeFailureCount = 0; + } if (hasHealthCheck) { updateFields.healthStatus = "starting"; } console.log( - `[health:attach] deployment ${deployment.id} transitioning from ${deployment.status} to ${newStatus}`, + `[health:attach] deployment ${deployment.id} transitioning from ${deployment.observedPhase} to ${newStatus}`, ); if (deployment.rolloutId) { @@ -437,12 +788,12 @@ export async function applyStatusReport( } } - if (deployment.status === "unknown") { + if (deployment.observedPhase === "unknown") { const newStatus = healthStatus === "healthy" || healthStatus === "none" ? "running" : "starting"; - updateFields.status = newStatus; + updateFields.observedPhase = newStatus; console.log( `[health:restore] deployment ${deployment.id} restored from unknown to ${newStatus}`, ); @@ -450,7 +801,7 @@ export async function applyStatusReport( const canAutoheal = container.status === "running" && - (deployment.status === "running" || deployment.status === "healthy"); + isObservedReady(deployment.observedPhase); const healthRecovered = healthStatus === "healthy" || healthStatus === "none"; if (canAutoheal && healthRecovered) { @@ -478,8 +829,10 @@ export async function applyStatusReport( console.log( `[autoheal] rollout deployment ${deployment.id} exceeded restart limit`, ); - Object.assign(updateFields, markDeploymentUndesired("failed")); - updateFields.failedStage = "autoheal"; + Object.assign( + updateFields, + markDeploymentFailedRemoved("autoheal"), + ); autohealFailed = true; } else { autohealRecreatePayload = prepareAutohealRecreatePayload({ @@ -537,7 +890,7 @@ export async function applyStatusReport( } if ( - deployment.status === "starting" && + deployment.observedPhase === "starting" && container.status === "running" && (healthStatus === "healthy" || healthStatus === "none") ) { @@ -548,9 +901,10 @@ export async function applyStatusReport( await db .update(deployments) .set({ - status: "healthy", + observedPhase: "healthy", autohealRestartCount: 0, autohealRecreateCount: 0, + serverlessWakeFailureCount: 0, }) .where(eq(deployments.id, deployment.id)); @@ -586,7 +940,10 @@ export async function applyStatusReport( } } - if (deployment.status === "starting" && healthStatus === "unhealthy") { + if ( + deployment.observedPhase === "starting" && + healthStatus === "unhealthy" + ) { console.log(`[health] deployment ${deployment.id} failed health check`); const rollout = deployment.rolloutId ? await db diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 96f83a03..c4838999 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -11,10 +11,16 @@ import { } from "@/db/schema"; import { getAllCertificatesForDomains } from "@/lib/acme-manager"; import { - dnsDeploymentStatuses, - expectedDeploymentStatuses, - routableDeploymentStatuses, + activeTrafficStates, + isDeploymentRoutable, + observedReadyPhases, + runtimeExpectedStates, } from "@/lib/deployment-status"; +import { + getDeployedServerlessConfig, + getDeployedServicePorts, + isDeployedServerlessService, +} from "@/lib/service-config"; import { slugify } from "@/lib/utils"; import { getWireGuardPeers } from "@/lib/wireguard"; @@ -22,12 +28,26 @@ type Server = typeof servers.$inferSelect; type Service = typeof services.$inferSelect; type Deployment = typeof deployments.$inferSelect; type ServicePort = typeof servicePorts.$inferSelect; +const SERVERLESS_GATEWAY_PORT = 18080; + +type RouteServicePort = Pick< + ServicePort, + | "id" + | "serviceId" + | "port" + | "isPublic" + | "domain" + | "protocol" + | "externalPort" + | "tlsPassthrough" +>; export type ExpectedContainer = { deploymentId: string; serviceId: string; serviceName: string; name: string; + desiredState: "running" | "stopped"; image: string; ipAddress: string | null; ports: Array<{ containerPort: number; hostPort: number }>; @@ -67,10 +87,30 @@ type UdpRoute = { externalPort: number; }; +export type ServerlessRouteUpstream = { + deploymentId: string; + serverId: string; + url: string; + local: boolean; + alwaysOn: boolean; +}; + +export type ServerlessRoute = { + serviceId: string; + domain: string; + port: number; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; + localDeploymentIds: string[]; + upstreams: ServerlessRouteUpstream[]; +}; + export type AgentExpectedState = { serverName: string; containers: ExpectedContainer[]; dns: { records: Array<{ name: string; ips: string[] }> }; + serverless: { routes: ServerlessRoute[] }; traefik: { httpRoutes: HttpRoute[]; tcpRoutes: TcpRoute[]; @@ -86,6 +126,7 @@ type DeploymentPortRow = { hostPort: number; containerPort: number; }; +const SERVERLESS_GATEWAY_CAPABILITY = "serverless_gateway"; type SecretRow = { serviceId: string; @@ -105,6 +146,17 @@ type RoutableDeploymentRow = { serverId: string; }; +type ServerlessDeploymentRow = { + id: string; + serviceId: string; + serverId: string; + ipAddress: string | null; + runtimeDesiredState: Deployment["runtimeDesiredState"]; + trafficState: Deployment["trafficState"]; + observedPhase: Deployment["observedPhase"]; + serverIsProxy: boolean; +}; + export async function getServer(serverId: string) { return db .select() @@ -120,12 +172,18 @@ export async function buildAgentExpectedState( const containers = await buildExpectedContainers(server.id); const dnsRecords = await buildDnsRecords(allServices); const traefikConfig = await buildTraefikConfig(server, allServices); + const serverless = await buildServerlessExpectedState( + server, + allServices, + containers, + ); const wireguardPeers = await getWireGuardPeers(server.id, server.privateIp); return { serverName: server.name, containers, dns: { records: dnsRecords }, + serverless, traefik: traefikConfig, wireguard: { peers: wireguardPeers }, }; @@ -144,37 +202,40 @@ async function buildExpectedContainers( .where( and( eq(deployments.serverId, serverId), - eq(deployments.desired, true), - inArray(deployments.status, expectedDeploymentStatuses), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), ); const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); if (serviceIds.length === 0) return []; - const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all([ - db - .select() - .from(services) - .where(and(inArray(services.id, serviceIds), isNull(services.deletedAt))), - fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), - db - .select({ - serviceId: secrets.serviceId, - key: secrets.key, - encryptedValue: secrets.encryptedValue, - }) - .from(secrets) - .where(inArray(secrets.serviceId, serviceIds)), - db - .select({ - serviceId: serviceVolumes.serviceId, - name: serviceVolumes.name, - containerPath: serviceVolumes.containerPath, - }) - .from(serviceVolumes) - .where(inArray(serviceVolumes.serviceId, serviceIds)), - ]); + const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all( + [ + db + .select() + .from(services) + .where( + and(inArray(services.id, serviceIds), isNull(services.deletedAt)), + ), + fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), + db + .select({ + serviceId: secrets.serviceId, + key: secrets.key, + encryptedValue: secrets.encryptedValue, + }) + .from(secrets) + .where(inArray(secrets.serviceId, serviceIds)), + db + .select({ + serviceId: serviceVolumes.serviceId, + name: serviceVolumes.name, + containerPath: serviceVolumes.containerPath, + }) + .from(serviceVolumes) + .where(inArray(serviceVolumes.serviceId, serviceIds)), + ], + ); return buildExpectedContainersFromRows({ deployments: serverDeployments, @@ -199,6 +260,15 @@ async function fetchDeploymentPorts(deploymentIds: string[]) { .where(inArray(deploymentPorts.deploymentId, deploymentIds)); } +async function fetchServicePorts(serviceIds: string[]) { + if (serviceIds.length === 0) return []; + + return db + .select() + .from(servicePorts) + .where(inArray(servicePorts.serviceId, serviceIds)); +} + export function buildExpectedContainersFromRows({ deployments: deploymentRows, services: serviceRows, @@ -212,8 +282,13 @@ export function buildExpectedContainersFromRows({ secrets: SecretRow[]; volumes: VolumeRow[]; }): ExpectedContainer[] { - const servicesById = new Map(serviceRows.map((service) => [service.id, service])); - const portsByDeploymentId = groupBy(deploymentPortRows, (port) => port.deploymentId); + const servicesById = new Map( + serviceRows.map((service) => [service.id, service]), + ); + const portsByDeploymentId = groupBy( + deploymentPortRows, + (port) => port.deploymentId, + ); const secretsByServiceId = groupBy(secretRows, (secret) => secret.serviceId); const volumesByServiceId = groupBy(volumeRows, (volume) => volume.serviceId); @@ -230,6 +305,8 @@ export function buildExpectedContainersFromRows({ serviceId: dep.serviceId, serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, + desiredState: + dep.runtimeDesiredState === "stopped" ? "stopped" : "running", image: normalizeImage(service.image), ipAddress: dep.ipAddress, ports: (portsByDeploymentId.get(dep.id) ?? []) @@ -256,6 +333,146 @@ export function buildExpectedContainersFromRows({ }); } +async function buildServerlessExpectedState( + server: Server, + allServices: Service[], + containers: ExpectedContainer[], +): Promise<{ routes: ServerlessRoute[] }> { + if ( + !server.isProxy || + !hasAgentCapability(server, SERVERLESS_GATEWAY_CAPABILITY) + ) { + return { routes: [] }; + } + + const serverlessServices = allServices.filter(isDeployedServerlessService); + if (serverlessServices.length === 0) return { routes: [] }; + + const serviceIds = serverlessServices.map((service) => service.id); + const [ports, deploymentRows] = await Promise.all([ + db + .select() + .from(servicePorts) + .where(inArray(servicePorts.serviceId, serviceIds)), + db + .select({ + id: deployments.id, + serviceId: deployments.serviceId, + serverId: deployments.serverId, + ipAddress: deployments.ipAddress, + runtimeDesiredState: deployments.runtimeDesiredState, + trafficState: deployments.trafficState, + observedPhase: deployments.observedPhase, + serverIsProxy: servers.isProxy, + }) + .from(deployments) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where( + and( + inArray(deployments.serviceId, serviceIds), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + ), + ), + ]); + + return { + routes: buildServerlessRoutesFromRows({ + serverId: server.id, + services: serverlessServices, + ports, + deployments: deploymentRows, + containers, + }), + }; +} + +export function buildServerlessRoutesFromRows({ + serverId, + services: serviceRows, + ports, + deployments: deploymentRows, + containers, +}: { + serverId: string; + services: Service[]; + ports: RouteServicePort[]; + deployments: ServerlessDeploymentRow[]; + containers: ExpectedContainer[]; +}): ServerlessRoute[] { + const deploymentsByServiceId = groupBy( + deploymentRows, + (deployment) => deployment.serviceId, + ); + const portsByServiceId = groupBy(ports, (port) => port.serviceId); + const expectedDeploymentIds = new Set( + containers.map((container) => container.deploymentId), + ); + + return serviceRows + .flatMap((service) => + getRuntimeServicePortsForRoutes( + service, + portsByServiceId.get(service.id) ?? [], + ).map((port) => ({ service, port })), + ) + .sort((a, b) => compareServicePorts(a.port, b.port)) + .flatMap(({ service, port }) => { + if (!port.isPublic || port.protocol !== "http" || !port.domain) { + return []; + } + + const serviceDeployments = deploymentsByServiceId.get(service.id) ?? []; + if (!serviceDeployments.some((deployment) => deployment.serverIsProxy)) { + return []; + } + + const localDeploymentIds = serviceDeployments + .filter( + (deployment) => + deployment.serverId === serverId && + deployment.serverIsProxy && + deployment.trafficState === "active" && + expectedDeploymentIds.has(deployment.id), + ) + .map((deployment) => deployment.id) + .sort(); + + const upstreams = serviceDeployments + .filter( + (deployment) => + deployment.ipAddress && + deployment.runtimeDesiredState === "running" && + isDeploymentRoutable(deployment), + ) + .map((deployment) => ({ + deploymentId: deployment.id, + serverId: deployment.serverId, + url: `${deployment.ipAddress}:${port.port}`, + local: deployment.serverId === serverId, + alwaysOn: !deployment.serverIsProxy, + })) + .sort(compareServerlessUpstreams); + + return [ + { + serviceId: service.id, + domain: port.domain, + port: port.port, + sleepAfterSeconds: + getDeployedServerlessConfig(service).sleepAfterSeconds, + wakeTimeoutSeconds: + getDeployedServerlessConfig(service).wakeTimeoutSeconds, + minReadyReplicas: Math.max( + 1, + getDeployedServerlessConfig(service).minReadyReplicas, + ), + localDeploymentIds, + upstreams, + }, + ]; + }); +} + async function buildDnsRecords(allServices: Service[]) { const serviceIds = allServices.map((service) => service.id); if (serviceIds.length === 0) return []; @@ -269,12 +486,16 @@ async function buildDnsRecords(allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, dnsDeploymentStatuses), + eq(deployments.runtimeDesiredState, "running"), + inArray(deployments.trafficState, activeTrafficStates), + inArray(deployments.observedPhase, observedReadyPhases), ), ); - const ipsByServiceId = groupBy(dnsDeployments, (deployment) => deployment.serviceId); + const ipsByServiceId = groupBy( + dnsDeployments, + (deployment) => deployment.serviceId, + ); return allServices .flatMap((service) => { @@ -296,35 +517,89 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { if (!server.isProxy) return emptyConfig; const serviceIds = allServices.map((service) => service.id); - const [ports, routableDeployments] = await Promise.all([ - serviceIds.length > 0 - ? db - .select() - .from(servicePorts) - .where(inArray(servicePorts.serviceId, serviceIds)) - : Promise.resolve([]), - serviceIds.length > 0 - ? db - .select({ - serviceId: deployments.serviceId, - ipAddress: deployments.ipAddress, - serverId: deployments.serverId, - }) - .from(deployments) - .where( - and( - inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, routableDeploymentStatuses), - ), - ) - : Promise.resolve([]), - ]); + const supportsServerlessGateway = hasAgentCapability( + server, + SERVERLESS_GATEWAY_CAPABILITY, + ); + const [ports, routableDeployments, proxyHostedServerlessDeployments] = + await Promise.all([ + serviceIds.length > 0 + ? db + .select() + .from(servicePorts) + .where(inArray(servicePorts.serviceId, serviceIds)) + : Promise.resolve([]), + serviceIds.length > 0 + ? db + .select({ + serviceId: deployments.serviceId, + ipAddress: deployments.ipAddress, + serverId: deployments.serverId, + }) + .from(deployments) + .where( + and( + inArray(deployments.serviceId, serviceIds), + eq(deployments.runtimeDesiredState, "running"), + inArray(deployments.trafficState, activeTrafficStates), + inArray(deployments.observedPhase, observedReadyPhases), + ), + ) + : Promise.resolve([]), + serviceIds.length > 0 + ? db + .select({ + serviceId: deployments.serviceId, + serverId: deployments.serverId, + }) + .from(deployments) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where( + and( + inArray(deployments.serviceId, serviceIds), + eq(servers.isProxy, true), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + inArray(deployments.trafficState, activeTrafficStates), + ), + ) + : Promise.resolve([]), + ]); + const proxyHostedServerlessServiceIds = new Set( + proxyHostedServerlessDeployments.map((deployment) => deployment.serviceId), + ); + const localProxyHostedServerlessServiceIds = new Set( + proxyHostedServerlessDeployments + .filter((deployment) => deployment.serverId === server.id) + .map((deployment) => deployment.serviceId), + ); + const serverlessProxyRoutedServiceIds = new Set( + allServices + .filter( + (service) => + isDeployedServerlessService(service) && + proxyHostedServerlessServiceIds.has(service.id), + ) + .map((service) => service.id), + ); + const serverlessServiceIds = new Set( + [...serverlessProxyRoutedServiceIds].filter( + (serviceId) => + supportsServerlessGateway && + localProxyHostedServerlessServiceIds.has(serviceId), + ), + ); + const serverlessRouteSuppressedServiceIds = new Set( + [...serverlessProxyRoutedServiceIds].filter( + (serviceId) => !serverlessServiceIds.has(serviceId), + ), + ); const routes = buildTraefikRoutes({ serverId: server.id, - ports, + ports: buildRuntimeRoutePorts(allServices, ports), routableDeployments, + serverlessServiceIds, + serverlessRouteSuppressedServiceIds, }); const routedDomains = routes.httpRoutes.map((r) => r.domain); const certificates = await getAllCertificatesForDomains(routedDomains); @@ -343,14 +618,22 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { }; } +function hasAgentCapability(server: Server, capability: string) { + return server.agentHealth?.capabilities?.includes(capability) === true; +} + export function buildTraefikRoutes({ serverId, ports, routableDeployments, + serverlessServiceIds = new Set(), + serverlessRouteSuppressedServiceIds = new Set(), }: { serverId: string; - ports: ServicePort[]; + ports: RouteServicePort[]; routableDeployments: RoutableDeploymentRow[]; + serverlessServiceIds?: Set; + serverlessRouteSuppressedServiceIds?: Set; }) { const httpRoutes: HttpRoute[] = []; const tcpRoutes: TcpRoute[] = []; @@ -364,6 +647,22 @@ export function buildTraefikRoutes({ const serviceDeployments = deploymentsByServiceId.get(port.serviceId) ?? []; if (port.isPublic && port.protocol === "http" && port.domain) { + if (serverlessServiceIds.has(port.serviceId)) { + httpRoutes.push({ + id: port.domain, + domain: port.domain, + upstreams: [ + { url: `127.0.0.1:${SERVERLESS_GATEWAY_PORT}`, weight: 1 }, + ], + serviceId: port.serviceId, + }); + continue; + } + + if (serverlessRouteSuppressedServiceIds.has(port.serviceId)) { + continue; + } + const localDeployments = serviceDeployments.filter( (d) => d.serverId === serverId && d.ipAddress, ); @@ -372,14 +671,18 @@ export function buildTraefikRoutes({ ); const upstreams = [ - ...localDeployments.map((d) => ({ - url: `${d.ipAddress}:${port.port}`, - weight: 5, - })).sort((a, b) => a.url.localeCompare(b.url)), - ...remoteDeployments.map((d) => ({ - url: `${d.ipAddress}:${port.port}`, - weight: 1, - })).sort((a, b) => a.url.localeCompare(b.url)), + ...localDeployments + .map((d) => ({ + url: `${d.ipAddress}:${port.port}`, + weight: 5, + })) + .sort((a, b) => a.url.localeCompare(b.url)), + ...remoteDeployments + .map((d) => ({ + url: `${d.ipAddress}:${port.port}`, + weight: 1, + })) + .sort((a, b) => a.url.localeCompare(b.url)), ]; if (upstreams.length > 0) { @@ -433,7 +736,9 @@ function buildHealthCheck(service: Service): ExpectedContainer["healthCheck"] { function buildEnv(secretRows: SecretRow[]) { const env: Record = {}; - for (const secret of secretRows.slice().sort((a, b) => a.key.localeCompare(b.key))) { + for (const secret of secretRows + .slice() + .sort((a, b) => a.key.localeCompare(b.key))) { env[secret.key] = secret.encryptedValue; } return env; @@ -447,6 +752,15 @@ function upstreamUrls(deployments: RoutableDeploymentRow[], port: number) { .sort(); } +function compareServerlessUpstreams( + a: ServerlessRouteUpstream, + b: ServerlessRouteUpstream, +) { + if (a.local !== b.local) return a.local ? -1 : 1; + if (a.alwaysOn !== b.alwaysOn) return a.alwaysOn ? -1 : 1; + return a.url.localeCompare(b.url); +} + function normalizeImage(image: string) { if (!image.includes("/")) { return `docker.io/library/${image}`; @@ -471,7 +785,48 @@ function groupBy(items: T[], keyFn: (item: T) => K) { return groups; } -function compareServicePorts(a: ServicePort, b: ServicePort) { +export function buildRuntimeRoutePorts( + serviceRows: Service[], + portRows: ServicePort[], +): RouteServicePort[] { + const portsByServiceId = groupBy(portRows, (port) => port.serviceId); + return serviceRows.flatMap((service) => + getRuntimeServicePortsForRoutes( + service, + portsByServiceId.get(service.id) ?? [], + ), + ); +} + +function getRuntimeServicePortsForRoutes( + service: Service, + livePorts: RouteServicePort[], +): RouteServicePort[] { + const ports = isDeployedServerlessService(service) + ? getDeployedServicePorts(service, livePorts) + : livePorts; + return ports.map((port, index) => { + const routePort = port as Partial; + return { + id: + typeof routePort.id === "string" + ? routePort.id + : `${service.id}:deployed:${index}`, + serviceId: service.id, + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? "http", + externalPort: + typeof routePort.externalPort === "number" + ? routePort.externalPort + : null, + tlsPassthrough: routePort.tlsPassthrough ?? false, + }; + }); +} + +function compareServicePorts(a: RouteServicePort, b: RouteServicePort) { return ( a.serviceId.localeCompare(b.serviceId) || a.protocol.localeCompare(b.protocol) || diff --git a/web/lib/autoheal-policy.ts b/web/lib/autoheal-policy.ts index eecfcae3..d93d59ae 100644 --- a/web/lib/autoheal-policy.ts +++ b/web/lib/autoheal-policy.ts @@ -23,8 +23,15 @@ export function getStartingHealthCheckFailureUpdate({ return { update: { - status: "failed" as const, - desired: !isRolloutDeployment && !recreateLimitReached, + observedPhase: "failed" as const, + runtimeDesiredState: + !isRolloutDeployment && !recreateLimitReached + ? ("running" as const) + : ("removed" as const), + trafficState: + !isRolloutDeployment && !recreateLimitReached + ? ("active" as const) + : ("inactive" as const), failedStage, ...(isRolloutDeployment ? {} @@ -54,8 +61,9 @@ export function getSteadyStateRecreateDecision({ return { limitReached: true, updateFields: { - status: "failed" as const, - desired: false, + observedPhase: "failed" as const, + runtimeDesiredState: "removed" as const, + trafficState: "inactive" as const, failedStage: "autoheal_recreate_limit", unhealthyReportCount: 0, autohealRestartCount: 0, @@ -67,8 +75,9 @@ export function getSteadyStateRecreateDecision({ return { limitReached: false, updateFields: { - status: "failed" as const, - desired: true, + observedPhase: "failed" as const, + runtimeDesiredState: "running" as const, + trafficState: "active" as const, failedStage: "autoheal_recreate", unhealthyReportCount: 0, autohealRestartCount: 0, diff --git a/web/lib/backups/trigger-backup.ts b/web/lib/backups/trigger-backup.ts index 0b0c29ed..abb79dda 100644 --- a/web/lib/backups/trigger-backup.ts +++ b/web/lib/backups/trigger-backup.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { @@ -54,7 +54,7 @@ export async function triggerBackup({ .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/cli-service.ts b/web/lib/cli-service.ts index 3d1222e3..e41ca193 100644 --- a/web/lib/cli-service.ts +++ b/web/lib/cli-service.ts @@ -761,7 +761,7 @@ export async function getManifestStatus(identity: ManifestIdentity) { const serviceDeployments = await db .select({ id: deployments.id, - status: deployments.status, + status: deployments.observedPhase, serverId: deployments.serverId, createdAt: deployments.createdAt, }) diff --git a/web/lib/deployment-status.ts b/web/lib/deployment-status.ts index a1aa2cd4..ed4619ad 100644 --- a/web/lib/deployment-status.ts +++ b/web/lib/deployment-status.ts @@ -1,42 +1,95 @@ import type { deployments } from "@/db/schema"; -export type DeploymentStatus = typeof deployments.$inferSelect.status; +export type RuntimeDesiredState = + typeof deployments.$inferSelect.runtimeDesiredState; +export type TrafficState = typeof deployments.$inferSelect.trafficState; +export type ObservedPhase = typeof deployments.$inferSelect.observedPhase; -export type UndesiredDeploymentStatus = Extract< - DeploymentStatus, - "stopping" | "stopped" | "failed" | "rolled_back" ->; - -type DeploymentStatusCapabilities = { - expected: boolean; - routable: boolean; - dns: boolean; +export type DeploymentState = { + runtimeDesiredState: RuntimeDesiredState; + trafficState: TrafficState; + observedPhase: ObservedPhase; }; -const deploymentStatusCapabilities = { - pending: { expected: true, routable: false, dns: false }, - pulling: { expected: true, routable: false, dns: false }, - starting: { expected: true, routable: false, dns: false }, - healthy: { expected: true, routable: true, dns: true }, - running: { expected: true, routable: true, dns: true }, - draining: { expected: true, routable: false, dns: false }, - stopping: { expected: false, routable: false, dns: false }, - stopped: { expected: false, routable: false, dns: false }, - failed: { expected: false, routable: false, dns: false }, - rolled_back: { expected: false, routable: false, dns: false }, - unknown: { expected: true, routable: false, dns: false }, -} satisfies Record; - -export const expectedDeploymentStatuses = statusesWithCapability("expected"); -export const routableDeploymentStatuses = statusesWithCapability("routable"); -export const dnsDeploymentStatuses = statusesWithCapability("dns"); - -export function markDeploymentUndesired(status: UndesiredDeploymentStatus) { - return { status, desired: false }; -} - -function statusesWithCapability(capability: keyof DeploymentStatusCapabilities) { - return Object.entries(deploymentStatusCapabilities) - .filter(([, capabilities]) => capabilities[capability]) - .map(([status]) => status as DeploymentStatus); +export const runtimeExpectedStates = ["running", "stopped"] as const satisfies + readonly RuntimeDesiredState[]; + +export const activeTrafficStates = ["active"] as const satisfies + readonly TrafficState[]; + +export const observedReadyPhases = ["healthy", "running"] as const satisfies + readonly ObservedPhase[]; + +export const observedStartingPhases = [ + "pending", + "pulling", + "starting", + "waking", +] as const satisfies readonly ObservedPhase[]; + +export const observedActiveContainerPhases = [ + "starting", + "healthy", + "running", +] as const satisfies readonly ObservedPhase[]; + +export function isRuntimeExpected( + runtimeDesiredState: RuntimeDesiredState, +): boolean { + return runtimeDesiredState !== "removed"; +} + +export function isObservedReady(observedPhase: ObservedPhase): boolean { + return (observedReadyPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isObservedStarting(observedPhase: ObservedPhase): boolean { + return (observedStartingPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isObservedActiveContainer( + observedPhase: ObservedPhase, +): boolean { + return (observedActiveContainerPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isTrafficActive(trafficState: TrafficState): boolean { + return trafficState === "active"; +} + +export function isDeploymentExpected( + deployment: Pick, +): boolean { + return isRuntimeExpected(deployment.runtimeDesiredState); +} + +export function isDeploymentRoutable( + deployment: Pick, +): boolean { + return ( + isTrafficActive(deployment.trafficState) && + isObservedReady(deployment.observedPhase) + ); +} + +export function markDeploymentRemoved() { + return { + runtimeDesiredState: "removed" as const, + trafficState: "inactive" as const, + }; +} + +export function markDeploymentFailedRemoved(failedStage: string) { + return { + ...markDeploymentRemoved(), + observedPhase: "failed" as const, + healthStatus: null, + failedStage, + }; } diff --git a/web/lib/inngest/functions/migration-workflow.ts b/web/lib/inngest/functions/migration-workflow.ts index a5dcab97..bdc03276 100644 --- a/web/lib/inngest/functions/migration-workflow.ts +++ b/web/lib/inngest/functions/migration-workflow.ts @@ -9,7 +9,7 @@ import { volumeBackups, } from "@/db/schema"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentRemoved } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -65,7 +65,7 @@ export const migrationWorkflow = inngest.createFunction( await db .update(deployments) - .set(markDeploymentUndesired("stopped")) + .set(markDeploymentRemoved()) .where(eq(deployments.id, sourceDeploymentId)); }); diff --git a/web/lib/inngest/functions/on-deployment-failed.ts b/web/lib/inngest/functions/on-deployment-failed.ts index b863cf82..61969e25 100644 --- a/web/lib/inngest/functions/on-deployment-failed.ts +++ b/web/lib/inngest/functions/on-deployment-failed.ts @@ -17,7 +17,7 @@ export const onDeploymentFailed = inngest.createFunction( id: deployments.id, rolloutId: deployments.rolloutId, serviceId: deployments.serviceId, - status: deployments.status, + observedPhase: deployments.observedPhase, failedStage: deployments.failedStage, }) .from(deployments) @@ -27,7 +27,7 @@ export const onDeploymentFailed = inngest.createFunction( if ( !deployment?.rolloutId || - (deployment.status !== "failed" && deployment.status !== "rolled_back") + deployment.observedPhase !== "failed" ) { return; } diff --git a/web/lib/inngest/functions/restore-trigger-workflow.ts b/web/lib/inngest/functions/restore-trigger-workflow.ts index e217daf1..6409cfa1 100644 --- a/web/lib/inngest/functions/restore-trigger-workflow.ts +++ b/web/lib/inngest/functions/restore-trigger-workflow.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { deployments, volumeBackups } from "@/db/schema"; @@ -51,7 +51,7 @@ export const restoreTriggerWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 055c7dc9..abe53c72 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -13,7 +13,7 @@ import { serviceVolumes, } from "@/db/schema"; import { getCertificate, issueCertificate } from "@/lib/acme-manager"; -import { buildCurrentConfig } from "@/lib/service-config"; +import { buildCurrentConfig, getDeployedStateful } from "@/lib/service-config"; import { assignContainerIp } from "@/lib/wireguard"; import { enqueueWork } from "@/lib/work-queue"; @@ -33,6 +33,21 @@ export type DeploymentContext = { isRollingUpdate: boolean; }; +export function isActiveDeploymentForRollout( + deployment: { trafficState: string }, + service: { + deployedConfig?: string | null; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; + }, +) { + void service; + return deployment.trafficState === "active"; +} + export function normalizeImage(image: string): string { if (!image.includes("/")) { return `docker.io/library/${image}`; @@ -167,13 +182,18 @@ export async function validateServers( export async function prepareRollingUpdate( serviceId: string, ): Promise<{ deploymentIds: string[] }> { + const service = await getService(serviceId); + if (!service) { + return { deploymentIds: [] }; + } + const existingDeployments = await db .select() .from(deployments) .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter( - (d) => d.status === "running" || d.status === "healthy", + (d) => isActiveDeploymentForRollout(d, service), ); return { deploymentIds: runningDeployments.map((d) => d.id) }; @@ -188,7 +208,7 @@ export async function cleanupTerminalDeployments( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["rolled_back", "stopped", "failed"]), + eq(deployments.runtimeDesiredState, "removed"), ), ); @@ -344,7 +364,9 @@ export async function createDeploymentRecords( serviceId, serverId: server.id, ipAddress, - status: "pending", + runtimeDesiredState: "running", + trafficState: "candidate", + observedPhase: "pending", rolloutId, }); @@ -431,6 +453,8 @@ export async function saveDeployedConfig( port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol, + tlsPassthrough: p.tlsPassthrough, })); const updatedService = await getService(serviceId); @@ -454,7 +478,7 @@ export async function checkForRollingUpdate( serviceId: string, ): Promise { const service = await getService(serviceId); - if (!service || service.stateful) { + if (!service || getDeployedStateful(service)) { return false; } @@ -464,7 +488,7 @@ export async function checkForRollingUpdate( .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter( - (d) => d.status === "running" || d.status === "healthy", + (d) => isActiveDeploymentForRollout(d, service), ); return runningDeployments.length > 0; diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 955bb089..f78c3857 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -1,9 +1,39 @@ -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, ne } from "drizzle-orm"; import { db } from "@/db"; import { deployments, rollouts } from "@/db/schema"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; import { sendDeploymentFailureAlert } from "@/lib/email"; +export function shouldRollBackDeploymentState(deployment: { + trafficState: string; + runtimeDesiredState: string; +}) { + void deployment.trafficState; + return deployment.runtimeDesiredState !== "removed"; +} + +export function shouldRestoreDrainingDeployment(deployment: { + trafficState: string; + runtimeDesiredState: string; +}) { + return ( + deployment.trafficState === "draining" && + deployment.runtimeDesiredState !== "removed" + ); +} + +export async function restoreDrainingDeploymentsForRollback(serviceId: string) { + await db + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + ), + ); +} + export async function handleRolloutFailure( rolloutId: string, serviceId: string, @@ -41,31 +71,16 @@ export async function handleRolloutFailure( const serverId = rolloutDeployments[0].serverId; if (isRollingUpdate) { - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - ), - ); + await restoreDrainingDeploymentsForRollback(serviceId); } await db .update(deployments) - .set({ ...markDeploymentUndesired("rolled_back"), failedStage: reason }) + .set(markDeploymentFailedRemoved(reason)) .where( and( eq(deployments.rolloutId, rolloutId), - inArray(deployments.status, [ - "pending", - "pulling", - "starting", - "healthy", - "running", - "failed", - ]), + ne(deployments.runtimeDesiredState, "removed"), ), ); diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index b57cc455..d2f6d5ee 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -385,23 +385,6 @@ export const rolloutWorkflow = inngest.createFunction( return result; }); - await step.run("save-deployed-config", async () => { - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - - const serverMap = await validateServers(placements); - - await saveDeployedConfig(serviceId, { - service, - placements, - serverMap, - totalReplicas, - isRollingUpdate, - }); - }); - await step.run("start-health-check", async () => { const service = await getService(serviceId); const hasHealthCheck = service?.healthCheckCmd != null; @@ -431,7 +414,7 @@ export const rolloutWorkflow = inngest.createFunction( .where( and( inArray(deployments.id, deploymentIds), - inArray(deployments.status, ["healthy", "running"]), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ); @@ -460,7 +443,7 @@ export const rolloutWorkflow = inngest.createFunction( const deploymentStates = await db .select({ id: deployments.id, - status: deployments.status, + observedPhase: deployments.observedPhase, serverName: servers.name, }) .from(deployments) @@ -469,7 +452,8 @@ export const rolloutWorkflow = inngest.createFunction( return deploymentStates.filter( (deployment) => - deployment.status !== "healthy" && deployment.status !== "running", + deployment.observedPhase !== "healthy" && + deployment.observedPhase !== "running", ); }, ); @@ -505,34 +489,37 @@ export const rolloutWorkflow = inngest.createFunction( } await step.run("start-dns-sync", async () => { - await db - .update(rollouts) - .set({ currentStage: "dns_sync" }) - .where(eq(rollouts.id, rolloutId)); + await db.transaction(async (tx) => { + await tx + .update(rollouts) + .set({ currentStage: "dns_sync" }) + .where(eq(rollouts.id, rolloutId)); - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.rolloutId, rolloutId), - eq(deployments.status, "healthy"), - ), - ); + await tx + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.rolloutId, rolloutId), + eq(deployments.trafficState, "candidate"), + inArray(deployments.observedPhase, ["healthy", "running"]), + ), + ); - await db - .update(deployments) - .set({ status: "draining" }) - .where( - and( - eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), - or( - ne(deployments.rolloutId, rolloutId), - isNull(deployments.rolloutId), + await tx + .update(deployments) + .set({ trafficState: "draining" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "active"), + or( + ne(deployments.rolloutId, rolloutId), + isNull(deployments.rolloutId), + ), ), - ), - ); + ); + }); await ingestRolloutLog( rolloutId, @@ -598,28 +585,66 @@ export const rolloutWorkflow = inngest.createFunction( if (isRollingUpdate) { await step.run("stop-old-deployments", async () => { - const stoppedDeployments = await db + const stoppedDeploymentsWithoutContainers = await db + .update(deployments) + .set({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + isNull(deployments.containerId), + ), + ) + .returning({ id: deployments.id }); + + const stoppingDeployments = await db .update(deployments) - .set({ status: "stopping", desired: false }) + .set({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), + eq(deployments.trafficState, "draining"), ), ) .returning({ id: deployments.id }); - if (stoppedDeployments.length > 0) { + const stoppedCount = + stoppedDeploymentsWithoutContainers.length + + stoppingDeployments.length; + if (stoppedCount > 0) { await ingestRolloutLog( rolloutId, serviceId, "dns_sync", - `Stopping ${stoppedDeployments.length} old deployment(s) after DNS sync`, + `Stopping ${stoppedCount} old deployment(s) after DNS sync`, ); } }); } + await step.run("save-deployed-config", async () => { + const service = await getService(serviceId); + if (!service) { + throw new Error("Service not found"); + } + + const serverMap = await validateServers(placements); + + await saveDeployedConfig(serviceId, { + service, + placements, + serverMap, + totalReplicas, + isRollingUpdate, + }); + }); + await step.run("complete-rollout", async () => { await db .update(rollouts) diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index d13911ab..60a15634 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -13,7 +13,7 @@ import { volumeBackups, } from "@/db/schema"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentRemoved } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -81,7 +81,7 @@ export const serviceDeletionWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), + inArray(deployments.observedPhase, ["running", "healthy"]), ), ) .then((r) => r[0]); @@ -237,16 +237,12 @@ export const serviceDeletionWorkflow = inngest.createFunction( .where(eq(deployments.serviceId, serviceId)); for (const deployment of allDeployments) { - if ( - (deployment.status === "running" || - deployment.status === "healthy") && - deployment.containerId - ) { - await db - .update(deployments) - .set(markDeploymentUndesired("stopping")) - .where(eq(deployments.id, deployment.id)); + await db + .update(deployments) + .set(markDeploymentRemoved()) + .where(eq(deployments.id, deployment.id)); + if (deployment.containerId) { await enqueueWork(deployment.serverId, "stop", { deploymentId: deployment.id, containerId: deployment.containerId, @@ -470,7 +466,7 @@ export const serviceRestoreWorkflow = inngest.createFunction( async () => { return db .select({ - status: deployments.status, + observedPhase: deployments.observedPhase, failedStage: deployments.failedStage, }) .from(deployments) @@ -485,11 +481,11 @@ export const serviceRestoreWorkflow = inngest.createFunction( const healthyDeployment = restoredDeployments.find( (deployment) => - deployment.status === "healthy" || deployment.status === "running", + deployment.observedPhase === "healthy" || + deployment.observedPhase === "running", ); const failedDeployment = restoredDeployments.find( - (deployment) => - deployment.status === "failed" || deployment.status === "rolled_back", + (deployment) => deployment.observedPhase === "failed", ); if (!healthyDeployment || failedDeployment) { diff --git a/web/lib/migrations.ts b/web/lib/migrations.ts index 3d041c08..ecfe684f 100644 --- a/web/lib/migrations.ts +++ b/web/lib/migrations.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { deployments, services, serviceVolumes } from "@/db/schema"; @@ -53,7 +53,7 @@ export async function startMigrationInternal( .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/project-deletion.ts b/web/lib/project-deletion.ts new file mode 100644 index 00000000..5b11f732 --- /dev/null +++ b/web/lib/project-deletion.ts @@ -0,0 +1,11 @@ +import type { deployments } from "@/db/schema"; + +type ProjectDeletionDeployment = Pick< + typeof deployments.$inferSelect, + "runtimeDesiredState" +>; + +export function blocksProjectDeletion(deployment: ProjectDeletionDeployment) { + // Sleeping serverless deployments still have runtime intent and can wake. + return deployment.runtimeDesiredState !== "removed"; +} diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index c7177135..3bd882d2 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -26,14 +26,6 @@ async function triggerRecoveryForOfflineServers( ): Promise { if (offlineServerIds.length === 0) return; - const activeStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - ] as const; - const affectedDeployments = await db .select({ deploymentId: deployments.id, @@ -49,7 +41,8 @@ async function triggerRecoveryForOfflineServers( .where( and( inArray(deployments.serverId, offlineServerIds), - inArray(deployments.status, activeStatuses), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + inArray(deployments.trafficState, ["candidate", "active"]), isNull(services.deletedAt), ), ); diff --git a/web/lib/serverless-wake-failures.ts b/web/lib/serverless-wake-failures.ts new file mode 100644 index 00000000..a5c47ec4 --- /dev/null +++ b/web/lib/serverless-wake-failures.ts @@ -0,0 +1,34 @@ +import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; + +export const SERVERLESS_WAKE_FAILURE_LIMIT = 3; + +export function getServerlessWakeFailureUpdate({ + serverlessEnabled, + currentFailureCount, + failedStage, +}: { + serverlessEnabled: boolean; + currentFailureCount: number | null | undefined; + failedStage: string; +}) { + const nextFailureCount = (currentFailureCount ?? 0) + 1; + const baseUpdate = { + containerId: null, + healthStatus: null, + serverlessWakeFailureCount: nextFailureCount, + }; + + if (serverlessEnabled && nextFailureCount < SERVERLESS_WAKE_FAILURE_LIMIT) { + return { + ...baseUpdate, + runtimeDesiredState: "stopped" as const, + observedPhase: "sleeping" as const, + failedStage: null, + }; + } + + return { + ...markDeploymentFailedRemoved(failedStage), + ...baseUpdate, + }; +} diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 7e6aec83..263e34c9 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -44,15 +44,24 @@ export type PlacementConfig = { replicas: number; }; +export type ServerlessConfig = { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; +}; + export type DeployedConfig = { source: SourceConfig; hostname?: string; + stateful?: boolean; placement?: PlacementConfig; replicas: ReplicaConfig[]; healthCheck: HealthCheckConfig | null; startCommand?: string | null; resourceLimits?: ResourceLimitsConfig; ports: PortConfig[]; + serverless?: ServerlessConfig; secretKeys?: string[]; secrets?: SecretConfig[]; volumes?: VolumeConfig[]; @@ -77,9 +86,20 @@ export function buildCurrentConfig( resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; replicas: number; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; }, replicas: { serverId: string; serverName: string; count: number }[], - ports: { port: number; isPublic: boolean; domain: string | null }[], + ports: { + port: number; + isPublic: boolean; + domain: string | null; + protocol?: "http" | "tcp" | "udp" | null; + tlsPassthrough?: boolean | null; + }[], secrets?: { key: string; updatedAt: Date | string }[], volumes?: { name: string; containerPath: string }[], ): DeployedConfig { @@ -92,6 +112,7 @@ export function buildCurrentConfig( image: service.image, }, hostname: service.hostname ?? undefined, + stateful: service.stateful ?? false, placement: { replicas: replicas.reduce((sum, r) => sum + r.count, 0), }, @@ -120,7 +141,15 @@ export function buildCurrentConfig( port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol ?? "http", + tlsPassthrough: p.tlsPassthrough ?? undefined, })), + serverless: { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, + minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, + }, secrets: (secrets ?? []).map((s) => ({ key: s.key, updatedAt: @@ -182,6 +211,21 @@ export function diffConfigs( to: `${current.resourceLimits.memoryMb} MB`, }); } + if (current.stateful) { + changes.push({ + field: "Service type", + from: "(not deployed)", + to: "Stateful", + }); + } + const currentServerless = normalizeServerlessConfig(current.serverless); + if (currentServerless.enabled) { + changes.push({ + field: "Serverless", + from: "(not deployed)", + to: "Enabled", + }); + } for (const port of current.ports) { const portType = port.isPublic ? "public" : "internal"; changes.push({ @@ -207,6 +251,56 @@ export function diffConfigs( return changes; } + if ((deployed.stateful ?? false) !== (current.stateful ?? false)) { + changes.push({ + field: "Service type", + from: deployed.stateful ? "Stateful" : "Stateless", + to: current.stateful ? "Stateful" : "Stateless", + }); + } + + const deployedServerless = normalizeServerlessConfig(deployed.serverless); + const currentServerless = normalizeServerlessConfig(current.serverless); + if (deployedServerless.enabled !== currentServerless.enabled) { + changes.push({ + field: "Serverless", + from: deployedServerless.enabled ? "Enabled" : "Disabled", + to: currentServerless.enabled ? "Enabled" : "Disabled", + }); + } + if (deployedServerless.enabled && currentServerless.enabled) { + if ( + deployedServerless.sleepAfterSeconds !== + currentServerless.sleepAfterSeconds + ) { + changes.push({ + field: "Serverless sleep timeout", + from: `${deployedServerless.sleepAfterSeconds}s`, + to: `${currentServerless.sleepAfterSeconds}s`, + }); + } + if ( + deployedServerless.wakeTimeoutSeconds !== + currentServerless.wakeTimeoutSeconds + ) { + changes.push({ + field: "Serverless wake timeout", + from: `${deployedServerless.wakeTimeoutSeconds}s`, + to: `${currentServerless.wakeTimeoutSeconds}s`, + }); + } + if ( + deployedServerless.minReadyReplicas !== + currentServerless.minReadyReplicas + ) { + changes.push({ + field: "Serverless min ready", + from: String(deployedServerless.minReadyReplicas), + to: String(currentServerless.minReadyReplicas), + }); + } + } + if (deployed.source.image !== current.source.image) { changes.push({ field: "Image", @@ -466,3 +560,77 @@ export function parseDeployedConfig( return null; } } + +export function normalizeServerlessConfig( + config: ServerlessConfig | undefined, +): ServerlessConfig { + return { + enabled: config?.enabled ?? false, + sleepAfterSeconds: config?.sleepAfterSeconds ?? 300, + wakeTimeoutSeconds: config?.wakeTimeoutSeconds ?? 300, + minReadyReplicas: config?.minReadyReplicas ?? 1, + }; +} + +export function getCurrentServerlessConfig(service: { + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}): ServerlessConfig { + return { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, + minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, + }; +} + +export function getDeployedServerlessConfig(service: { + deployedConfig?: string | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}): ServerlessConfig { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + return deployed?.serverless + ? normalizeServerlessConfig(deployed.serverless) + : getCurrentServerlessConfig(service); +} + +export function getDeployedStateful(service: { + deployedConfig?: string | null; + stateful?: boolean | null; +}) { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + return deployed?.stateful ?? service.stateful ?? false; +} + +export function isDeployedServerlessService(service: { + deployedConfig?: string | null; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}) { + return getDeployedServerlessConfig(service).enabled; +} + +export function getDeployedServicePorts( + service: { id: string; deployedConfig?: string | null }, + livePorts: PortConfig[], +): PortConfig[] { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + if (!deployed?.ports) { + return livePorts; + } + return deployed.ports.map((port) => ({ + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? "http", + tlsPassthrough: port.tlsPassthrough, + })); +} diff --git a/web/lib/service-deployment-actions.ts b/web/lib/service-deployment-actions.ts new file mode 100644 index 00000000..d8646241 --- /dev/null +++ b/web/lib/service-deployment-actions.ts @@ -0,0 +1,38 @@ +import type { ServiceWithDetails } from "@/db/types"; +import { isObservedReady, isRuntimeExpected } from "@/lib/deployment-status"; + +type ServiceDeploymentActionInput = Pick< + ServiceWithDetails, + "configuredReplicas" | "deployments" +>; + +export function getServiceDeploymentActionState( + service: ServiceDeploymentActionInput, +) { + const hasDeployments = service.deployments.length > 0; + const hasExpectedDeployments = service.deployments.some((deployment) => + isRuntimeExpected(deployment.runtimeDesiredState), + ); + const hasRestartableDeployments = service.deployments.some( + (deployment) => + deployment.runtimeDesiredState === "running" && + isObservedReady(deployment.observedPhase) && + !!deployment.containerId, + ); + const hasStoppedOrFailedDeployments = + !hasExpectedDeployments && + service.deployments.some( + (deployment) => + deployment.runtimeDesiredState === "removed" || + deployment.observedPhase === "failed", + ); + const canStartAll = + hasStoppedOrFailedDeployments && service.configuredReplicas.length > 0; + + return { + hasDeployments, + hasExpectedDeployments, + hasRestartableDeployments, + canStartAll, + }; +} diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts index c592442b..807334a5 100644 --- a/web/lib/work-queue.ts +++ b/web/lib/work-queue.ts @@ -347,8 +347,8 @@ async function runForceCleanupCompletionSideEffects( .update(deployments) .set({ containerId: null, - status: "pending", - desired: true, + runtimeDesiredState: "running", + observedPhase: "pending", healthStatus: null, unhealthyReportCount: 0, autohealRestartCount: 0, @@ -357,7 +357,7 @@ async function runForceCleanupCompletionSideEffects( .where( and( eq(deployments.id, payload.deploymentId), - eq(deployments.status, "failed"), + eq(deployments.observedPhase, "failed"), eq(deployments.failedStage, "autoheal_recreate"), ), ); diff --git a/web/tests/agent-status.test.ts b/web/tests/agent-status.test.ts new file mode 100644 index 00000000..b3df6253 --- /dev/null +++ b/web/tests/agent-status.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/lib/inngest/client", () => ({ + inngest: { send: vi.fn() }, +})); +vi.mock("@/lib/inngest/events", () => ({ + inngestEvents: { + resourceStatusChanged: { create: vi.fn((payload) => payload) }, + serverDnsSynced: { create: vi.fn((payload) => payload) }, + }, +})); +vi.mock("@/lib/victoria-logs", () => ({ + ingestRolloutLog: vi.fn(), +})); +vi.mock("@/lib/work-queue", () => ({ + enqueueWork: vi.fn(), +})); + +import { + getStoppedContainerReportUpdate, + shouldAttachReportedContainer, +} from "@/lib/agent-status"; + +describe("agent status serverless attachment", () => { + it("does not attach reported containers to sleeping deployments", () => { + expect(shouldAttachReportedContainer("pending")).toBe(true); + expect(shouldAttachReportedContainer("pulling")).toBe(true); + expect(shouldAttachReportedContainer("waking")).toBe(true); + expect(shouldAttachReportedContainer("sleeping")).toBe(false); + expect(shouldAttachReportedContainer("failed")).toBe(false); + }); + + it("preserves sleeping observation for intended-stopped container reports", () => { + expect( + getStoppedContainerReportUpdate({ runtimeDesiredState: "stopped" }), + ).toEqual({ + containerId: null, + observedPhase: "sleeping", + healthStatus: null, + }); + + expect( + getStoppedContainerReportUpdate({ runtimeDesiredState: "running" }), + ).toEqual({ + observedPhase: "stopped", + healthStatus: "none", + }); + }); +}); diff --git a/web/tests/autoheal-policy.test.ts b/web/tests/autoheal-policy.test.ts index bb54169f..f3f0f7ca 100644 --- a/web/tests/autoheal-policy.test.ts +++ b/web/tests/autoheal-policy.test.ts @@ -13,8 +13,9 @@ describe("autoheal policy", () => { expect(decision.recreateLimitReached).toBe(false); expect(decision.update).toMatchObject({ - status: "failed", - desired: true, + observedPhase: "failed", + runtimeDesiredState: "running", + trafficState: "active", failedStage: "autoheal_recreate", autohealRecreateCount: 3, }); @@ -28,8 +29,9 @@ describe("autoheal policy", () => { expect(decision.recreateLimitReached).toBe(true); expect(decision.update).toMatchObject({ - status: "failed", - desired: false, + observedPhase: "failed", + runtimeDesiredState: "removed", + trafficState: "inactive", failedStage: "autoheal_recreate_limit", autohealRecreateCount: 3, }); @@ -42,8 +44,9 @@ describe("autoheal policy", () => { }); expect(decision.update).toEqual({ - status: "failed", - desired: false, + observedPhase: "failed", + runtimeDesiredState: "removed", + trafficState: "inactive", failedStage: "health_check", }); }); @@ -60,8 +63,9 @@ describe("autoheal policy", () => { expect(decision.limitReached).toBe(false); expect(decision.updateFields).toMatchObject({ - status: "failed", - desired: true, + observedPhase: "failed", + runtimeDesiredState: "running", + trafficState: "active", failedStage: "autoheal_recreate", autohealRecreateCount: 2, }); diff --git a/web/tests/deployment-status.test.ts b/web/tests/deployment-status.test.ts new file mode 100644 index 00000000..06d958c7 --- /dev/null +++ b/web/tests/deployment-status.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { + isDeploymentRoutable, + isRuntimeExpected, + isObservedReady, + markDeploymentRemoved, +} from "@/lib/deployment-status"; + +describe("deployment state helpers", () => { + it("keeps runtime intent separate from observed phase", () => { + expect(isRuntimeExpected("running")).toBe(true); + expect(isRuntimeExpected("stopped")).toBe(true); + expect(isRuntimeExpected("removed")).toBe(false); + + expect(isObservedReady("healthy")).toBe(true); + expect(isObservedReady("running")).toBe(true); + expect(isObservedReady("sleeping")).toBe(false); + expect(isObservedReady("waking")).toBe(false); + }); + + it("requires active traffic intent and a ready observation for routing", () => { + expect( + isDeploymentRoutable({ + trafficState: "active", + observedPhase: "healthy", + }), + ).toBe(true); + expect( + isDeploymentRoutable({ + trafficState: "candidate", + observedPhase: "healthy", + }), + ).toBe(false); + expect( + isDeploymentRoutable({ + trafficState: "active", + observedPhase: "sleeping", + }), + ).toBe(false); + }); + + it("represents removal as explicit runtime and traffic intent", () => { + expect(markDeploymentRemoved()).toEqual({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }); + }); +}); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index cf224ab0..52d93b25 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -8,10 +8,33 @@ vi.mock("@/lib/wireguard", () => ({ getWireGuardPeers: vi.fn() })); import { buildExpectedContainersFromRows, + buildRuntimeRoutePorts, + buildServerlessRoutesFromRows, buildTraefikRoutes, } from "@/lib/agent/expected-state"; describe("expected-state pure builders", () => { + const deployedServerlessConfig = JSON.stringify({ + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [ + { + port: 3000, + isPublic: true, + domain: "sleepy.example.com", + protocol: "http", + }, + ], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }); + it("groups container inputs by deployment and service deterministically", () => { const containers = buildExpectedContainersFromRows({ deployments: [ @@ -19,11 +42,17 @@ describe("expected-state pure builders", () => { id: "dep_bbbbbbbb", serviceId: "svc_1", ipAddress: "10.0.0.2", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", }, { id: "dep_aaaaaaaa", serviceId: "svc_1", ipAddress: "10.0.0.1", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", }, ] as any, services: [ @@ -39,6 +68,7 @@ describe("expected-state pure builders", () => { healthCheckStartPeriod: null, resourceCpuLimit: 1, resourceMemoryLimitMb: 512, + serverlessEnabled: true, }, ] as any, deploymentPorts: [ @@ -61,6 +91,7 @@ describe("expected-state pure builders", () => { ]); expect(containers[0]).toMatchObject({ name: "svc_1-dep_aaaa", + desiredState: "running", image: "docker.io/library/nginx", ports: [ { containerPort: 80, hostPort: 80 }, @@ -79,6 +110,10 @@ describe("expected-state pure builders", () => { retries: 3, startPeriod: 30, }); + expect(containers[1]).toMatchObject({ + deploymentId: "dep_bbbbbbbb", + desiredState: "stopped", + }); }); it("keeps HTTP local upstreams before remote upstreams", () => { @@ -95,7 +130,11 @@ describe("expected-state pure builders", () => { }, ] as any, routableDeployments: [ - { serviceId: "svc_1", serverId: "server_remote", ipAddress: "10.0.0.2" }, + { + serviceId: "svc_1", + serverId: "server_remote", + ipAddress: "10.0.0.2", + }, { serviceId: "svc_1", serverId: "server_local", ipAddress: "10.0.0.3" }, { serviceId: "svc_1", serverId: "server_local", ipAddress: "10.0.0.1" }, ] as any, @@ -114,4 +153,615 @@ describe("expected-state pure builders", () => { }, ]); }); + + it("keeps non-public serverless deployments running in expected state", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_private", + ipAddress: "10.0.0.10", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", + }, + ] as any, + services: [ + { + id: "svc_private", + name: "private-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "running", + }); + }); + + it("projects stopped runtime intent without checking route eligibility", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_stopped", + serviceId: "svc_private", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_private", + name: "private-api", + image: "nginx", + serverlessEnabled: false, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_stopped", + desiredState: "stopped", + }); + }); + + it("marks public serverless deployments stopped while sleeping", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "stopped", + }); + }); + + it("keeps deployed serverless sleeping while live serverless settings are disabled", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: false, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "stopped", + }); + }); + + it("keeps drained serverless deployments without containers stopped", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_draining", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "draining", + observedPhase: "sleeping", + containerId: null, + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_draining", + desiredState: "stopped", + }); + }); + + it("routes owner proxy serverless HTTP services through the local wake gateway", () => { + const routes = buildTraefikRoutes({ + serverId: "server_local", + ports: [ + { + id: "port_1", + serviceId: "svc_serverless", + port: 3000, + isPublic: true, + protocol: "http", + domain: "sleepy.example.com", + }, + ] as any, + routableDeployments: [], + serverlessServiceIds: new Set(["svc_serverless"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "sleepy.example.com", + domain: "sleepy.example.com", + serviceId: "svc_serverless", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); + + it("omits serverless HTTP routes on non-owner proxies", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_2", + ports: [ + { + id: "port_1", + serviceId: "svc_serverless", + port: 3000, + isPublic: true, + protocol: "http", + domain: "sleepy.example.com", + }, + ] as any, + routableDeployments: [], + serverlessRouteSuppressedServiceIds: new Set(["svc_serverless"]), + }); + + expect(routes.httpRoutes).toEqual([]); + }); + + it("keeps worker-only serverless services on direct routes", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports: [ + { + id: "port_1", + serviceId: "svc_worker_only", + port: 3000, + isPublic: true, + protocol: "http", + domain: "worker-only.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_worker_only", + serverId: "worker_1", + ipAddress: "10.0.0.20", + }, + ] as any, + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "worker-only.example.com", + domain: "worker-only.example.com", + serviceId: "svc_worker_only", + upstreams: [{ url: "10.0.0.20:3000", weight: 1 }], + }, + ]); + }); + + it("routes mixed serverless services through the gateway on owner proxies", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports: [ + { + id: "port_1", + serviceId: "svc_mixed", + port: 3000, + isPublic: true, + protocol: "http", + domain: "mixed.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_mixed", + serverId: "worker_1", + ipAddress: "10.0.0.30", + }, + ] as any, + serverlessServiceIds: new Set(["svc_mixed"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "mixed.example.com", + domain: "mixed.example.com", + serviceId: "svc_mixed", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); + + it("does not emit worker-direct fallback routes for non-owner mixed serverless services", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_2", + ports: [ + { + id: "port_1", + serviceId: "svc_mixed", + port: 3000, + isPublic: true, + protocol: "http", + domain: "mixed.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_mixed", + serverId: "worker_1", + ipAddress: "10.0.0.30", + }, + ] as any, + serverlessRouteSuppressedServiceIds: new Set(["svc_mixed"]), + }); + + expect(routes.httpRoutes).toEqual([]); + }); + + it("builds proxy-local serverless metadata with always-on worker upstreams", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_proxy", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, + { + id: "dep_stopped_stale_ready", + serviceId: "svc_1", + serverId: "worker_2", + ipAddress: "10.0.0.30", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, + ] as any, + containers: [ + { + deploymentId: "dep_proxy", + desiredState: "stopped", + }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_1", + domain: "app.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_proxy"], + upstreams: [ + { + deploymentId: "dep_worker", + serverId: "worker_1", + url: "10.0.0.20:3000", + local: false, + alwaysOn: true, + }, + ], + }, + ]); + }); + + it("builds proxy-local serverless metadata for stateful services", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_stateful", + serverlessEnabled: true, + stateful: true, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_stateful", + port: 3000, + isPublic: true, + protocol: "http", + domain: "db.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_stateful", + serviceId: "svc_stateful", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, + ] as any, + containers: [ + { + deploymentId: "dep_stateful", + desiredState: "stopped", + }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_stateful", + domain: "db.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_stateful"], + upstreams: [], + }, + ]); + }); + + it("does not include draining serverless deployments as wakeable local deployments", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_old", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "draining", + observedPhase: "sleeping", + serverIsProxy: true, + }, + { + id: "dep_new", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.11", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", + serverIsProxy: true, + }, + ] as any, + containers: [ + { deploymentId: "dep_old", desiredState: "stopped" }, + { deploymentId: "dep_new", desiredState: "running" }, + ] as any, + }); + + expect(routes[0]?.localDeploymentIds).toEqual(["dep_new"]); + expect( + routes[0]?.upstreams.map((upstream) => upstream.deploymentId), + ).toEqual(["dep_new"]); + }); + + it("keeps wake metadata from deployed serverless config while live settings are disabled", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: false, + stateful: false, + serverlessSleepAfterSeconds: 60, + serverlessWakeTimeoutSeconds: 60, + serverlessMinReadyReplicas: 1, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + ports: [], + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, + ] as any, + containers: [ + { deploymentId: "dep_sleeping", desiredState: "stopped" }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_1", + domain: "sleepy.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_sleeping"], + upstreams: [], + }, + ]); + }); + + it("keeps Traefik gateway route from deployed serverless ports while live ports are removed", () => { + const ports = buildRuntimeRoutePorts( + [ + { + id: "svc_public", + serverlessEnabled: false, + stateful: false, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + [], + ); + + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports, + routableDeployments: [], + serverlessServiceIds: new Set(["svc_public"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "sleepy.example.com", + domain: "sleepy.example.com", + serviceId: "svc_public", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); + + it("omits serverless metadata for worker-only services", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, + ] as any, + containers: [], + }); + + expect(routes).toEqual([]); + }); }); diff --git a/web/tests/project-deletion.test.ts b/web/tests/project-deletion.test.ts new file mode 100644 index 00000000..519ff95c --- /dev/null +++ b/web/tests/project-deletion.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { blocksProjectDeletion } from "@/lib/project-deletion"; + +describe("project deletion guard", () => { + it("blocks deletion for deployments that still have runtime intent", () => { + expect(blocksProjectDeletion({ runtimeDesiredState: "running" })).toBe(true); + expect(blocksProjectDeletion({ runtimeDesiredState: "stopped" })).toBe(true); + expect(blocksProjectDeletion({ runtimeDesiredState: "removed" })).toBe( + false, + ); + }); +}); diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts new file mode 100644 index 00000000..65f96954 --- /dev/null +++ b/web/tests/rollout-helpers.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/db/queries", () => ({ getService: vi.fn() })); +vi.mock("@/lib/acme-manager", () => ({ + getCertificate: vi.fn(), + issueCertificate: vi.fn(), +})); +vi.mock("@/lib/wireguard", () => ({ + assignContainerIp: vi.fn(), +})); +vi.mock("@/lib/work-queue", () => ({ + enqueueWork: vi.fn(), +})); + +import { isActiveDeploymentForRollout } from "@/lib/inngest/functions/rollout-helpers"; + +describe("rollout helpers", () => { + it("treats active traffic deployments as the live rollout version", () => { + expect( + isActiveDeploymentForRollout( + { trafficState: "active" }, + { serverlessEnabled: true }, + ), + ).toBe(true); + expect( + isActiveDeploymentForRollout( + { trafficState: "candidate" }, + { serverlessEnabled: true }, + ), + ).toBe(false); + expect( + isActiveDeploymentForRollout( + { trafficState: "draining" }, + { serverlessEnabled: false }, + ), + ).toBe(false); + }); +}); diff --git a/web/tests/rollout-utils.test.ts b/web/tests/rollout-utils.test.ts new file mode 100644 index 00000000..7dc9bb7e --- /dev/null +++ b/web/tests/rollout-utils.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/lib/email", () => ({ + sendDeploymentFailureAlert: vi.fn(), +})); + +import { + shouldRestoreDrainingDeployment, + shouldRollBackDeploymentState, +} from "@/lib/inngest/functions/rollout-utils"; + +describe("rollout failure helpers", () => { + it("cleans up live rollout deployments even after DNS promotion", () => { + expect( + shouldRollBackDeploymentState({ + trafficState: "candidate", + runtimeDesiredState: "running", + }), + ).toBe(true); + expect( + shouldRollBackDeploymentState({ + trafficState: "active", + runtimeDesiredState: "running", + }), + ).toBe(true); + expect( + shouldRollBackDeploymentState({ + trafficState: "candidate", + runtimeDesiredState: "removed", + }), + ).toBe(false); + }); + + it("restores draining deployments by traffic intent only", () => { + expect( + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "running", + }), + ).toBe(true); + expect( + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "stopped", + }), + ).toBe(true); + expect( + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "removed", + }), + ).toBe(false); + }); +}); diff --git a/web/tests/serverless-wake-failures.test.ts b/web/tests/serverless-wake-failures.test.ts new file mode 100644 index 00000000..71a96e39 --- /dev/null +++ b/web/tests/serverless-wake-failures.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; + +describe("serverless wake failure policy", () => { + it("keeps enabled serverless deployments retryable below the failure limit", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: true, + currentFailureCount: 1, + failedStage: "serverless_wake", + }), + ).toMatchObject({ + runtimeDesiredState: "stopped", + observedPhase: "sleeping", + failedStage: null, + serverlessWakeFailureCount: 2, + }); + }); + + it("parks enabled serverless deployments after repeated wake failures", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: true, + currentFailureCount: 2, + failedStage: "serverless_wake", + }), + ).toMatchObject({ + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "failed", + failedStage: "serverless_wake", + serverlessWakeFailureCount: 3, + }); + }); + + it("fails disabled transition deployments immediately", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: false, + currentFailureCount: 0, + failedStage: "serverless_wake_timeout", + }), + ).toMatchObject({ + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "failed", + failedStage: "serverless_wake_timeout", + serverlessWakeFailureCount: 1, + }); + }); +}); diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts new file mode 100644 index 00000000..5564eabf --- /dev/null +++ b/web/tests/service-config.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { + diffConfigs, + getDeployedServerlessConfig, + isDeployedServerlessService, + type DeployedConfig, +} from "@/lib/service-config"; + +function deployedConfig( + overrides: Partial = {}, +): DeployedConfig { + return { + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + ...overrides, + }; +} + +describe("service config", () => { + it("uses deployed serverless settings as the runtime mode", () => { + const service = { + serverlessEnabled: false, + serverlessSleepAfterSeconds: 60, + serverlessWakeTimeoutSeconds: 60, + serverlessMinReadyReplicas: 1, + stateful: true, + deployedConfig: JSON.stringify(deployedConfig()), + }; + + expect(getDeployedServerlessConfig(service)).toMatchObject({ + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }); + expect(isDeployedServerlessService(service)).toBe(true); + }); + + it("allows deployed stateful services to be serverless", () => { + const service = { + serverlessEnabled: true, + stateful: true, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + deployedConfig: JSON.stringify(deployedConfig({ stateful: true })), + }; + + expect(isDeployedServerlessService(service)).toBe(true); + }); + + it("reports serverless changes as pending config", () => { + const changes = diffConfigs(deployedConfig(), { + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }); + + expect(changes).toContainEqual({ + field: "Serverless", + from: "Enabled", + to: "Disabled", + }); + }); +}); diff --git a/web/tests/service-deployment-actions.test.ts b/web/tests/service-deployment-actions.test.ts new file mode 100644 index 00000000..955d1a52 --- /dev/null +++ b/web/tests/service-deployment-actions.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { getServiceDeploymentActionState } from "@/lib/service-deployment-actions"; + +describe("service deployment action state", () => { + it("shows redeploy actions for sleeping serverless deployments", () => { + const state = getServiceDeploymentActionState({ + configuredReplicas: [{ serverId: "proxy_1", count: 1 }], + deployments: [ + { + runtimeDesiredState: "stopped", + observedPhase: "sleeping", + containerId: null, + }, + ], + } as any); + + expect(state).toMatchObject({ + hasDeployments: true, + hasExpectedDeployments: true, + hasRestartableDeployments: false, + canStartAll: false, + }); + }); + + it("only shows restart when a running-intent deployment has a ready container", () => { + const state = getServiceDeploymentActionState({ + configuredReplicas: [{ serverId: "proxy_1", count: 1 }], + deployments: [ + { + runtimeDesiredState: "running", + observedPhase: "healthy", + containerId: "container_1", + }, + ], + } as any); + + expect(state).toMatchObject({ + hasExpectedDeployments: true, + hasRestartableDeployments: true, + canStartAll: false, + }); + }); + + it("shows start actions after deployments have been removed", () => { + const state = getServiceDeploymentActionState({ + configuredReplicas: [{ serverId: "proxy_1", count: 1 }], + deployments: [ + { + runtimeDesiredState: "removed", + observedPhase: "stopped", + containerId: null, + }, + ], + } as any); + + expect(state).toMatchObject({ + hasDeployments: true, + hasExpectedDeployments: false, + hasRestartableDeployments: false, + canStartAll: true, + }); + }); +});