From 37e4f845f5c486bf054d8aa10df225adc29afb57 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Fri, 3 Jul 2026 23:21:15 +1000 Subject: [PATCH 01/29] Move service instance summary under status --- web/components/service/details/service-details-overview.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index 274d8795..2d8926d5 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -404,11 +404,11 @@ function ServiceConfigPanel({
- -

{formatInstanceSummary(overview)}

+ + From cae53067b1e61c5f9568543e33862d67e633d054 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Fri, 3 Jul 2026 23:40:19 +1000 Subject: [PATCH 02/29] Show friendly deployment schedule labels --- .../service/details/schedule-section.tsx | 5 ++++- .../details/service-details-overview.tsx | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/web/components/service/details/schedule-section.tsx b/web/components/service/details/schedule-section.tsx index 1247ef65..b15387dd 100644 --- a/web/components/service/details/schedule-section.tsx +++ b/web/components/service/details/schedule-section.tsx @@ -32,7 +32,10 @@ export const ScheduleSection = memo(function ScheduleSection({ return { description: null, error: null }; } try { - return { description: cronstrue.toString(schedule), error: null }; + return { + description: cronstrue.toString(schedule, { verbose: true }), + error: null, + }; } catch { return { description: null, error: "Invalid cron expression" }; } diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index 2d8926d5..a876dafc 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -1,5 +1,6 @@ "use client"; +import cronstrue from "cronstrue"; import { Activity, Box, @@ -1007,9 +1008,20 @@ function formatBackupLabel(service: Service): string { } function formatDeployScheduleLabel(service: Service): string { - return service.deploymentSchedule - ? `Deploy ${service.deploymentSchedule}` - : "Manual Deploy"; + if (!service.deploymentSchedule) return "Manual Deploy"; + + try { + const scheduleDescription = cronstrue.toString(service.deploymentSchedule, { + verbose: true, + }); + return `Deploy ${lowercaseFirstLetter(scheduleDescription)}`; + } catch { + return "Scheduled Deploy"; + } +} + +function lowercaseFirstLetter(value: string): string { + return value.charAt(0).toLowerCase() + value.slice(1); } function formatCount(count: number, singular: string): string { From 9ba90a4dd21d5cb1213b3ed56d2e420cc757b578 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 09:07:38 +1000 Subject: [PATCH 03/29] Add serverless container wake path --- agent/internal/agent/drift.go | 45 +- agent/internal/agent/handlers.go | 23 + agent/internal/agent/run.go | 8 + agent/internal/agent/workqueue.go | 4 +- agent/internal/http/client.go | 93 ++ agent/internal/serverless/gateway.go | 158 +++ web/actions/projects.ts | 104 +- .../[serviceId]/configuration/page.tsx | 3 + .../api/v1/agent/serverless/activity/route.ts | 75 ++ web/app/api/v1/agent/serverless/wake/route.ts | 95 ++ .../service/details/serverless-section.tsx | 177 ++++ web/db/schema.ts | 47 + web/db/types.ts | 3 + web/lib/agent-status.ts | 83 +- web/lib/agent/expected-state.ts | 110 +- web/lib/deployment-status.ts | 6 +- web/lib/inngest/functions/crons.ts | 14 + web/lib/inngest/functions/index.ts | 1 + web/lib/scheduler.ts | 15 + web/lib/serverless.ts | 945 ++++++++++++++++++ web/tests/deployment-status.test.ts | 20 + web/tests/expected-state.test.ts | 40 +- 22 files changed, 2010 insertions(+), 59 deletions(-) create mode 100644 agent/internal/serverless/gateway.go create mode 100644 web/app/api/v1/agent/serverless/activity/route.ts create mode 100644 web/app/api/v1/agent/serverless/wake/route.ts create mode 100644 web/components/service/details/serverless-section.tsx create mode 100644 web/lib/serverless.ts create mode 100644 web/tests/deployment-status.test.ts diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 4ab0fc78..c7dc0397 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -23,6 +23,7 @@ const ( actionStopUnexpectedContainer reconcileActionKind = "stop_unexpected_container" actionRemoveUnexpectedContainer reconcileActionKind = "remove_unexpected_container" actionDeployMissingContainer reconcileActionKind = "deploy_missing_container" + actionStopExpectedContainer reconcileActionKind = "stop_expected_container" actionStartContainer reconcileActionKind = "start_container" actionRedeployContainer reconcileActionKind = "redeploy_container" actionUpdateDNS reconcileActionKind = "update_dns" @@ -250,6 +251,9 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS for id, exp := range expectedMap { if _, exists := actualMap[id]; !exists { + if desiredContainerState(exp) == "stopped" { + continue + } expectedContainer := exp actions = append(actions, reconcileAction{ Kind: actionDeployMissingContainer, @@ -264,26 +268,40 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS if act, exists := actualMap[id]; exists { expectedContainer := exp actualContainer := act - if act.State == "created" || act.State == "exited" { + + if desiredContainerState(exp) == "stopped" { + if act.State == "running" || act.State == "paused" { + actions = append(actions, reconcileAction{ + Kind: actionStopExpectedContainer, + Description: fmt.Sprintf("STOP %s (desired state: stopped)", exp.Name), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) + } + continue + } + + if normalizeImage(exp.Image) != normalizeImage(act.Image) { actions = append(actions, reconcileAction{ - Kind: actionStartContainer, - Description: fmt.Sprintf("START %s (state: %s)", exp.Name, act.State), + Kind: actionRedeployContainer, + Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, }) - } else if act.State != "running" { + } else if act.State == "created" || act.State == "exited" { actions = append(actions, reconcileAction{ - Kind: actionRedeployContainer, - Description: fmt.Sprintf("REDEPLOY %s (state: %s)", exp.Name, act.State), + Kind: actionStartContainer, + Description: fmt.Sprintf("START %s (state: %s)", exp.Name, act.State), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, }) - } else if normalizeImage(exp.Image) != normalizeImage(act.Image) { + } else if act.State != "running" { actions = append(actions, reconcileAction{ Kind: actionRedeployContainer, - Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), + Description: fmt.Sprintf("REDEPLOY %s (state: %s)", exp.Name, act.State), DeploymentID: id, Expected: &expectedContainer, Actual: &actualContainer, @@ -389,16 +407,23 @@ func normalizeImage(image string) string { return image + digest } +func desiredContainerState(container agenthttp.ExpectedContainer) string { + if container.DesiredState == "stopped" { + return "stopped" + } + return "running" +} + func (a *Agent) applyReconcileAction(action reconcileAction) error { log.Printf("[reconcile] %s", action.Description) switch action.Kind { - case actionStopOrphanNoDeploymentID, actionStopUnexpectedContainer: + case actionStopOrphanNoDeploymentID, actionStopUnexpectedContainer, actionStopExpectedContainer: if action.Actual == nil { return fmt.Errorf("missing actual container for %s", action.Kind) } if err := container.Stop(action.Actual.ID); err != nil { - return fmt.Errorf("failed to stop orphan container: %w", err) + return fmt.Errorf("failed to stop container: %w", err) } return nil diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go index 5dd68cb0..b8ce1263 100644 --- a/agent/internal/agent/handlers.go +++ b/agent/internal/agent/handlers.go @@ -56,6 +56,29 @@ func (a *Agent) ProcessStop(item agenthttp.WorkQueueItem) error { return nil } +func (a *Agent) ProcessSleep(item agenthttp.WorkQueueItem) error { + var payload struct { + DeploymentID string `json:"deploymentId"` + ContainerID string `json:"containerId"` + } + + if err := json.Unmarshal([]byte(item.Payload), &payload); err != nil { + return fmt.Errorf("failed to parse sleep payload: %w", err) + } + + if payload.ContainerID == "" { + return fmt.Errorf("sleep payload missing containerId") + } + + log.Printf("[sleep] removing container %s for deployment %s", Truncate(payload.ContainerID, 12), Truncate(payload.DeploymentID, 8)) + + if err := container.ForceRemove(payload.ContainerID); err != nil { + return fmt.Errorf("failed to remove sleeping container: %w", err) + } + + return nil +} + func (a *Agent) ProcessForceCleanup(item agenthttp.WorkQueueItem) error { var payload struct { ServiceID string `json:"serviceId"` diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index cbeb2d62..801c221f 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -6,6 +6,7 @@ import ( "time" "techulus/cloud-agent/internal/container" + "techulus/cloud-agent/internal/serverless" ) func (a *Agent) Run(ctx context.Context) { @@ -32,6 +33,13 @@ func (a *Agent) Run(ctx context.Context) { a.TraefikLogCollector.Start() } + if a.IsProxy { + gateway := serverless.NewGateway(a.Client) + if err := gateway.Start(ctx); err != nil { + log.Printf("[serverless-gateway] failed to start: %v", err) + } + } + var cleanupTickerC <-chan time.Time if a.Builder != nil { cleanupTicker := time.NewTicker(BuildCleanupInterval) diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go index 77aead2b..d61dd1bf 100644 --- a/agent/internal/agent/workqueue.go +++ b/agent/internal/agent/workqueue.go @@ -125,7 +125,9 @@ func (a *Agent) ProcessWorkItem(item agenthttp.WorkQueueItem) error { return a.ProcessRestart(item) case "stop": return a.ProcessStop(item) - case "deploy", "reconcile": + case "sleep": + return a.ProcessSleep(item) + case "deploy", "reconcile", "wake": a.RequestReconcile("reconcile work item " + Truncate(item.ID, 8)) return nil case "force_cleanup": diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 7189f791..1643e617 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -22,6 +22,7 @@ type Client struct { keyPair *crypto.KeyPair client *http.Client longClient *http.Client + wakeClient *http.Client dataDir string } @@ -37,6 +38,9 @@ func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string longClient: &http.Client{ Timeout: 40 * time.Second, }, + wakeClient: &http.Client{ + Timeout: 16 * time.Minute, + }, } } @@ -73,6 +77,7 @@ type ExpectedContainer struct { ServiceID string `json:"serviceId"` ServiceName string `json:"serviceName"` Name string `json:"name"` + DesiredState string `json:"desiredState"` Image string `json:"image"` IPAddress string `json:"ipAddress"` Ports []PortMapping `json:"ports"` @@ -94,6 +99,19 @@ type Upstream struct { Weight int `json:"weight"` } +type ServerlessUpstream struct { + Url string `json:"url"` +} + +type ServerlessWakeResult struct { + Status string `json:"status"` + ServiceID string `json:"serviceId"` + Upstreams []ServerlessUpstream `json:"upstreams"` + WakingDeployments int `json:"wakingDeployments"` + QueuedWakeServers int `json:"queuedWakeServers"` + MinReadyReplicas int `json:"minReadyReplicas"` +} + type TraefikRoute struct { ID string `json:"id"` Domain string `json:"domain"` @@ -420,6 +438,81 @@ func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkIte return &statusResponse, nil } +func (c *Client) WakeServerlessService(host string) (*ServerlessWakeResult, error) { + payload := map[string]interface{}{ + "host": host, + "wait": true, + } + + body, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to marshal serverless wake request: %w", err) + } + + req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/serverless/wake", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create serverless wake request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + c.signRequest(req, string(body)) + + resp, err := c.wakeClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to wake serverless service: %w", err) + } + defer resp.Body.Close() + + var response struct { + OK bool `json:"ok"` + Error string `json:"error"` + Result ServerlessWakeResult `json:"result"` + } + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to decode serverless wake response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + if response.Error != "" { + return nil, fmt.Errorf("serverless wake failed with status %d: %s", resp.StatusCode, response.Error) + } + return nil, fmt.Errorf("serverless wake failed with status %d", resp.StatusCode) + } + + return &response.Result, nil +} + +func (c *Client) RecordServerlessActivity(host, event string) error { + payload := map[string]string{ + "host": host, + "event": event, + } + + body, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal serverless activity request: %w", err) + } + + req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/serverless/activity", bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("failed to create serverless activity request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + c.signRequest(req, string(body)) + + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("failed to record serverless activity: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("serverless activity failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + func (c *Client) GetBuildStatus(buildID string) (string, error) { req, err := http.NewRequest("GET", c.baseURL+"/api/v1/agent/builds/"+buildID+"/status", nil) if err != nil { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go new file mode 100644 index 00000000..878250cc --- /dev/null +++ b/agent/internal/serverless/gateway.go @@ -0,0 +1,158 @@ +package serverless + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync/atomic" + "time" + + agenthttp "techulus/cloud-agent/internal/http" +) + +const ( + GatewayPort = 18080 + activityHeartbeatInterval = 60 * time.Second +) + +type Gateway struct { + client *agenthttp.Client + counter uint64 + server *http.Server +} + +func NewGateway(client *agenthttp.Client) *Gateway { + return &Gateway{client: client} +} + +func (g *Gateway) Start(ctx context.Context) error { + addr := fmt.Sprintf("127.0.0.1:%d", GatewayPort) + listener, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", addr, err) + } + + g.server = &http.Server{ + Handler: g, + ReadHeaderTimeout: 30 * time.Second, + } + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := g.server.Shutdown(shutdownCtx); err != nil { + log.Printf("[serverless-gateway] shutdown error: %v", err) + } + }() + + go func() { + log.Printf("[serverless-gateway] listening on %s", addr) + if err := g.server.Serve(listener); err != nil && err != http.ErrServerClosed { + log.Printf("[serverless-gateway] server error: %v", err) + } + }() + + return nil +} + +func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { + host := normalizeHost(r.Host) + if host == "" { + http.Error(w, "missing host", http.StatusBadRequest) + return + } + + wakeResult, err := g.client.WakeServerlessService(host) + if err != nil { + log.Printf("[serverless-gateway] wake failed for host %s: %v", host, err) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + + if len(wakeResult.Upstreams) == 0 { + log.Printf("[serverless-gateway] no upstreams for host %s after wake (status=%s)", host, wakeResult.Status) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + + if err := g.client.RecordServerlessActivity(host, "start"); err != nil { + log.Printf("[serverless-gateway] failed to record request start for %s: %v", host, err) + } + activityDone := make(chan struct{}) + go g.recordActivityHeartbeats(host, activityDone) + defer func() { + close(activityDone) + if err := g.client.RecordServerlessActivity(host, "finish"); err != nil { + log.Printf("[serverless-gateway] failed to record request finish for %s: %v", host, err) + } + }() + + upstream := wakeResult.Upstreams[g.nextIndex(len(wakeResult.Upstreams))] + target, err := url.Parse("http://" + upstream.Url) + if err != nil { + log.Printf("[serverless-gateway] invalid upstream %q for host %s: %v", upstream.Url, host, err) + http.Error(w, "bad upstream", http.StatusBadGateway) + return + } + + proxy := httputil.NewSingleHostReverseProxy(target) + originalDirector := proxy.Director + originalHost := r.Host + proxy.Director = func(req *http.Request) { + originalDirector(req) + req.Host = originalHost + req.Header.Set("X-Forwarded-Host", originalHost) + req.Header.Set("X-Forwarded-Proto", forwardedProto(r)) + } + proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) { + log.Printf("[serverless-gateway] proxy error for host %s to %s: %v", host, upstream.Url, err) + http.Error(w, "bad gateway", http.StatusBadGateway) + } + proxy.ServeHTTP(w, r) +} + +func (g *Gateway) recordActivityHeartbeats(host string, done <-chan struct{}) { + ticker := time.NewTicker(activityHeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := g.client.RecordServerlessActivity(host, "heartbeat"); err != nil { + log.Printf("[serverless-gateway] failed to record request heartbeat for %s: %v", host, err) + } + case <-done: + return + } + } +} + +func (g *Gateway) nextIndex(length int) int { + if length <= 1 { + return 0 + } + return int(atomic.AddUint64(&g.counter, 1)-1) % length +} + +func normalizeHost(host string) string { + if h, _, err := net.SplitHostPort(host); err == nil { + return strings.ToLower(strings.TrimSpace(h)) + } + return strings.ToLower(strings.TrimSpace(host)) +} + +func forwardedProto(r *http.Request) string { + if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { + return proto + } + if r.TLS != nil { + return "https" + } + return "http" +} diff --git a/web/actions/projects.ts b/web/actions/projects.ts index a9ef3af2..ab15a155 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import cronstrue from "cronstrue"; -import { and, desc, eq, inArray, isNotNull } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { ZodError, z } from "zod"; import { db } from "@/db"; @@ -1013,6 +1013,108 @@ export async function updateServiceSchedule( return { success: true }; } +const serverlessSettingsSchema = z.object({ + enabled: z.boolean(), + sleepAfterSeconds: z.number().int().min(60).max(86_400), + wakeTimeoutSeconds: z.number().int().min(10).max(900), + minReadyReplicas: z.number().int().min(1).max(10), +}); + +export async function updateServiceServerlessSettings( + serviceId: string, + settings: { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; + }, +) { + await requireDeveloperRole(); + const validated = serverlessSettingsSchema.parse(settings); + + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + + const [service] = await tx + .select() + .from(services) + .where(eq(services.id, serviceId)) + .limit(1); + + if (!service || service.deletedAt) { + throw new Error("Service not found"); + } + if (service.stateful && validated.enabled) { + throw new Error("Serverless is only supported for stateless services"); + } + + if (validated.enabled) { + const configuredReplicas = await tx + .select({ count: serviceReplicas.count }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const totalConfiguredReplicas = configuredReplicas.reduce( + (total, replica) => total + replica.count, + 0, + ); + + if (totalConfiguredReplicas < 1) { + throw new Error("Serverless services require at least one replica"); + } + if (validated.minReadyReplicas > totalConfiguredReplicas) { + throw new Error( + "Minimum ready replicas cannot exceed configured replicas", + ); + } + } + + await tx + .update(services) + .set({ + serverlessEnabled: validated.enabled, + serverlessSleepAfterSeconds: validated.sleepAfterSeconds, + serverlessWakeTimeoutSeconds: validated.wakeTimeoutSeconds, + serverlessMinReadyReplicas: validated.minReadyReplicas, + }) + .where(eq(services.id, serviceId)); + + if (!validated.enabled) { + const now = new Date(); + const wakingDeployments = await tx + .update(deployments) + .set({ + status: "waking", + healthStatus: null, + serverlessWakeStartedAt: now, + }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "sleeping"), + ), + ) + .returning({ serverId: deployments.serverId }); + + const serverIds = new Set( + wakingDeployments.map((deployment) => deployment.serverId), + ); + for (const serverId of serverIds) { + await tx.insert(workQueue).values({ + id: randomUUID(), + serverId, + type: "wake", + payload: JSON.stringify({ + reason: "serverless_disabled", + serviceId, + }), + }); + } + } + }); + + return { success: true }; +} + export type ServiceConfigUpdate = { source?: { type: "image"; image: string }; healthCheck?: ServiceHealthCheckConfig | null; diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx index c014b426..849b10a3 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx @@ -12,6 +12,7 @@ import { ReplicasSection } from "@/components/service/details/replicas-section"; import { ResourceLimitsSection } from "@/components/service/details/resource-limits-section"; import { ScheduleSection } from "@/components/service/details/schedule-section"; import { SecretsSection } from "@/components/service/details/secrets-section"; +import { ServerlessSection } from "@/components/service/details/serverless-section"; import { SourceSection } from "@/components/service/details/source-section"; import { StartCommandSection } from "@/components/service/details/start-command-section"; import { TCPProxySection } from "@/components/service/details/tcp-proxy-section"; @@ -98,6 +99,8 @@ export default function ConfigurationPage() { + + diff --git a/web/app/api/v1/agent/serverless/activity/route.ts b/web/app/api/v1/agent/serverless/activity/route.ts new file mode 100644 index 00000000..6a69ff11 --- /dev/null +++ b/web/app/api/v1/agent/serverless/activity/route.ts @@ -0,0 +1,75 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { verifyAgentRequest } from "@/lib/agent-auth"; +import { + isProxyServer, + recordServerlessRequestFinish, + recordServerlessRequestHeartbeat, + recordServerlessRequestStart, + resolveServerlessServiceId, +} from "@/lib/serverless"; + +type ActivityRequestBody = { + serviceId?: string; + host?: string; + event?: "start" | "heartbeat" | "finish"; +}; + +export async function POST(request: NextRequest) { + const body = await request.text(); + const auth = await verifyAgentRequest(request, body); + if (!auth.success) { + return NextResponse.json({ error: auth.error }, { status: auth.status }); + } + + if (!(await isProxyServer(auth.serverId))) { + return NextResponse.json( + { error: "Serverless activity is only available to proxy servers" }, + { status: 403 }, + ); + } + + let data: ActivityRequestBody; + try { + data = body ? JSON.parse(body) : {}; + } catch { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + if ( + data.event !== "start" && + data.event !== "heartbeat" && + data.event !== "finish" + ) { + return NextResponse.json( + { error: "event must be 'start', 'heartbeat', or 'finish'" }, + { status: 400 }, + ); + } + + const serviceId = await resolveServerlessServiceId({ + serviceId: data.serviceId, + host: data.host, + }); + if (!serviceId) { + return NextResponse.json({ error: "Service not found" }, { status: 404 }); + } + + if (data.event === "start") { + await recordServerlessRequestStart({ + serviceId, + proxyServerId: auth.serverId, + }); + } else if (data.event === "heartbeat") { + await recordServerlessRequestHeartbeat({ + serviceId, + proxyServerId: auth.serverId, + }); + } else { + await recordServerlessRequestFinish({ + serviceId, + proxyServerId: auth.serverId, + }); + } + + return NextResponse.json({ ok: true, serviceId }); +} diff --git a/web/app/api/v1/agent/serverless/wake/route.ts b/web/app/api/v1/agent/serverless/wake/route.ts new file mode 100644 index 00000000..75a4206c --- /dev/null +++ b/web/app/api/v1/agent/serverless/wake/route.ts @@ -0,0 +1,95 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { verifyAgentRequest } from "@/lib/agent-auth"; +import { + isProxyServer, + resolveServerlessTarget, + wakeAndWaitForServerlessService, + wakeServerlessService, +} from "@/lib/serverless"; + +export const maxDuration = 960; + +type WakeRequestBody = { + serviceId?: string; + host?: string; + wait?: boolean; +}; + +export async function POST(request: NextRequest) { + const body = await request.text(); + const auth = await verifyAgentRequest(request, body); + if (!auth.success) { + return NextResponse.json({ error: auth.error }, { status: auth.status }); + } + + if (!(await isProxyServer(auth.serverId))) { + return NextResponse.json( + { error: "Serverless wake is only available to proxy servers" }, + { status: 403 }, + ); + } + + let data: WakeRequestBody; + try { + data = body ? JSON.parse(body) : {}; + } catch { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const target = await resolveServerlessTarget({ + serviceId: data.serviceId, + host: data.host, + }); + if (!target) { + return NextResponse.json({ error: "Service not found" }, { status: 404 }); + } + + const result = + data.wait === false + ? await wakeServerlessService({ + serviceId: target.serviceId, + port: target.port, + proxyServerId: auth.serverId, + }) + : await wakeAndWaitForServerlessService({ + serviceId: target.serviceId, + port: target.port, + proxyServerId: auth.serverId, + }); + + if (result.status === "not_found") { + return NextResponse.json( + { error: "Service not found", result }, + { status: 404 }, + ); + } + if (result.status === "not_serverless") { + return NextResponse.json( + { error: "Service is not serverless", result }, + { status: 409 }, + ); + } + if (result.status === "unsupported") { + return NextResponse.json( + { + error: "Serverless wake only supports stateless public HTTP services", + result, + }, + { status: 422 }, + ); + } + if (result.status === "no_deployments") { + return NextResponse.json( + { error: "No sleeping or ready deployments found", result }, + { status: 409 }, + ); + } + if ("timedOut" in result && result.timedOut) { + return NextResponse.json( + { error: "Timed out waiting for serverless service to wake", result }, + { status: 504 }, + ); + } + + return NextResponse.json({ ok: true, result }); +} diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx new file mode 100644 index 00000000..59796ef9 --- /dev/null +++ b/web/components/service/details/serverless-section.tsx @@ -0,0 +1,177 @@ +"use client"; + +import { Moon } from "lucide-react"; +import { memo, useMemo, useState } from "react"; +import { updateServiceServerlessSettings } from "@/actions/projects"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Item, ItemContent, ItemMedia, ItemTitle } from "@/components/ui/item"; +import { Switch } from "@/components/ui/switch"; +import type { ServiceWithDetails as Service } from "@/db/types"; + +export const ServerlessSection = memo(function ServerlessSection({ + service, + onUpdate, +}: { + service: Service; + onUpdate: () => void; +}) { + const [isSaving, setIsSaving] = useState(false); + const [enabled, setEnabled] = useState(service.serverlessEnabled); + const [sleepAfterSeconds, setSleepAfterSeconds] = useState( + String(service.serverlessSleepAfterSeconds ?? 300), + ); + const [wakeTimeoutSeconds, setWakeTimeoutSeconds] = useState( + String(service.serverlessWakeTimeoutSeconds ?? 300), + ); + const [minReadyReplicas, setMinReadyReplicas] = useState( + String(service.serverlessMinReadyReplicas ?? 1), + ); + + const parsed = useMemo( + () => ({ + sleepAfterSeconds: Number.parseInt(sleepAfterSeconds, 10), + wakeTimeoutSeconds: Number.parseInt(wakeTimeoutSeconds, 10), + minReadyReplicas: Number.parseInt(minReadyReplicas, 10), + }), + [sleepAfterSeconds, wakeTimeoutSeconds, minReadyReplicas], + ); + + const validationError = useMemo(() => { + if (service.stateful && enabled) { + return "Serverless is only supported for stateless services"; + } + if ( + !Number.isInteger(parsed.sleepAfterSeconds) || + parsed.sleepAfterSeconds < 60 || + parsed.sleepAfterSeconds > 86_400 + ) { + return "Sleep timeout must be between 60 seconds and 24 hours"; + } + if ( + !Number.isInteger(parsed.wakeTimeoutSeconds) || + parsed.wakeTimeoutSeconds < 10 || + parsed.wakeTimeoutSeconds > 900 + ) { + return "Wake timeout must be between 10 and 900 seconds"; + } + if ( + !Number.isInteger(parsed.minReadyReplicas) || + parsed.minReadyReplicas < 1 || + parsed.minReadyReplicas > 10 + ) { + return "Minimum ready replicas must be between 1 and 10"; + } + return null; + }, [enabled, parsed, service.stateful]); + + const hasChanges = + enabled !== service.serverlessEnabled || + parsed.sleepAfterSeconds !== service.serverlessSleepAfterSeconds || + parsed.wakeTimeoutSeconds !== service.serverlessWakeTimeoutSeconds || + parsed.minReadyReplicas !== service.serverlessMinReadyReplicas; + + const handleSave = async () => { + setIsSaving(true); + try { + await updateServiceServerlessSettings(service.id, { + enabled, + sleepAfterSeconds: parsed.sleepAfterSeconds, + wakeTimeoutSeconds: parsed.wakeTimeoutSeconds, + minReadyReplicas: parsed.minReadyReplicas, + }); + onUpdate(); + } finally { + setIsSaving(false); + } + }; + + return ( +
+ + + + + + Serverless + + + +
+
+
+ + setSleepAfterSeconds(event.target.value)} + /> +
+
+ + setWakeTimeoutSeconds(event.target.value)} + /> +
+
+ + setMinReadyReplicas(event.target.value)} + /> +
+
+ + {validationError && ( +

{validationError}

+ )} + + {hasChanges && ( + + )} +
+
+ ); +}); diff --git a/web/db/schema.ts b/web/db/schema.ts index 6a4aee88..99db8063 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -379,6 +379,16 @@ export const services = pgTable("services", { startCommand: text("start_command"), resourceCpuLimit: real("resource_cpu_limit").default(2), resourceMemoryLimitMb: integer("resource_memory_limit_mb").default(1024), + serverlessEnabled: boolean("serverless_enabled").notNull().default(false), + serverlessSleepAfterSeconds: integer("serverless_sleep_after_seconds") + .notNull() + .default(300), + serverlessWakeTimeoutSeconds: integer("serverless_wake_timeout_seconds") + .notNull() + .default(300), + serverlessMinReadyReplicas: integer("serverless_min_ready_replicas") + .notNull() + .default(1), deployedConfig: text("deployed_config"), deploymentSchedule: text("deployment_schedule"), lastScheduledDeploymentRunAt: timestamp("last_scheduled_deployment_run_at", { @@ -526,9 +536,11 @@ export const deployments = pgTable( "pending", "pulling", "starting", + "waking", "healthy", "running", "draining", + "sleeping", "stopping", "stopped", "failed", @@ -554,6 +566,9 @@ export const deployments = pgTable( rolloutId: text("rollout_id"), previousDeploymentId: text("previous_deployment_id"), failedStage: text("failed_stage"), + serverlessWakeStartedAt: timestamp("serverless_wake_started_at", { + withTimezone: true, + }), createdAt: timestamp("created_at", { withTimezone: true }) .defaultNow() .notNull(), @@ -564,6 +579,9 @@ export const deployments = pgTable( index("deployments_service_id_idx").on(table.serviceId), index("deployments_server_id_idx").on(table.serverId), index("deployments_status_idx").on(table.status), + index("deployments_serverless_wake_started_at_idx").on( + table.serverlessWakeStartedAt, + ), ], ); @@ -621,6 +639,8 @@ export const workQueue = pgTable( "backup_volume", "restore_volume", "create_manifest", + "sleep", + "wake", "upgrade_agent", ], }).notNull(), @@ -646,6 +666,33 @@ export const workQueue = pgTable( ], ); +export const serverlessServiceActivity = pgTable( + "serverless_service_activity", + { + id: text("id").primaryKey(), + serviceId: text("service_id") + .notNull() + .references(() => services.id, { onDelete: "cascade" }), + proxyServerId: text("proxy_server_id") + .notNull() + .references(() => servers.id, { onDelete: "cascade" }), + lastRequestAt: timestamp("last_request_at", { withTimezone: true }), + activeRequests: integer("active_requests").notNull().default(0), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .defaultNow() + .notNull() + .$onUpdate(() => new Date()), + }, + (table) => [ + uniqueIndex("serverless_service_activity_service_proxy_idx").on( + table.serviceId, + table.proxyServerId, + ), + index("serverless_service_activity_service_idx").on(table.serviceId), + index("serverless_service_activity_updated_at_idx").on(table.updatedAt), + ], +); + export const githubInstallations = pgTable("github_installations", { id: text("id").primaryKey(), installationId: integer("installation_id").notNull().unique(), diff --git a/web/db/types.ts b/web/db/types.ts index 04239040..ba95148c 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -9,6 +9,7 @@ import type { projects, rollouts, secrets, + serverlessServiceActivity, servers, servicePorts, serviceReplicas, @@ -27,6 +28,8 @@ export type ServicePort = typeof servicePorts.$inferSelect; export type ServiceVolume = typeof serviceVolumes.$inferSelect; export type ServiceReplica = typeof serviceReplicas.$inferSelect; export type Secret = typeof secrets.$inferSelect; +export type ServerlessServiceActivity = + typeof serverlessServiceActivity.$inferSelect; export type Deployment = typeof deployments.$inferSelect; export type DeploymentPort = typeof deploymentPorts.$inferSelect; export type Rollout = typeof rollouts.$inferSelect; diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index a6d49beb..cd71de5f 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -79,12 +79,13 @@ async function applyDeploymentErrors( id: deployments.id, serviceId: deployments.serviceId, rolloutId: deployments.rolloutId, + status: deployments.status, rolloutStatus: rollouts.status, serverName: servers.name, }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) - .innerJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) + .leftJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) .where( and( eq(deployments.id, error.deploymentId), @@ -93,27 +94,63 @@ async function applyDeploymentErrors( ) .then((rows) => rows[0]); - if ( - !deployment || - !deployment.rolloutId || - deployment.rolloutStatus !== "in_progress" - ) { + if (!deployment) { + continue; + } + + const isActiveRolloutDeployment = + deployment.rolloutId && deployment.rolloutStatus === "in_progress"; + const isServerlessWakeDeployment = + !deployment.rolloutId && deployment.status === "waking"; + + if (!isActiveRolloutDeployment && !isServerlessWakeDeployment) { continue; } const updated = await db .update(deployments) - .set({ status: "failed", failedStage: "deploying" }) + .set( + isServerlessWakeDeployment + ? { + ...markDeploymentUndesired("failed"), + failedStage: "serverless_wake", + healthStatus: null, + serverlessWakeStartedAt: null, + } + : { status: "failed", failedStage: "deploying" }, + ) .where( and( eq(deployments.id, deployment.id), - inArray(deployments.status, ["pending", "pulling", "starting"]), + inArray( + deployments.status, + isServerlessWakeDeployment + ? ["waking"] + : ["pending", "pulling", "starting"], + ), ), ) .returning({ id: deployments.id }); if (updated.length === 0) continue; + if (isServerlessWakeDeployment) { + console.log( + `[serverless:wake] deployment ${deployment.id} failed on server ${deployment.serverName}: ${formatDeploymentError(error.message)}`, + ); + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "service", + parentId: deployment.serviceId, + }), + ); + continue; + } + + if (!deployment.rolloutId) continue; + await ingestRolloutLog( deployment.rolloutId, deployment.serviceId, @@ -241,6 +278,7 @@ export async function applyStatusReport( .filter((id) => id !== ""); const activeStatuses = [ + "waking", "starting", "healthy", "running", @@ -302,7 +340,7 @@ export async function applyStatusReport( } if (!deployment) { - const stuckStatuses = ["pending", "pulling"] as const; + const stuckStatuses = ["pending", "pulling", "waking"] as const; const [stuckDeployment] = await db .select() .from(deployments) @@ -334,6 +372,7 @@ export async function applyStatusReport( containerId: container.containerId, status: newStatus, healthStatus: hasHealthCheck ? "starting" : "none", + serverlessWakeStartedAt: null, }) .where(eq(deployments.id, stuckDeployment.id)); @@ -370,6 +409,23 @@ export async function applyStatusReport( continue; } + if (deployment.status === "sleeping") { + const updateFields: Record = {}; + if (deployment.containerId !== container.containerId) { + updateFields.containerId = container.containerId; + } + if (deployment.healthStatus !== null) { + updateFields.healthStatus = null; + } + if (Object.keys(updateFields).length > 0) { + await db + .update(deployments) + .set(updateFields) + .where(eq(deployments.id, deployment.id)); + } + continue; + } + const updateFields: Record = { healthStatus }; let autohealRestartPayload: Record | null = null; let autohealRecreatePayload: Record | null = null; @@ -379,7 +435,11 @@ export async function applyStatusReport( updateFields.containerId = container.containerId; } - if (deployment.status === "pending" || deployment.status === "pulling") { + if ( + deployment.status === "pending" || + deployment.status === "pulling" || + deployment.status === "waking" + ) { if (container.status !== "running") { continue; } @@ -393,6 +453,9 @@ export async function applyStatusReport( const hasHealthCheck = service?.healthCheckCmd != null; const newStatus = hasHealthCheck ? "starting" : "healthy"; updateFields.status = newStatus; + if (deployment.status === "waking") { + updateFields.serverlessWakeStartedAt = null; + } if (hasHealthCheck) { updateFields.healthStatus = "starting"; } diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 96f83a03..ee0da943 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -22,12 +22,14 @@ type Server = typeof servers.$inferSelect; type Service = typeof services.$inferSelect; type Deployment = typeof deployments.$inferSelect; type ServicePort = typeof servicePorts.$inferSelect; +const SERVERLESS_GATEWAY_PORT = 18080; export type ExpectedContainer = { deploymentId: string; serviceId: string; serviceName: string; name: string; + desiredState: "running" | "stopped"; image: string; ipAddress: string | null; ports: Array<{ containerPort: number; hostPort: number }>; @@ -152,29 +154,33 @@ async function buildExpectedContainers( const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); if (serviceIds.length === 0) return []; - const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all([ - db - .select() - .from(services) - .where(and(inArray(services.id, serviceIds), isNull(services.deletedAt))), - fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), - db - .select({ - serviceId: secrets.serviceId, - key: secrets.key, - encryptedValue: secrets.encryptedValue, - }) - .from(secrets) - .where(inArray(secrets.serviceId, serviceIds)), - db - .select({ - serviceId: serviceVolumes.serviceId, - name: serviceVolumes.name, - containerPath: serviceVolumes.containerPath, - }) - .from(serviceVolumes) - .where(inArray(serviceVolumes.serviceId, serviceIds)), - ]); + const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all( + [ + db + .select() + .from(services) + .where( + and(inArray(services.id, serviceIds), isNull(services.deletedAt)), + ), + fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), + db + .select({ + serviceId: secrets.serviceId, + key: secrets.key, + encryptedValue: secrets.encryptedValue, + }) + .from(secrets) + .where(inArray(secrets.serviceId, serviceIds)), + db + .select({ + serviceId: serviceVolumes.serviceId, + name: serviceVolumes.name, + containerPath: serviceVolumes.containerPath, + }) + .from(serviceVolumes) + .where(inArray(serviceVolumes.serviceId, serviceIds)), + ], + ); return buildExpectedContainersFromRows({ deployments: serverDeployments, @@ -212,8 +218,13 @@ export function buildExpectedContainersFromRows({ secrets: SecretRow[]; volumes: VolumeRow[]; }): ExpectedContainer[] { - const servicesById = new Map(serviceRows.map((service) => [service.id, service])); - const portsByDeploymentId = groupBy(deploymentPortRows, (port) => port.deploymentId); + const servicesById = new Map( + serviceRows.map((service) => [service.id, service]), + ); + const portsByDeploymentId = groupBy( + deploymentPortRows, + (port) => port.deploymentId, + ); const secretsByServiceId = groupBy(secretRows, (secret) => secret.serviceId); const volumesByServiceId = groupBy(volumeRows, (volume) => volume.serviceId); @@ -230,6 +241,7 @@ export function buildExpectedContainersFromRows({ serviceId: dep.serviceId, serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, + desiredState: dep.status === "sleeping" ? "stopped" : "running", image: normalizeImage(service.image), ipAddress: dep.ipAddress, ports: (portsByDeploymentId.get(dep.id) ?? []) @@ -274,7 +286,10 @@ async function buildDnsRecords(allServices: Service[]) { ), ); - const ipsByServiceId = groupBy(dnsDeployments, (deployment) => deployment.serviceId); + const ipsByServiceId = groupBy( + dnsDeployments, + (deployment) => deployment.serviceId, + ); return allServices .flatMap((service) => { @@ -325,6 +340,11 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { serverId: server.id, ports, routableDeployments, + serverlessServiceIds: new Set( + allServices + .filter((service) => service.serverlessEnabled && !service.stateful) + .map((service) => service.id), + ), }); const routedDomains = routes.httpRoutes.map((r) => r.domain); const certificates = await getAllCertificatesForDomains(routedDomains); @@ -347,10 +367,12 @@ export function buildTraefikRoutes({ serverId, ports, routableDeployments, + serverlessServiceIds = new Set(), }: { serverId: string; ports: ServicePort[]; routableDeployments: RoutableDeploymentRow[]; + serverlessServiceIds?: Set; }) { const httpRoutes: HttpRoute[] = []; const tcpRoutes: TcpRoute[] = []; @@ -364,6 +386,18 @@ export function buildTraefikRoutes({ const serviceDeployments = deploymentsByServiceId.get(port.serviceId) ?? []; if (port.isPublic && port.protocol === "http" && port.domain) { + if (serverlessServiceIds.has(port.serviceId)) { + httpRoutes.push({ + id: port.domain, + domain: port.domain, + upstreams: [ + { url: `127.0.0.1:${SERVERLESS_GATEWAY_PORT}`, weight: 1 }, + ], + serviceId: port.serviceId, + }); + continue; + } + const localDeployments = serviceDeployments.filter( (d) => d.serverId === serverId && d.ipAddress, ); @@ -372,14 +406,18 @@ export function buildTraefikRoutes({ ); const upstreams = [ - ...localDeployments.map((d) => ({ - url: `${d.ipAddress}:${port.port}`, - weight: 5, - })).sort((a, b) => a.url.localeCompare(b.url)), - ...remoteDeployments.map((d) => ({ - url: `${d.ipAddress}:${port.port}`, - weight: 1, - })).sort((a, b) => a.url.localeCompare(b.url)), + ...localDeployments + .map((d) => ({ + url: `${d.ipAddress}:${port.port}`, + weight: 5, + })) + .sort((a, b) => a.url.localeCompare(b.url)), + ...remoteDeployments + .map((d) => ({ + url: `${d.ipAddress}:${port.port}`, + weight: 1, + })) + .sort((a, b) => a.url.localeCompare(b.url)), ]; if (upstreams.length > 0) { @@ -433,7 +471,9 @@ function buildHealthCheck(service: Service): ExpectedContainer["healthCheck"] { function buildEnv(secretRows: SecretRow[]) { const env: Record = {}; - for (const secret of secretRows.slice().sort((a, b) => a.key.localeCompare(b.key))) { + for (const secret of secretRows + .slice() + .sort((a, b) => a.key.localeCompare(b.key))) { env[secret.key] = secret.encryptedValue; } return env; diff --git a/web/lib/deployment-status.ts b/web/lib/deployment-status.ts index a1aa2cd4..beba89b5 100644 --- a/web/lib/deployment-status.ts +++ b/web/lib/deployment-status.ts @@ -17,9 +17,11 @@ const deploymentStatusCapabilities = { pending: { expected: true, routable: false, dns: false }, pulling: { expected: true, routable: false, dns: false }, starting: { expected: true, routable: false, dns: false }, + waking: { expected: true, routable: false, dns: false }, healthy: { expected: true, routable: true, dns: true }, running: { expected: true, routable: true, dns: true }, draining: { expected: true, routable: false, dns: false }, + sleeping: { expected: true, routable: false, dns: false }, stopping: { expected: false, routable: false, dns: false }, stopped: { expected: false, routable: false, dns: false }, failed: { expected: false, routable: false, dns: false }, @@ -35,7 +37,9 @@ export function markDeploymentUndesired(status: UndesiredDeploymentStatus) { return { status, desired: false }; } -function statusesWithCapability(capability: keyof DeploymentStatusCapabilities) { +function statusesWithCapability( + capability: keyof DeploymentStatusCapabilities, +) { return Object.entries(deploymentStatusCapabilities) .filter(([, capabilities]) => capabilities[capability]) .map(([status]) => status as DeploymentStatus); diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index 92c3a068..df26825c 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -8,6 +8,7 @@ import { checkAndPersistControlPlaneUpdate } from "@/lib/control-plane-updates"; import { checkAndRecoverStaleServers, checkAndRunScheduledDeployments, + checkAndSleepIdleServerlessServices, cleanupStaleItems, failTimedOutAgentUpgrades, } from "@/lib/scheduler"; @@ -124,6 +125,19 @@ export const staleItemsCleanup = inngest.createFunction( }, ); +export const serverlessSleepCheck = inngest.createFunction( + { + id: "cron-serverless-sleep-check", + triggers: [cron("* * * * *")], + singleton: { mode: "skip" }, + }, + async ({ step }) => { + await step.run("sleep-idle-serverless-services", async () => { + await checkAndSleepIdleServerlessServices(); + }); + }, +); + export const agentUpgradeTimeoutCheck = inngest.createFunction( { id: "cron-agent-upgrade-timeout-check", diff --git a/web/lib/inngest/functions/index.ts b/web/lib/inngest/functions/index.ts index ef862f53..63bb58df 100644 --- a/web/lib/inngest/functions/index.ts +++ b/web/lib/inngest/functions/index.ts @@ -9,6 +9,7 @@ export { oldBackupsCleanup, scheduledBackupsCheck, scheduledDeploymentsCheck, + serverlessSleepCheck, staleItemsCleanup, staleServerCheck, } from "./crons"; diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index c7177135..d34e3ce9 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -18,6 +18,7 @@ import { WORK_QUEUE_LEASE_DURATION_MS, WORK_QUEUE_MAX_ATTEMPTS, } from "@/lib/work-queue"; +import { sleepIdleServerlessServices } from "./serverless"; const STALE_THRESHOLD_MS = 75_000; // 75 seconds @@ -320,3 +321,17 @@ export async function cleanupStaleItems(): Promise { ); } } + +export async function checkAndSleepIdleServerlessServices(): Promise { + const result = await sleepIdleServerlessServices(); + if (result.deploymentsSlept > 0) { + console.log( + `[scheduler] slept ${result.deploymentsSlept} serverless deployment(s) across ${result.servicesSlept} service(s)`, + ); + } + if (result.wakingDeploymentsFailed > 0) { + console.log( + `[scheduler] failed ${result.wakingDeploymentsFailed} timed-out serverless wake deployment(s)`, + ); + } +} diff --git a/web/lib/serverless.ts b/web/lib/serverless.ts new file mode 100644 index 00000000..5cfd2a7f --- /dev/null +++ b/web/lib/serverless.ts @@ -0,0 +1,945 @@ +import { randomUUID } from "node:crypto"; +import { + and, + eq, + gt, + inArray, + isNotNull, + isNull, + lte, + or, + sql, +} from "drizzle-orm"; +import { db } from "@/db"; +import { + deployments, + rollouts, + serverlessServiceActivity, + servers, + servicePorts, + services, + workQueue, +} from "@/db/schema"; +import { markDeploymentUndesired } from "@/lib/deployment-status"; + +const READY_DEPLOYMENT_STATUSES = ["healthy", "running"] as const; +const WAKE_IN_PROGRESS_STATUSES = [ + "pending", + "pulling", + "starting", + "waking", +] as const; +const SLEEPABLE_DEPLOYMENT_STATUSES = ["healthy", "running"] as const; +const SERVERLESS_BUSY_DEPLOYMENT_STATUSES = [ + "pending", + "pulling", + "starting", + "waking", + "draining", + "stopping", +] as const; + +const WAKE_POLL_INTERVAL_MS = 500; +const MIN_ACTIVE_REQUEST_STALE_SECONDS = 120; + +export type ServerlessServiceRef = { + serviceId?: string; + host?: string; +}; + +export type ServerlessDeployment = { + id: string; + serverId: string; + ipAddress: string | null; + status: string; +}; + +export type ServerlessUpstream = { + url: string; +}; + +export type ServerlessWakeResult = { + status: + | "ready" + | "waking" + | "not_found" + | "not_serverless" + | "unsupported" + | "no_deployments"; + serviceId: string; + readyDeployments: ServerlessDeployment[]; + upstreams: ServerlessUpstream[]; + wakingDeployments: number; + queuedWakeServers: number; + minReadyReplicas: number; +}; + +export type ServerlessWaitResult = ServerlessWakeResult & { + timedOut: boolean; +}; + +export type ServerlessSleepResult = { + servicesChecked: number; + servicesSlept: number; + deploymentsSlept: number; + wakingDeploymentsFailed: number; +}; + +type ServerlessService = typeof services.$inferSelect; +type DeploymentRow = typeof deployments.$inferSelect; +type DbExecutor = Pick; + +type ServerlessTarget = { + serviceId: string; + port: number; +}; + +export async function resolveServerlessServiceId({ + serviceId, + host, +}: ServerlessServiceRef): Promise { + return ( + (await resolveServerlessTarget({ serviceId, host }))?.serviceId ?? null + ); +} + +export async function resolveServerlessTarget({ + serviceId, + host, +}: ServerlessServiceRef): Promise { + if (serviceId) { + const [port] = await db + .select({ port: servicePorts.port }) + .from(servicePorts) + .where( + and( + eq(servicePorts.serviceId, serviceId), + eq(servicePorts.protocol, "http"), + eq(servicePorts.isPublic, true), + isNotNull(servicePorts.domain), + ), + ) + .limit(1); + return port ? { serviceId, port: port.port } : null; + } + + const normalizedHost = normalizeHost(host); + if (!normalizedHost) return null; + + const [port] = await db + .select({ serviceId: servicePorts.serviceId, port: servicePorts.port }) + .from(servicePorts) + .innerJoin(services, eq(services.id, servicePorts.serviceId)) + .where( + and( + eq(servicePorts.domain, normalizedHost), + eq(servicePorts.protocol, "http"), + eq(servicePorts.isPublic, true), + isNull(services.deletedAt), + ), + ) + .limit(1); + + return port ? { serviceId: port.serviceId, port: port.port } : null; +} + +export async function recordServerlessRequestStart({ + serviceId, + proxyServerId, + now = new Date(), +}: { + serviceId: string; + proxyServerId: string; + now?: Date; +}) { + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + const service = await getServerlessService(tx, serviceId); + if (service) { + await resetStaleActiveRequests(tx, service, now); + } + await touchServerlessRequestStart(tx, { + serviceId, + proxyServerId, + now, + }); + }); +} + +export async function recordServerlessRequestFinish({ + serviceId, + proxyServerId, + now = new Date(), +}: { + serviceId: string; + proxyServerId: string; + now?: Date; +}) { + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + await touchServerlessRequestFinish(tx, { + serviceId, + proxyServerId, + now, + }); + }); +} + +export async function recordServerlessRequestHeartbeat({ + serviceId, + proxyServerId, + now = new Date(), +}: { + serviceId: string; + proxyServerId: string; + now?: Date; +}) { + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + await touchServerlessRequestHeartbeat(tx, { + serviceId, + proxyServerId, + now, + }); + }); +} + +async function touchServerlessRequestStart( + executor: DbExecutor, + { + serviceId, + proxyServerId, + now, + }: { + serviceId: string; + proxyServerId: string; + now: Date; + }, +) { + await executor + .insert(serverlessServiceActivity) + .values({ + id: randomUUID(), + serviceId, + proxyServerId, + lastRequestAt: now, + activeRequests: 1, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + serverlessServiceActivity.serviceId, + serverlessServiceActivity.proxyServerId, + ], + set: { + lastRequestAt: now, + activeRequests: sql`${serverlessServiceActivity.activeRequests} + 1`, + updatedAt: now, + }, + }); +} + +async function touchServerlessRequestFinish( + executor: DbExecutor, + { + serviceId, + proxyServerId, + now, + }: { + serviceId: string; + proxyServerId: string; + now: Date; + }, +) { + await executor + .insert(serverlessServiceActivity) + .values({ + id: randomUUID(), + serviceId, + proxyServerId, + lastRequestAt: now, + activeRequests: 0, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + serverlessServiceActivity.serviceId, + serverlessServiceActivity.proxyServerId, + ], + set: { + activeRequests: sql`GREATEST(${serverlessServiceActivity.activeRequests} - 1, 0)`, + lastRequestAt: now, + updatedAt: now, + }, + }); +} + +async function touchServerlessRequestHeartbeat( + executor: DbExecutor, + { + serviceId, + proxyServerId, + now, + }: { + serviceId: string; + proxyServerId: string; + now: Date; + }, +) { + await executor + .insert(serverlessServiceActivity) + .values({ + id: randomUUID(), + serviceId, + proxyServerId, + lastRequestAt: now, + activeRequests: 0, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + serverlessServiceActivity.serviceId, + serverlessServiceActivity.proxyServerId, + ], + set: { + lastRequestAt: now, + updatedAt: now, + }, + }); +} + +export async function wakeServerlessService({ + serviceId, + port, + proxyServerId, + now = new Date(), +}: { + serviceId: string; + port?: number; + proxyServerId: string; + now?: Date; +}): Promise { + return db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + + const [service] = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .limit(1); + + if (!service) return emptyWakeResult("not_found", serviceId); + const unsupported = await getServerlessUnsupportedReason(tx, service); + if (unsupported) return emptyWakeResult(unsupported, serviceId); + + await touchServerlessActivity(tx, { serviceId, proxyServerId, now }); + + const minReadyReplicas = getMinReadyReplicas(service); + const targetPort = port ?? (await getDefaultHttpPort(tx, serviceId)); + if (!targetPort) return emptyWakeResult("unsupported", serviceId); + + const readyDeployments = await getReadyDeployments(tx, serviceId); + if (readyDeployments.length >= minReadyReplicas) { + return { + status: "ready", + serviceId, + readyDeployments, + upstreams: buildUpstreams(readyDeployments, targetPort), + wakingDeployments: 0, + queuedWakeServers: 0, + minReadyReplicas, + }; + } + + const wakeableDeployments = await tx + .select() + .from(deployments) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.desired, true), + eq(deployments.status, "sleeping"), + ), + ); + + if (readyDeployments.length === 0 && wakeableDeployments.length === 0) { + const wakingDeployments = await countWakeInProgressDeployments( + tx, + serviceId, + ); + return { + status: wakingDeployments > 0 ? "waking" : "no_deployments", + serviceId, + readyDeployments, + upstreams: [], + wakingDeployments, + queuedWakeServers: 0, + minReadyReplicas, + }; + } + + if (wakeableDeployments.length > 0) { + const wakeableIds = wakeableDeployments.map( + (deployment) => deployment.id, + ); + // Wake all desired replicas; minReadyReplicas controls when requests can resume. + await tx + .update(deployments) + .set({ + status: "waking", + healthStatus: null, + unhealthyReportCount: 0, + serverlessWakeStartedAt: now, + }) + .where(inArray(deployments.id, wakeableIds)); + } + + const queuedWakeServers = await enqueueWakeReconcilers( + tx, + serviceId, + wakeableDeployments, + ); + const wakingDeployments = await countWakeInProgressDeployments( + tx, + serviceId, + ); + + return { + status: "waking", + serviceId, + readyDeployments, + upstreams: [], + wakingDeployments, + queuedWakeServers, + minReadyReplicas, + }; + }); +} + +export async function wakeAndWaitForServerlessService({ + serviceId, + port, + proxyServerId, + timeoutSeconds, + now = new Date(), +}: { + serviceId: string; + port?: number; + proxyServerId: string; + timeoutSeconds?: number; + now?: Date; +}): Promise { + const wakeResult = await wakeServerlessService({ + serviceId, + port, + proxyServerId, + now, + }); + + if (wakeResult.status !== "waking") { + return { ...wakeResult, timedOut: false }; + } + + const configuredTimeoutSeconds = + timeoutSeconds ?? (await getWakeTimeoutSeconds(serviceId)) ?? 300; + const targetPort = port ?? (await getDefaultHttpPort(db, serviceId)); + const timeoutMs = Math.max(1, configuredTimeoutSeconds) * 1000; + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + await sleep(WAKE_POLL_INTERVAL_MS); + const readyDeployments = await getReadyDeployments(db, serviceId); + if (targetPort && readyDeployments.length >= wakeResult.minReadyReplicas) { + return { + ...wakeResult, + status: "ready", + readyDeployments, + upstreams: buildUpstreams(readyDeployments, targetPort), + timedOut: false, + }; + } + } + + return { ...wakeResult, timedOut: true }; +} + +export async function sleepIdleServerlessServices({ + now = new Date(), +}: { + now?: Date; +} = {}): Promise { + const candidates = await db + .select() + .from(services) + .where( + and( + eq(services.serverlessEnabled, true), + eq(services.stateful, false), + isNull(services.deletedAt), + ), + ); + + let servicesSlept = 0; + let deploymentsSlept = 0; + let wakingDeploymentsFailed = 0; + + for (const service of candidates) { + wakingDeploymentsFailed += await failTimedOutWakingDeployments({ + service, + now, + }); + if (!(await hasPublicHttpEndpoint(db, service.id))) continue; + if (!(await isServiceIdle(db, service, now))) continue; + + const result = await sleepServerlessService({ + serviceId: service.id, + now, + }); + if (result.deploymentsSlept > 0) { + servicesSlept += 1; + deploymentsSlept += result.deploymentsSlept; + } + } + + return { + servicesChecked: candidates.length, + servicesSlept, + deploymentsSlept, + wakingDeploymentsFailed, + }; +} + +export async function sleepServerlessService({ + serviceId, + now = new Date(), +}: { + serviceId: string; + now?: Date; +}): Promise<{ deploymentsSlept: number }> { + return db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + + const [service] = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .limit(1); + + if (!service) return { deploymentsSlept: 0 }; + const unsupported = await getServerlessUnsupportedReason(tx, service); + if (unsupported) return { deploymentsSlept: 0 }; + if (!(await isServiceIdle(tx, service, now))) { + return { deploymentsSlept: 0 }; + } + + const sleepableDeployments = await tx + .select() + .from(deployments) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.desired, true), + inArray(deployments.status, SLEEPABLE_DEPLOYMENT_STATUSES), + isNotNull(deployments.containerId), + ), + ); + + if (sleepableDeployments.length === 0) return { deploymentsSlept: 0 }; + + await tx + .update(deployments) + .set({ + status: "sleeping", + healthStatus: null, + serverlessWakeStartedAt: null, + }) + .where( + inArray( + deployments.id, + sleepableDeployments.map((deployment) => deployment.id), + ), + ); + + for (const deployment of sleepableDeployments) { + await tx.insert(workQueue).values({ + id: randomUUID(), + serverId: deployment.serverId, + type: "sleep", + payload: JSON.stringify({ + reason: "serverless_idle_timeout", + deploymentId: deployment.id, + serviceId, + containerId: deployment.containerId, + }), + }); + } + + return { deploymentsSlept: sleepableDeployments.length }; + }); +} + +async function getServerlessUnsupportedReason( + executor: DbExecutor, + service: ServerlessService, +): Promise<"not_serverless" | "unsupported" | null> { + if (!service.serverlessEnabled) return "not_serverless"; + if (service.stateful) return "unsupported"; + if (!(await hasPublicHttpEndpoint(executor, service.id))) + return "unsupported"; + return null; +} + +async function hasPublicHttpEndpoint(executor: DbExecutor, serviceId: string) { + const [port] = await executor + .select({ id: servicePorts.id }) + .from(servicePorts) + .where( + and( + eq(servicePorts.serviceId, serviceId), + eq(servicePorts.protocol, "http"), + eq(servicePorts.isPublic, true), + isNotNull(servicePorts.domain), + ), + ) + .limit(1); + + return Boolean(port); +} + +export async function isProxyServer(serverId: string) { + const [server] = await db + .select({ isProxy: servers.isProxy }) + .from(servers) + .where(eq(servers.id, serverId)) + .limit(1); + return server?.isProxy === true; +} + +async function isServiceIdle( + executor: DbExecutor, + service: ServerlessService, + now: Date, +) { + const [activeRollout] = await executor + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, service.id), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1); + if (activeRollout) return false; + + const [busyDeployment] = await executor + .select({ id: deployments.id }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, service.id), + inArray(deployments.status, SERVERLESS_BUSY_DEPLOYMENT_STATUSES), + ), + ) + .limit(1); + if (busyDeployment) return false; + + await resetStaleActiveRequests(executor, service, now); + + const activityRows = await executor + .select({ + lastRequestAt: serverlessServiceActivity.lastRequestAt, + activeRequests: serverlessServiceActivity.activeRequests, + updatedAt: serverlessServiceActivity.updatedAt, + }) + .from(serverlessServiceActivity) + .where(eq(serverlessServiceActivity.serviceId, service.id)); + + const activeRequestStaleCutoff = getActiveRequestStaleCutoff(service, now); + const activeRequests = activityRows.reduce((total, row) => { + if (row.activeRequests <= 0) return total; + if (row.updatedAt <= activeRequestStaleCutoff) return total; + return total + row.activeRequests; + }, 0); + if (activeRequests > 0) return false; + + const lastRequestAt = maxDate( + activityRows + .map((row) => row.lastRequestAt) + .filter((value): value is Date => value !== null), + ); + const sleepableDeployments = await executor + .select({ createdAt: deployments.createdAt }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, service.id), + eq(deployments.desired, true), + inArray(deployments.status, SLEEPABLE_DEPLOYMENT_STATUSES), + ), + ); + + if (sleepableDeployments.length === 0) return false; + + const newestDeploymentCreatedAt = maxDate( + sleepableDeployments.map((deployment) => deployment.createdAt), + ); + const lastActivityAt = maxDate( + [lastRequestAt, newestDeploymentCreatedAt].filter( + (value): value is Date => value !== null, + ), + ); + if (!lastActivityAt) return false; + + const idleMs = now.getTime() - lastActivityAt.getTime(); + return idleMs >= service.serverlessSleepAfterSeconds * 1000; +} + +async function failTimedOutWakingDeployments({ + service, + now, +}: { + service: ServerlessService; + now: Date; +}) { + return db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${service.id}))`, + ); + + const timeoutCutoff = new Date( + now.getTime() - getWakeTimeoutSecondsForService(service) * 1000, + ); + const timedOut = await tx + .update(deployments) + .set({ + ...markDeploymentUndesired("failed"), + failedStage: "serverless_wake_timeout", + healthStatus: null, + serverlessWakeStartedAt: null, + }) + .where( + and( + eq(deployments.serviceId, service.id), + eq(deployments.status, "waking"), + eq(deployments.desired, true), + or( + lte(deployments.serverlessWakeStartedAt, timeoutCutoff), + and( + isNull(deployments.serverlessWakeStartedAt), + lte(deployments.createdAt, timeoutCutoff), + ), + ), + ), + ) + .returning({ id: deployments.id }); + + return timedOut.length; + }); +} + +async function touchServerlessActivity( + executor: DbExecutor, + { + serviceId, + proxyServerId, + now, + }: { + serviceId: string; + proxyServerId: string; + now: Date; + }, +) { + await executor + .insert(serverlessServiceActivity) + .values({ + id: randomUUID(), + serviceId, + proxyServerId, + lastRequestAt: now, + activeRequests: 0, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + serverlessServiceActivity.serviceId, + serverlessServiceActivity.proxyServerId, + ], + set: { + lastRequestAt: now, + updatedAt: now, + }, + }); +} + +async function getServerlessService( + executor: DbExecutor, + serviceId: string, +): Promise { + const [service] = await executor + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .limit(1); + return service ?? null; +} + +async function getWakeTimeoutSeconds(serviceId: string) { + const [service] = await db + .select({ wakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds }) + .from(services) + .where(eq(services.id, serviceId)) + .limit(1); + return service?.wakeTimeoutSeconds ?? null; +} + +async function getDefaultHttpPort(executor: DbExecutor, serviceId: string) { + const [port] = await executor + .select({ port: servicePorts.port }) + .from(servicePorts) + .where( + and( + eq(servicePorts.serviceId, serviceId), + eq(servicePorts.protocol, "http"), + eq(servicePorts.isPublic, true), + isNotNull(servicePorts.domain), + ), + ) + .limit(1); + return port?.port ?? null; +} + +async function getReadyDeployments( + executor: DbExecutor, + serviceId: string, +): Promise { + return executor + .select({ + id: deployments.id, + serverId: deployments.serverId, + ipAddress: deployments.ipAddress, + status: deployments.status, + }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.desired, true), + inArray(deployments.status, READY_DEPLOYMENT_STATUSES), + isNotNull(deployments.ipAddress), + ), + ); +} + +async function countWakeInProgressDeployments( + executor: DbExecutor, + serviceId: string, +) { + const rows = await executor + .select({ id: deployments.id }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.desired, true), + inArray(deployments.status, WAKE_IN_PROGRESS_STATUSES), + ), + ); + return rows.length; +} + +async function resetStaleActiveRequests( + executor: DbExecutor, + service: ServerlessService, + now: Date, +) { + const staleCutoff = getActiveRequestStaleCutoff(service, now); + await executor + .update(serverlessServiceActivity) + .set({ activeRequests: 0, updatedAt: now }) + .where( + and( + eq(serverlessServiceActivity.serviceId, service.id), + gt(serverlessServiceActivity.activeRequests, 0), + lte(serverlessServiceActivity.updatedAt, staleCutoff), + ), + ); +} + +function buildUpstreams( + deployments: ServerlessDeployment[], + port: number, +): ServerlessUpstream[] { + return deployments + .filter((deployment) => deployment.ipAddress) + .map((deployment) => ({ url: `${deployment.ipAddress}:${port}` })) + .sort((a, b) => a.url.localeCompare(b.url)); +} + +async function enqueueWakeReconcilers( + executor: DbExecutor, + serviceId: string, + wakeableDeployments: DeploymentRow[], +) { + const serverIds = new Set( + wakeableDeployments.map((deployment) => deployment.serverId), + ); + + for (const serverId of serverIds) { + await executor.insert(workQueue).values({ + id: randomUUID(), + serverId, + type: "wake", + payload: JSON.stringify({ + reason: "serverless_wake", + serviceId, + }), + }); + } + + return serverIds.size; +} + +function emptyWakeResult( + status: ServerlessWakeResult["status"], + serviceId: string, +): ServerlessWakeResult { + return { + status, + serviceId, + readyDeployments: [], + upstreams: [], + wakingDeployments: 0, + queuedWakeServers: 0, + minReadyReplicas: 1, + }; +} + +function getMinReadyReplicas(service: ServerlessService) { + return Math.max(1, service.serverlessMinReadyReplicas ?? 1); +} + +function getWakeTimeoutSecondsForService(service: ServerlessService) { + return Math.max(1, service.serverlessWakeTimeoutSeconds ?? 300); +} + +function getActiveRequestStaleCutoff(service: ServerlessService, now: Date) { + const staleSeconds = Math.max( + MIN_ACTIVE_REQUEST_STALE_SECONDS, + getWakeTimeoutSecondsForService(service), + ); + return new Date(now.getTime() - staleSeconds * 1000); +} + +function normalizeHost(host: string | undefined) { + return host?.trim().toLowerCase().replace(/:\d+$/, "") || null; +} + +function maxDate(values: Date[]) { + if (values.length === 0) return null; + return values.reduce((max, value) => (value > max ? value : max), values[0]); +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/web/tests/deployment-status.test.ts b/web/tests/deployment-status.test.ts new file mode 100644 index 00000000..69f58a4d --- /dev/null +++ b/web/tests/deployment-status.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { + dnsDeploymentStatuses, + expectedDeploymentStatuses, + routableDeploymentStatuses, +} from "@/lib/deployment-status"; + +describe("deployment status capabilities", () => { + it("keeps sleeping deployments expected but unroutable", () => { + expect(expectedDeploymentStatuses).toContain("sleeping"); + expect(routableDeploymentStatuses).not.toContain("sleeping"); + expect(dnsDeploymentStatuses).not.toContain("sleeping"); + }); + + it("keeps waking deployments expected but unroutable", () => { + expect(expectedDeploymentStatuses).toContain("waking"); + expect(routableDeploymentStatuses).not.toContain("waking"); + expect(dnsDeploymentStatuses).not.toContain("waking"); + }); +}); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index cf224ab0..f09e787b 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -19,11 +19,13 @@ describe("expected-state pure builders", () => { id: "dep_bbbbbbbb", serviceId: "svc_1", ipAddress: "10.0.0.2", + status: "sleeping", }, { id: "dep_aaaaaaaa", serviceId: "svc_1", ipAddress: "10.0.0.1", + status: "running", }, ] as any, services: [ @@ -61,6 +63,7 @@ describe("expected-state pure builders", () => { ]); expect(containers[0]).toMatchObject({ name: "svc_1-dep_aaaa", + desiredState: "running", image: "docker.io/library/nginx", ports: [ { containerPort: 80, hostPort: 80 }, @@ -79,6 +82,10 @@ describe("expected-state pure builders", () => { retries: 3, startPeriod: 30, }); + expect(containers[1]).toMatchObject({ + deploymentId: "dep_bbbbbbbb", + desiredState: "stopped", + }); }); it("keeps HTTP local upstreams before remote upstreams", () => { @@ -95,7 +102,11 @@ describe("expected-state pure builders", () => { }, ] as any, routableDeployments: [ - { serviceId: "svc_1", serverId: "server_remote", ipAddress: "10.0.0.2" }, + { + serviceId: "svc_1", + serverId: "server_remote", + ipAddress: "10.0.0.2", + }, { serviceId: "svc_1", serverId: "server_local", ipAddress: "10.0.0.3" }, { serviceId: "svc_1", serverId: "server_local", ipAddress: "10.0.0.1" }, ] as any, @@ -114,4 +125,31 @@ describe("expected-state pure builders", () => { }, ]); }); + + it("routes serverless HTTP services through the local wake gateway", () => { + const routes = buildTraefikRoutes({ + serverId: "server_local", + ports: [ + { + id: "port_1", + serviceId: "svc_serverless", + port: 3000, + isPublic: true, + protocol: "http", + domain: "sleepy.example.com", + }, + ] as any, + routableDeployments: [], + serverlessServiceIds: new Set(["svc_serverless"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "sleepy.example.com", + domain: "sleepy.example.com", + serviceId: "svc_serverless", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); }); From 37c6be3d86b3883a4f9495e278956c37618287c9 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 09:33:11 +1000 Subject: [PATCH 04/29] Fix serverless wake lifecycle edge cases --- agent/internal/agent/drift.go | 11 +- agent/internal/agent/handlers.go | 26 +++ agent/internal/serverless/gateway.go | 117 +++++++++- web/actions/projects.ts | 13 ++ .../service/details/serverless-section.tsx | 8 + web/lib/agent-status.ts | 32 ++- web/lib/agent/expected-state.ts | 71 +++--- web/lib/scheduler.ts | 4 +- web/lib/serverless.ts | 209 +++++++++++++++--- 9 files changed, 420 insertions(+), 71 deletions(-) diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index c7dc0397..a0a14c0d 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -270,7 +270,7 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS actualContainer := act if desiredContainerState(exp) == "stopped" { - if act.State == "running" || act.State == "paused" { + if shouldStopDesiredStoppedContainer(act.State) { actions = append(actions, reconcileAction{ Kind: actionStopExpectedContainer, Description: fmt.Sprintf("STOP %s (desired state: stopped)", exp.Name), @@ -414,6 +414,15 @@ func desiredContainerState(container agenthttp.ExpectedContainer) string { return "running" } +func shouldStopDesiredStoppedContainer(state string) bool { + switch state { + case "created", "exited", "stopped": + return false + default: + return true + } +} + func (a *Agent) applyReconcileAction(action reconcileAction) error { log.Printf("[reconcile] %s", action.Description) diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go index b8ce1263..2a91d5fa 100644 --- a/agent/internal/agent/handlers.go +++ b/agent/internal/agent/handlers.go @@ -70,6 +70,20 @@ func (a *Agent) ProcessSleep(item agenthttp.WorkQueueItem) error { return fmt.Errorf("sleep payload missing containerId") } + expectedState, fromCache, err := a.Client.GetExpectedStateWithFallback() + if err != nil { + log.Printf("[sleep] skipping container %s because expected state could not be verified: %v", Truncate(payload.ContainerID, 12), err) + return nil + } + if fromCache { + log.Printf("[sleep] skipping container %s because expected state came from cache", Truncate(payload.ContainerID, 12)) + return nil + } + if !deploymentWantsSleep(expectedState, payload.DeploymentID) { + log.Printf("[sleep] skipping stale sleep item for deployment %s", Truncate(payload.DeploymentID, 8)) + return nil + } + log.Printf("[sleep] removing container %s for deployment %s", Truncate(payload.ContainerID, 12), Truncate(payload.DeploymentID, 8)) if err := container.ForceRemove(payload.ContainerID); err != nil { @@ -79,6 +93,18 @@ func (a *Agent) ProcessSleep(item agenthttp.WorkQueueItem) error { return nil } +func deploymentWantsSleep(state *agenthttp.ExpectedState, deploymentID string) bool { + if state == nil || deploymentID == "" { + return false + } + for _, expected := range state.Containers { + if expected.DeploymentID == deploymentID { + return expected.DesiredState == "stopped" + } + } + return false +} + func (a *Agent) ProcessForceCleanup(item agenthttp.WorkQueueItem) error { var payload struct { ServiceID string `json:"serviceId"` diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 878250cc..88ae4592 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -9,6 +9,7 @@ import ( "net/http/httputil" "net/url" "strings" + "sync" "sync/atomic" "time" @@ -18,16 +19,35 @@ import ( const ( GatewayPort = 18080 activityHeartbeatInterval = 60 * time.Second + upstreamCacheTTL = 10 * time.Second ) type Gateway struct { - client *agenthttp.Client - counter uint64 - server *http.Server + client *agenthttp.Client + counter uint64 + server *http.Server + mu sync.Mutex + upstreamCache map[string]cachedUpstreams + wakeCalls map[string]*wakeCall +} + +type cachedUpstreams struct { + upstreams []agenthttp.ServerlessUpstream + expiresAt time.Time +} + +type wakeCall struct { + done chan struct{} + result *agenthttp.ServerlessWakeResult + err error } func NewGateway(client *agenthttp.Client) *Gateway { - return &Gateway{client: client} + return &Gateway{ + client: client, + upstreamCache: map[string]cachedUpstreams{}, + wakeCalls: map[string]*wakeCall{}, + } } func (g *Gateway) Start(ctx context.Context) error { @@ -68,15 +88,15 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - wakeResult, err := g.client.WakeServerlessService(host) + upstreams, err := g.getUpstreams(host) if err != nil { log.Printf("[serverless-gateway] wake failed for host %s: %v", host, err) http.Error(w, "service unavailable", http.StatusServiceUnavailable) return } - if len(wakeResult.Upstreams) == 0 { - log.Printf("[serverless-gateway] no upstreams for host %s after wake (status=%s)", host, wakeResult.Status) + if len(upstreams) == 0 { + log.Printf("[serverless-gateway] no upstreams for host %s after wake", host) http.Error(w, "service unavailable", http.StatusServiceUnavailable) return } @@ -93,7 +113,7 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { } }() - upstream := wakeResult.Upstreams[g.nextIndex(len(wakeResult.Upstreams))] + upstream := upstreams[g.nextIndex(len(upstreams))] target, err := url.Parse("http://" + upstream.Url) if err != nil { log.Printf("[serverless-gateway] invalid upstream %q for host %s: %v", upstream.Url, host, err) @@ -112,11 +132,88 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { } proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) { log.Printf("[serverless-gateway] proxy error for host %s to %s: %v", host, upstream.Url, err) + g.evictUpstreams(host) http.Error(w, "bad gateway", http.StatusBadGateway) } proxy.ServeHTTP(w, r) } +func (g *Gateway) getUpstreams(host string) ([]agenthttp.ServerlessUpstream, error) { + if upstreams, ok := g.cachedUpstreams(host); ok { + return upstreams, nil + } + + call, owner := g.beginWake(host) + if owner { + result, err := g.client.WakeServerlessService(host) + g.finishWake(host, call, result, err) + } + + <-call.done + if call.err != nil { + return nil, call.err + } + if call.result == nil || len(call.result.Upstreams) == 0 { + status := "" + if call.result != nil { + status = call.result.Status + } + return nil, fmt.Errorf("no upstreams returned after wake (status=%s)", status) + } + + return cloneUpstreams(call.result.Upstreams), nil +} + +func (g *Gateway) cachedUpstreams(host string) ([]agenthttp.ServerlessUpstream, bool) { + g.mu.Lock() + defer g.mu.Unlock() + + cached, ok := g.upstreamCache[host] + if !ok || time.Now().After(cached.expiresAt) { + if ok { + delete(g.upstreamCache, host) + } + return nil, false + } + + return cloneUpstreams(cached.upstreams), true +} + +func (g *Gateway) beginWake(host string) (*wakeCall, bool) { + g.mu.Lock() + defer g.mu.Unlock() + + if call, ok := g.wakeCalls[host]; ok { + return call, false + } + + call := &wakeCall{done: make(chan struct{})} + g.wakeCalls[host] = call + return call, true +} + +func (g *Gateway) finishWake(host string, call *wakeCall, result *agenthttp.ServerlessWakeResult, err error) { + g.mu.Lock() + defer g.mu.Unlock() + + call.result = result + call.err = err + if err == nil && result != nil && len(result.Upstreams) > 0 { + g.upstreamCache[host] = cachedUpstreams{ + upstreams: cloneUpstreams(result.Upstreams), + expiresAt: time.Now().Add(upstreamCacheTTL), + } + } + delete(g.wakeCalls, host) + close(call.done) +} + +func (g *Gateway) evictUpstreams(host string) { + g.mu.Lock() + defer g.mu.Unlock() + delete(g.upstreamCache, host) +} + func (g *Gateway) recordActivityHeartbeats(host string, done <-chan struct{}) { ticker := time.NewTicker(activityHeartbeatInterval) defer ticker.Stop() @@ -133,6 +230,10 @@ func (g *Gateway) recordActivityHeartbeats(host string, done <-chan struct{}) { } } +func cloneUpstreams(upstreams []agenthttp.ServerlessUpstream) []agenthttp.ServerlessUpstream { + return append([]agenthttp.ServerlessUpstream(nil), upstreams...) +} + func (g *Gateway) nextIndex(length int) int { if length <= 1 { return 0 diff --git a/web/actions/projects.ts b/web/actions/projects.ts index ab15a155..74641b76 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1084,7 +1084,9 @@ export async function updateServiceServerlessSettings( .update(deployments) .set({ status: "waking", + containerId: null, healthStatus: null, + unhealthyReportCount: 0, serverlessWakeStartedAt: now, }) .where( @@ -1249,8 +1251,10 @@ export async function updateServiceConfig( .delete(serviceReplicas) .where(eq(serviceReplicas.serviceId, serviceId)); + let totalReplicas = 0; for (const replica of config.replicas) { if (replica.count > 0) { + totalReplicas += replica.count; await db.insert(serviceReplicas).values({ id: randomUUID(), serviceId, @@ -1259,6 +1263,15 @@ export async function updateServiceConfig( }); } } + + await db + .update(services) + .set({ + serverlessMinReadyReplicas: sql`LEAST(${services.serverlessMinReadyReplicas}, ${Math.max(1, totalReplicas)})`, + }) + .where( + and(eq(services.id, serviceId), eq(services.serverlessEnabled, true)), + ); } return { success: true }; diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index 59796ef9..ef417d8a 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -2,6 +2,7 @@ import { Moon } from "lucide-react"; import { memo, useMemo, useState } from "react"; +import { toast } from "sonner"; import { updateServiceServerlessSettings } from "@/actions/projects"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -81,6 +82,13 @@ export const ServerlessSection = memo(function ServerlessSection({ minReadyReplicas: parsed.minReadyReplicas, }); onUpdate(); + toast.success("Serverless settings updated"); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to update serverless settings", + ); } finally { setIsSaving(false); } diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index cd71de5f..96a4ca72 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -80,11 +80,13 @@ async function applyDeploymentErrors( serviceId: deployments.serviceId, rolloutId: deployments.rolloutId, status: deployments.status, + serverlessEnabled: services.serverlessEnabled, rolloutStatus: rollouts.status, serverName: servers.name, }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) + .innerJoin(services, eq(deployments.serviceId, services.id)) .leftJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) .where( and( @@ -98,10 +100,11 @@ async function applyDeploymentErrors( continue; } + const isServerlessWakeDeployment = deployment.status === "waking"; const isActiveRolloutDeployment = - deployment.rolloutId && deployment.rolloutStatus === "in_progress"; - const isServerlessWakeDeployment = - !deployment.rolloutId && deployment.status === "waking"; + !isServerlessWakeDeployment && + deployment.rolloutId && + deployment.rolloutStatus === "in_progress"; if (!isActiveRolloutDeployment && !isServerlessWakeDeployment) { continue; @@ -111,12 +114,22 @@ async function applyDeploymentErrors( .update(deployments) .set( isServerlessWakeDeployment - ? { - ...markDeploymentUndesired("failed"), - failedStage: "serverless_wake", - healthStatus: null, - serverlessWakeStartedAt: null, - } + ? deployment.serverlessEnabled + ? { + status: "sleeping", + desired: true, + containerId: null, + failedStage: null, + healthStatus: null, + serverlessWakeStartedAt: null, + } + : { + ...markDeploymentUndesired("failed"), + containerId: null, + failedStage: "serverless_wake", + healthStatus: null, + serverlessWakeStartedAt: null, + } : { status: "failed", failedStage: "deploying" }, ) .where( @@ -278,7 +291,6 @@ export async function applyStatusReport( .filter((id) => id !== ""); const activeStatuses = [ - "waking", "starting", "healthy", "running", diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index ee0da943..5c68fac1 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -311,30 +311,47 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { if (!server.isProxy) return emptyConfig; const serviceIds = allServices.map((service) => service.id); - const [ports, routableDeployments] = await Promise.all([ - serviceIds.length > 0 - ? db - .select() - .from(servicePorts) - .where(inArray(servicePorts.serviceId, serviceIds)) - : Promise.resolve([]), - serviceIds.length > 0 - ? db - .select({ - serviceId: deployments.serviceId, - ipAddress: deployments.ipAddress, - serverId: deployments.serverId, - }) - .from(deployments) - .where( - and( - inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, routableDeploymentStatuses), - ), - ) - : Promise.resolve([]), - ]); + const [ports, routableDeployments, wakeRoutedDeployments] = await Promise.all( + [ + serviceIds.length > 0 + ? db + .select() + .from(servicePorts) + .where(inArray(servicePorts.serviceId, serviceIds)) + : Promise.resolve([]), + serviceIds.length > 0 + ? db + .select({ + serviceId: deployments.serviceId, + ipAddress: deployments.ipAddress, + serverId: deployments.serverId, + }) + .from(deployments) + .where( + and( + inArray(deployments.serviceId, serviceIds), + eq(deployments.desired, true), + inArray(deployments.status, routableDeploymentStatuses), + ), + ) + : Promise.resolve([]), + serviceIds.length > 0 + ? db + .select({ serviceId: deployments.serviceId }) + .from(deployments) + .where( + and( + inArray(deployments.serviceId, serviceIds), + eq(deployments.desired, true), + inArray(deployments.status, ["sleeping", "waking"]), + ), + ) + : Promise.resolve([]), + ], + ); + const wakeRoutedServiceIds = new Set( + wakeRoutedDeployments.map((deployment) => deployment.serviceId), + ); const routes = buildTraefikRoutes({ serverId: server.id, @@ -342,7 +359,11 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { routableDeployments, serverlessServiceIds: new Set( allServices - .filter((service) => service.serverlessEnabled && !service.stateful) + .filter( + (service) => + !service.stateful && + (service.serverlessEnabled || wakeRoutedServiceIds.has(service.id)), + ) .map((service) => service.id), ), }); diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index d34e3ce9..be8f4313 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -329,9 +329,9 @@ export async function checkAndSleepIdleServerlessServices(): Promise { `[scheduler] slept ${result.deploymentsSlept} serverless deployment(s) across ${result.servicesSlept} service(s)`, ); } - if (result.wakingDeploymentsFailed > 0) { + if (result.wakingDeploymentsReset > 0) { console.log( - `[scheduler] failed ${result.wakingDeploymentsFailed} timed-out serverless wake deployment(s)`, + `[scheduler] reset ${result.wakingDeploymentsReset} timed-out serverless wake deployment(s)`, ); } } diff --git a/web/lib/serverless.ts b/web/lib/serverless.ts index 5cfd2a7f..d2172933 100644 --- a/web/lib/serverless.ts +++ b/web/lib/serverless.ts @@ -17,6 +17,7 @@ import { serverlessServiceActivity, servers, servicePorts, + serviceReplicas, services, workQueue, } from "@/db/schema"; @@ -82,12 +83,15 @@ export type ServerlessSleepResult = { servicesChecked: number; servicesSlept: number; deploymentsSlept: number; - wakingDeploymentsFailed: number; + wakingDeploymentsReset: number; }; type ServerlessService = typeof services.$inferSelect; type DeploymentRow = typeof deployments.$inferSelect; -type DbExecutor = Pick; +type DbExecutor = Pick< + typeof db, + "delete" | "execute" | "insert" | "select" | "update" +>; type ServerlessTarget = { serviceId: string; @@ -329,12 +333,18 @@ export async function wakeServerlessService({ .limit(1); if (!service) return emptyWakeResult("not_found", serviceId); - const unsupported = await getServerlessUnsupportedReason(tx, service); + const inServerlessTransition = await hasDesiredWakeTransitionState( + tx, + serviceId, + ); + const unsupported = await getServerlessUnsupportedReason(tx, service, { + allowDisabledTransition: inServerlessTransition, + }); if (unsupported) return emptyWakeResult(unsupported, serviceId); await touchServerlessActivity(tx, { serviceId, proxyServerId, now }); - const minReadyReplicas = getMinReadyReplicas(service); + const minReadyReplicas = await getEffectiveMinReadyReplicas(tx, service); const targetPort = port ?? (await getDefaultHttpPort(tx, serviceId)); if (!targetPort) return emptyWakeResult("unsupported", serviceId); @@ -387,11 +397,13 @@ export async function wakeServerlessService({ .update(deployments) .set({ status: "waking", + containerId: null, healthStatus: null, unhealthyReportCount: 0, serverlessWakeStartedAt: now, }) .where(inArray(deployments.id, wakeableIds)); + await deletePendingSleepWorkItems(tx, wakeableDeployments); } const queuedWakeServers = await enqueueWakeReconcilers( @@ -468,26 +480,18 @@ export async function sleepIdleServerlessServices({ }: { now?: Date; } = {}): Promise { - const candidates = await db - .select() - .from(services) - .where( - and( - eq(services.serverlessEnabled, true), - eq(services.stateful, false), - isNull(services.deletedAt), - ), - ); + const candidates = await getServerlessSleepCandidates(); let servicesSlept = 0; let deploymentsSlept = 0; - let wakingDeploymentsFailed = 0; + let wakingDeploymentsReset = 0; for (const service of candidates) { - wakingDeploymentsFailed += await failTimedOutWakingDeployments({ + wakingDeploymentsReset += await resetTimedOutWakingDeployments({ service, now, }); + if (!service.serverlessEnabled) continue; if (!(await hasPublicHttpEndpoint(db, service.id))) continue; if (!(await isServiceIdle(db, service, now))) continue; @@ -505,7 +509,7 @@ export async function sleepIdleServerlessServices({ servicesChecked: candidates.length, servicesSlept, deploymentsSlept, - wakingDeploymentsFailed, + wakingDeploymentsReset, }; } @@ -550,6 +554,7 @@ export async function sleepServerlessService({ .update(deployments) .set({ status: "sleeping", + containerId: null, healthStatus: null, serverlessWakeStartedAt: null, }) @@ -581,8 +586,14 @@ export async function sleepServerlessService({ async function getServerlessUnsupportedReason( executor: DbExecutor, service: ServerlessService, + { + allowDisabledTransition = false, + }: { + allowDisabledTransition?: boolean; + } = {}, ): Promise<"not_serverless" | "unsupported" | null> { - if (!service.serverlessEnabled) return "not_serverless"; + if (!service.serverlessEnabled && !allowDisabledTransition) + return "not_serverless"; if (service.stateful) return "unsupported"; if (!(await hasPublicHttpEndpoint(executor, service.id))) return "unsupported"; @@ -606,6 +617,75 @@ async function hasPublicHttpEndpoint(executor: DbExecutor, serviceId: string) { return Boolean(port); } +async function getServerlessSleepCandidates(): Promise { + const [enabledServices, wakingTransitionRows] = await Promise.all([ + db + .select() + .from(services) + .where( + and( + eq(services.serverlessEnabled, true), + eq(services.stateful, false), + isNull(services.deletedAt), + ), + ), + db + .select({ serviceId: deployments.serviceId }) + .from(deployments) + .where( + and(eq(deployments.desired, true), eq(deployments.status, "waking")), + ), + ]); + const wakingServiceIds = unique( + wakingTransitionRows.map((row) => row.serviceId), + ); + const wakingTransitionServices = + wakingServiceIds.length > 0 + ? await db + .select() + .from(services) + .where( + and( + inArray(services.id, wakingServiceIds), + eq(services.stateful, false), + isNull(services.deletedAt), + ), + ) + : []; + + const servicesById = new Map(); + for (const service of enabledServices) { + servicesById.set(service.id, service); + } + for (const service of wakingTransitionServices) { + servicesById.set(service.id, service); + } + return [...servicesById.values()]; +} + +async function hasDesiredWakeTransitionState( + executor: DbExecutor, + serviceId: string, +) { + const [deployment] = await executor + .select({ id: deployments.id }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.desired, true), + inArray(deployments.status, [ + "sleeping", + "waking", + ...READY_DEPLOYMENT_STATUSES, + ]), + ), + ) + .limit(1); + + return Boolean(deployment); +} + export async function isProxyServer(serverId: string) { const [server] = await db .select({ isProxy: servers.isProxy }) @@ -695,7 +775,7 @@ async function isServiceIdle( return idleMs >= service.serverlessSleepAfterSeconds * 1000; } -async function failTimedOutWakingDeployments({ +async function resetTimedOutWakingDeployments({ service, now, }: { @@ -712,12 +792,7 @@ async function failTimedOutWakingDeployments({ ); const timedOut = await tx .update(deployments) - .set({ - ...markDeploymentUndesired("failed"), - failedStage: "serverless_wake_timeout", - healthStatus: null, - serverlessWakeStartedAt: null, - }) + .set(getTimedOutWakeUpdate(service)) .where( and( eq(deployments.serviceId, service.id), @@ -738,6 +813,27 @@ async function failTimedOutWakingDeployments({ }); } +function getTimedOutWakeUpdate(service: ServerlessService) { + if (service.serverlessEnabled) { + return { + status: "sleeping" as const, + desired: true, + containerId: null, + failedStage: null, + healthStatus: null, + serverlessWakeStartedAt: null, + }; + } + + return { + ...markDeploymentUndesired("failed"), + containerId: null, + failedStage: "serverless_wake_timeout", + healthStatus: null, + serverlessWakeStartedAt: null, + }; +} + async function touchServerlessActivity( executor: DbExecutor, { @@ -866,6 +962,29 @@ async function resetStaleActiveRequests( ); } +async function deletePendingSleepWorkItems( + executor: DbExecutor, + wakeableDeployments: DeploymentRow[], +) { + if (wakeableDeployments.length === 0) return; + const serviceId = wakeableDeployments[0]?.serviceId; + const serverIds = unique( + wakeableDeployments.map((deployment) => deployment.serverId), + ); + if (!serviceId || serverIds.length === 0) return; + + await executor + .delete(workQueue) + .where( + and( + eq(workQueue.type, "sleep"), + eq(workQueue.status, "pending"), + inArray(workQueue.serverId, serverIds), + sql`${workQueue.payload}::jsonb ->> 'serviceId' = ${serviceId}`, + ), + ); +} + function buildUpstreams( deployments: ServerlessDeployment[], port: number, @@ -919,6 +1038,42 @@ function getMinReadyReplicas(service: ServerlessService) { return Math.max(1, service.serverlessMinReadyReplicas ?? 1); } +async function getEffectiveMinReadyReplicas( + executor: DbExecutor, + service: ServerlessService, +) { + const [configuredReplicas, desiredDeployments] = await Promise.all([ + getConfiguredReplicaCount(executor, service.id), + countDesiredDeployments(executor, service.id), + ]); + const capacity = Math.max(1, desiredDeployments || configuredReplicas); + return Math.min(getMinReadyReplicas(service), capacity); +} + +async function getConfiguredReplicaCount( + executor: DbExecutor, + serviceId: string, +) { + const rows = await executor + .select({ count: serviceReplicas.count }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + return rows.reduce((total, row) => total + row.count, 0); +} + +async function countDesiredDeployments( + executor: DbExecutor, + serviceId: string, +) { + const rows = await executor + .select({ id: deployments.id }) + .from(deployments) + .where( + and(eq(deployments.serviceId, serviceId), eq(deployments.desired, true)), + ); + return rows.length; +} + function getWakeTimeoutSecondsForService(service: ServerlessService) { return Math.max(1, service.serverlessWakeTimeoutSeconds ?? 300); } @@ -940,6 +1095,10 @@ function maxDate(values: Date[]) { return values.reduce((max, value) => (value > max ? value : max), values[0]); } +function unique(items: T[]) { + return Array.from(new Set(items)); +} + function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } From a337957d08d42f2dad862ef0c1acacceaffa7610 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 10:11:33 +1000 Subject: [PATCH 05/29] Fix serverless wake review blockers --- agent/internal/agent/reporting.go | 14 ++- agent/internal/http/client.go | 5 +- agent/internal/serverless/gateway.go | 21 +++- agent/internal/serverless/gateway_test.go | 20 ++++ web/db/schema.ts | 4 + web/lib/agent-status.ts | 26 ++--- web/lib/agent/expected-state.ts | 33 +++++-- web/lib/serverless-wake-failures.ts | 36 +++++++ web/lib/serverless.ts | 106 ++++++++++++++------- web/tests/serverless-wake-failures.test.ts | 49 ++++++++++ 10 files changed, 244 insertions(+), 70 deletions(-) create mode 100644 agent/internal/serverless/gateway_test.go create mode 100644 web/lib/serverless-wake-failures.ts create mode 100644 web/tests/serverless-wake-failures.test.ts diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index e7459e0d..2842e01d 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -18,6 +18,8 @@ import ( var Version = "dev" +const serverlessGatewayCapability = "serverless_gateway" + var ( agentStartTime = time.Now() lastHealthCollect time.Time @@ -51,8 +53,9 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport report.NetworkHealth = health.CollectNetworkHealth("wg0") report.ContainerHealth = health.CollectContainerHealth() report.AgentHealth = &agenthttp.AgentHealth{ - Version: Version, - UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Version: Version, + UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Capabilities: a.agentCapabilities(), } lastHealthCollect = time.Now() log.Printf("[health] collected: cpu=%.1f%%, mem=%.1f%%, disk=%.1f%%, network=%v, containers=%d running", @@ -95,6 +98,13 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport return report } +func (a *Agent) agentCapabilities() []string { + if !a.IsProxy { + return nil + } + return []string{serverlessGatewayCapability} +} + func (a *Agent) RecordDeploymentError(deploymentID string, err error) { if deploymentID == "" || err == nil { return diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 1643e617..381a8a69 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -263,8 +263,9 @@ type Resources struct { } type AgentHealth struct { - Version string `json:"version"` - UptimeSecs int64 `json:"uptimeSecs"` + Version string `json:"version"` + UptimeSecs int64 `json:"uptimeSecs"` + Capabilities []string `json:"capabilities,omitempty"` } type StatusReport struct { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 88ae4592..99b7232d 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -2,6 +2,7 @@ package serverless import ( "context" + "errors" "fmt" "log" "net" @@ -88,8 +89,11 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - upstreams, err := g.getUpstreams(host) + upstreams, err := g.getUpstreams(r.Context(), host) if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } log.Printf("[serverless-gateway] wake failed for host %s: %v", host, err) http.Error(w, "service unavailable", http.StatusServiceUnavailable) return @@ -138,18 +142,25 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { proxy.ServeHTTP(w, r) } -func (g *Gateway) getUpstreams(host string) ([]agenthttp.ServerlessUpstream, error) { +func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.ServerlessUpstream, error) { if upstreams, ok := g.cachedUpstreams(host); ok { return upstreams, nil } call, owner := g.beginWake(host) if owner { - result, err := g.client.WakeServerlessService(host) - g.finishWake(host, call, result, err) + go func() { + result, err := g.client.WakeServerlessService(host) + g.finishWake(host, call, result, err) + }() + } + + select { + case <-call.done: + case <-ctx.Done(): + return nil, ctx.Err() } - <-call.done if call.err != nil { return nil, call.err } diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go new file mode 100644 index 00000000..cba1db28 --- /dev/null +++ b/agent/internal/serverless/gateway_test.go @@ -0,0 +1,20 @@ +package serverless + +import ( + "context" + "errors" + "testing" +) + +func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { + g := NewGateway(nil) + g.wakeCalls["sleepy.example.com"] = &wakeCall{done: make(chan struct{})} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := g.getUpstreams(ctx, "sleepy.example.com") + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} diff --git a/web/db/schema.ts b/web/db/schema.ts index 99db8063..e53b8278 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -272,6 +272,7 @@ export type ContainerHealth = { export type AgentHealth = { version: string; uptimeSecs: number; + capabilities?: string[]; }; export type AgentUpgradeStatus = @@ -569,6 +570,9 @@ export const deployments = pgTable( serverlessWakeStartedAt: timestamp("serverless_wake_started_at", { withTimezone: true, }), + serverlessWakeFailureCount: integer("serverless_wake_failure_count") + .notNull() + .default(0), createdAt: timestamp("created_at", { withTimezone: true }) .defaultNow() .notNull(), diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index 96a4ca72..956fe713 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -21,6 +21,7 @@ import { import { markDeploymentUndesired } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueWork } from "@/lib/work-queue"; @@ -80,6 +81,7 @@ async function applyDeploymentErrors( serviceId: deployments.serviceId, rolloutId: deployments.rolloutId, status: deployments.status, + serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, serverlessEnabled: services.serverlessEnabled, rolloutStatus: rollouts.status, serverName: servers.name, @@ -114,22 +116,11 @@ async function applyDeploymentErrors( .update(deployments) .set( isServerlessWakeDeployment - ? deployment.serverlessEnabled - ? { - status: "sleeping", - desired: true, - containerId: null, - failedStage: null, - healthStatus: null, - serverlessWakeStartedAt: null, - } - : { - ...markDeploymentUndesired("failed"), - containerId: null, - failedStage: "serverless_wake", - healthStatus: null, - serverlessWakeStartedAt: null, - } + ? getServerlessWakeFailureUpdate({ + serverlessEnabled: deployment.serverlessEnabled, + currentFailureCount: deployment.serverlessWakeFailureCount, + failedStage: "serverless_wake", + }) : { status: "failed", failedStage: "deploying" }, ) .where( @@ -385,6 +376,7 @@ export async function applyStatusReport( status: newStatus, healthStatus: hasHealthCheck ? "starting" : "none", serverlessWakeStartedAt: null, + serverlessWakeFailureCount: 0, }) .where(eq(deployments.id, stuckDeployment.id)); @@ -467,6 +459,7 @@ export async function applyStatusReport( updateFields.status = newStatus; if (deployment.status === "waking") { updateFields.serverlessWakeStartedAt = null; + updateFields.serverlessWakeFailureCount = 0; } if (hasHealthCheck) { updateFields.healthStatus = "starting"; @@ -626,6 +619,7 @@ export async function applyStatusReport( status: "healthy", autohealRestartCount: 0, autohealRecreateCount: 0, + serverlessWakeFailureCount: 0, }) .where(eq(deployments.id, deployment.id)); diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 5c68fac1..1ded78db 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -88,6 +88,7 @@ type DeploymentPortRow = { hostPort: number; containerPort: number; }; +const SERVERLESS_GATEWAY_CAPABILITY = "serverless_gateway"; type SecretRow = { serviceId: string; @@ -311,6 +312,10 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { if (!server.isProxy) return emptyConfig; const serviceIds = allServices.map((service) => service.id); + const supportsServerlessGateway = hasAgentCapability( + server, + SERVERLESS_GATEWAY_CAPABILITY, + ); const [ports, routableDeployments, wakeRoutedDeployments] = await Promise.all( [ serviceIds.length > 0 @@ -335,7 +340,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { ), ) : Promise.resolve([]), - serviceIds.length > 0 + supportsServerlessGateway && serviceIds.length > 0 ? db .select({ serviceId: deployments.serviceId }) .from(deployments) @@ -352,20 +357,24 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { const wakeRoutedServiceIds = new Set( wakeRoutedDeployments.map((deployment) => deployment.serviceId), ); + const serverlessServiceIds = supportsServerlessGateway + ? new Set( + allServices + .filter( + (service) => + !service.stateful && + (service.serverlessEnabled || + wakeRoutedServiceIds.has(service.id)), + ) + .map((service) => service.id), + ) + : new Set(); const routes = buildTraefikRoutes({ serverId: server.id, ports, routableDeployments, - serverlessServiceIds: new Set( - allServices - .filter( - (service) => - !service.stateful && - (service.serverlessEnabled || wakeRoutedServiceIds.has(service.id)), - ) - .map((service) => service.id), - ), + serverlessServiceIds, }); const routedDomains = routes.httpRoutes.map((r) => r.domain); const certificates = await getAllCertificatesForDomains(routedDomains); @@ -384,6 +393,10 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { }; } +function hasAgentCapability(server: Server, capability: string) { + return server.agentHealth?.capabilities?.includes(capability) === true; +} + export function buildTraefikRoutes({ serverId, ports, diff --git a/web/lib/serverless-wake-failures.ts b/web/lib/serverless-wake-failures.ts new file mode 100644 index 00000000..4f38947f --- /dev/null +++ b/web/lib/serverless-wake-failures.ts @@ -0,0 +1,36 @@ +import { markDeploymentUndesired } from "@/lib/deployment-status"; + +export const SERVERLESS_WAKE_FAILURE_LIMIT = 3; + +export function getServerlessWakeFailureUpdate({ + serverlessEnabled, + currentFailureCount, + failedStage, +}: { + serverlessEnabled: boolean; + currentFailureCount: number | null | undefined; + failedStage: string; +}) { + const nextFailureCount = (currentFailureCount ?? 0) + 1; + const baseUpdate = { + containerId: null, + healthStatus: null, + serverlessWakeStartedAt: null, + serverlessWakeFailureCount: nextFailureCount, + }; + + if (serverlessEnabled && nextFailureCount < SERVERLESS_WAKE_FAILURE_LIMIT) { + return { + ...baseUpdate, + status: "sleeping" as const, + desired: true, + failedStage: null, + }; + } + + return { + ...markDeploymentUndesired("failed"), + ...baseUpdate, + failedStage, + }; +} diff --git a/web/lib/serverless.ts b/web/lib/serverless.ts index d2172933..abf90592 100644 --- a/web/lib/serverless.ts +++ b/web/lib/serverless.ts @@ -21,7 +21,7 @@ import { services, workQueue, } from "@/db/schema"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; const READY_DEPLOYMENT_STATUSES = ["healthy", "running"] as const; const WAKE_IN_PROGRESS_STATUSES = [ @@ -371,18 +371,29 @@ export async function wakeServerlessService({ eq(deployments.status, "sleeping"), ), ); + const currentWakingDeployments = await countWakeInProgressDeployments( + tx, + serviceId, + ); - if (readyDeployments.length === 0 && wakeableDeployments.length === 0) { - const wakingDeployments = await countWakeInProgressDeployments( - tx, - serviceId, - ); + if (wakeableDeployments.length === 0) { + if (readyDeployments.length > 0 && currentWakingDeployments === 0) { + return { + status: "ready", + serviceId, + readyDeployments, + upstreams: buildUpstreams(readyDeployments, targetPort), + wakingDeployments: 0, + queuedWakeServers: 0, + minReadyReplicas, + }; + } return { - status: wakingDeployments > 0 ? "waking" : "no_deployments", + status: currentWakingDeployments > 0 ? "waking" : "no_deployments", serviceId, readyDeployments, upstreams: [], - wakingDeployments, + wakingDeployments: currentWakingDeployments, queuedWakeServers: 0, minReadyReplicas, }; @@ -470,6 +481,37 @@ export async function wakeAndWaitForServerlessService({ timedOut: false, }; } + const wakingDeployments = await countWakeInProgressDeployments( + db, + serviceId, + ); + if (targetPort && readyDeployments.length > 0 && wakingDeployments === 0) { + return { + ...wakeResult, + status: "ready", + readyDeployments, + upstreams: buildUpstreams(readyDeployments, targetPort), + wakingDeployments, + timedOut: false, + }; + } + } + + const readyDeployments = await getReadyDeployments(db, serviceId); + if (targetPort && readyDeployments.length > 0) { + return { + ...wakeResult, + status: "ready", + readyDeployments, + upstreams: buildUpstreams(readyDeployments, targetPort), + wakingDeployments: await countWakeInProgressDeployments(db, serviceId), + timedOut: false, + }; + } + + const service = await getServerlessService(db, serviceId); + if (service) { + await resetTimedOutWakingDeployments({ service, now: new Date() }); } return { ...wakeResult, timedOut: true }; @@ -790,9 +832,12 @@ async function resetTimedOutWakingDeployments({ const timeoutCutoff = new Date( now.getTime() - getWakeTimeoutSecondsForService(service) * 1000, ); - const timedOut = await tx - .update(deployments) - .set(getTimedOutWakeUpdate(service)) + const timedOutDeployments = await tx + .select({ + id: deployments.id, + serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, + }) + .from(deployments) .where( and( eq(deployments.serviceId, service.id), @@ -806,32 +851,23 @@ async function resetTimedOutWakingDeployments({ ), ), ), - ) - .returning({ id: deployments.id }); - - return timedOut.length; - }); -} + ); -function getTimedOutWakeUpdate(service: ServerlessService) { - if (service.serverlessEnabled) { - return { - status: "sleeping" as const, - desired: true, - containerId: null, - failedStage: null, - healthStatus: null, - serverlessWakeStartedAt: null, - }; - } + for (const deployment of timedOutDeployments) { + await tx + .update(deployments) + .set( + getServerlessWakeFailureUpdate({ + serverlessEnabled: service.serverlessEnabled, + currentFailureCount: deployment.serverlessWakeFailureCount, + failedStage: "serverless_wake_timeout", + }), + ) + .where(eq(deployments.id, deployment.id)); + } - return { - ...markDeploymentUndesired("failed"), - containerId: null, - failedStage: "serverless_wake_timeout", - healthStatus: null, - serverlessWakeStartedAt: null, - }; + return timedOutDeployments.length; + }); } async function touchServerlessActivity( diff --git a/web/tests/serverless-wake-failures.test.ts b/web/tests/serverless-wake-failures.test.ts new file mode 100644 index 00000000..1c5e6957 --- /dev/null +++ b/web/tests/serverless-wake-failures.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; + +describe("serverless wake failure policy", () => { + it("keeps enabled serverless deployments retryable below the failure limit", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: true, + currentFailureCount: 1, + failedStage: "serverless_wake", + }), + ).toMatchObject({ + status: "sleeping", + desired: true, + failedStage: null, + serverlessWakeFailureCount: 2, + }); + }); + + it("parks enabled serverless deployments after repeated wake failures", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: true, + currentFailureCount: 2, + failedStage: "serverless_wake", + }), + ).toMatchObject({ + status: "failed", + desired: false, + failedStage: "serverless_wake", + serverlessWakeFailureCount: 3, + }); + }); + + it("fails disabled transition deployments immediately", () => { + expect( + getServerlessWakeFailureUpdate({ + serverlessEnabled: false, + currentFailureCount: 0, + failedStage: "serverless_wake_timeout", + }), + ).toMatchObject({ + status: "failed", + desired: false, + failedStage: "serverless_wake_timeout", + serverlessWakeFailureCount: 1, + }); + }); +}); From aa265e2fca3ae22815458a08e4af0c9136d3d518 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 10:21:47 +1000 Subject: [PATCH 06/29] Guard timed-out serverless wake resets --- web/lib/serverless.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/web/lib/serverless.ts b/web/lib/serverless.ts index abf90592..1dce5f09 100644 --- a/web/lib/serverless.ts +++ b/web/lib/serverless.ts @@ -853,8 +853,9 @@ async function resetTimedOutWakingDeployments({ ), ); + let resetCount = 0; for (const deployment of timedOutDeployments) { - await tx + const updated = await tx .update(deployments) .set( getServerlessWakeFailureUpdate({ @@ -863,10 +864,18 @@ async function resetTimedOutWakingDeployments({ failedStage: "serverless_wake_timeout", }), ) - .where(eq(deployments.id, deployment.id)); + .where( + and( + eq(deployments.id, deployment.id), + eq(deployments.status, "waking"), + eq(deployments.desired, true), + ), + ) + .returning({ id: deployments.id }); + resetCount += updated.length; } - return timedOutDeployments.length; + return resetCount; }); } From f2d2f40a44d85dedf7e4274bfb734f03d995407f Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 11:16:00 +1000 Subject: [PATCH 07/29] Document serverless container architecture --- docs/agents/architecture.mdx | 22 +++++++++++++++ docs/architecture.mdx | 55 ++++++++++++++++++++++++++++++++++++ docs/services/scaling.mdx | 28 ++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/docs/agents/architecture.mdx b/docs/agents/architecture.mdx index 16e466bd..8ba5cdb1 100644 --- a/docs/agents/architecture.mdx +++ b/docs/agents/architecture.mdx @@ -75,3 +75,25 @@ reports, the command can be retried up to the fixed attempt limit. | `force_cleanup` | Force remove containers for a service | | `cleanup_volumes` | Remove volume directories for a service | | `deploy` | Handled through expected-state reconciliation | +| `wake` | Reconcile sleeping serverless deployments back to running | +| `sleep` | Remove a stopped serverless container after confirming fresh expected state | + +## Serverless Gateway + +Proxy agents run a local HTTP wake gateway on `127.0.0.1:18080`. The control +plane only emits Traefik routes to this gateway after the proxy reports the +`serverless_gateway` capability in its health status. + +For serverless service traffic, Traefik forwards the public request to the local +gateway. The gateway asks the control plane to wake the target service, waits for +ready upstreams, then proxies the original request to the selected container over +the WireGuard mesh. + +The gateway keeps a short upstream cache and collapses concurrent wake requests +for the same host. Request cancellation is respected while waiting for a shared +wake result. + +Sleep work is intentionally conservative. Before removing a local container, the +agent fetches fresh expected state and verifies that the deployment still wants +`desiredState: "stopped"`. Cached expected state is not enough to remove a +container. diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 6272152d..30a31a15 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -203,6 +203,61 @@ Proxy nodes receive routes and certificates from the control plane: Worker nodes do not run Traefik. +### Serverless Containers + +Stateless HTTP services can be configured as serverless. Serverless deployments +scale to zero by stopping their containers after an idle period, then wake on the +next public request. + +Serverless uses the same declarative expected-state model as normal +deployments: + +- A sleeping deployment remains `desired: true`. +- Expected state advertises the container with `desiredState: "stopped"`. +- The agent reconciles that state by stopping the local Podman container. +- A later wake changes the deployment back to `waking`, which makes expected + state advertise `desiredState: "running"` again. + +```mermaid +sequenceDiagram + participant User + participant Traefik + participant Gateway as Agent wake gateway + participant CP as Control plane + participant Agent + participant Container + + User->>Traefik: Request service domain + Traefik->>Gateway: Route to 127.0.0.1:18080 + Gateway->>CP: Wake service and wait + CP->>CP: Mark sleeping deployments waking + CP->>Agent: Lease wake/reconcile work + Agent->>Container: Start Podman container + Agent->>CP: Report healthy deployment + CP->>Gateway: Return ready upstreams + Gateway->>Container: Proxy original request +``` + +The route to the local wake gateway is emitted only when the proxy agent reports +the `serverless_gateway` capability. Older proxy agents keep normal direct +routes and are not pointed at `127.0.0.1:18080`. + +The wake gateway keeps the incoming request open while the control plane waits +for enough replicas to become ready. `Min Ready` controls how many ready +deployments are preferred before releasing queued requests. If fewer replicas +become ready and no wake is still in progress, the gateway serves any available +ready upstream instead of returning a 503. + +Sleep is driven by a scheduled control-plane check. A service is eligible to +sleep when it has no fresh active requests, no rollout in progress, no busy +deployments, and its last request or newest deployment is older than the +configured idle timeout. + +Wake failures are bounded. Transient failures return a deployment to `sleeping` +so a later request can retry. After repeated failures, the deployment is parked +as `failed` with a visible failed stage until a redeploy creates fresh +deployments. + ### Multiple Proxy Nodes The platform supports geographically distributed proxy nodes with proximity steering: diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 9e647b9a..bbcab900 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -9,6 +9,30 @@ Each service can run multiple replicas across your cluster. Configure how many r Replica count ranges from 1 to 10 per service. +## Serverless scaling + +Stateless HTTP services can be configured to sleep when idle. A sleeping +deployment keeps its desired replica record, but the container is stopped on the +agent. The next public HTTP request wakes the service and is held until an +upstream is ready or the wake timeout is reached. + +Serverless settings are configured per service: + +| Setting | Default | Description | +| --- | --- | --- | +| Enable serverless | Off | Allows stateless HTTP deployments to sleep and wake on request | +| Sleep after | `300s` | Idle period before running containers are stopped | +| Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | +| Min Ready | `1` | Preferred number of ready deployments before queued requests resume | + +`Min Ready` does not cap how many replicas are started. A cold wake starts all +desired sleeping replicas. `Min Ready` only controls when the held request can +resume. If fewer replicas become ready and no wake is still in progress, the +gateway serves the ready upstreams that exist instead of failing the request. + +Serverless services require at least one configured replica. `Min Ready` must be +between `1` and `10`, and cannot exceed the configured replica count. + ## Placement Services use manual placement. Select the target servers and replica counts explicitly before deploying. @@ -27,3 +51,7 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services are always pinned to their locked server. - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. +- Serverless scaling is supported only for stateless HTTP services with a public domain. +- Upgrade agents before enabling serverless. Older container-hosting agents treat + wake and sleep work as unknown commands, so sleep becomes a no-op until the + agent is updated. From 302373d8105ba1e74625aef03c31485e763155fa Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:25:45 +1000 Subject: [PATCH 08/29] Debounce serverless gateway activity --- agent/internal/serverless/gateway.go | 139 +++++++++++++++++++--- agent/internal/serverless/gateway_test.go | 71 +++++++++++ docs/agents/architecture.mdx | 6 + 3 files changed, 200 insertions(+), 16 deletions(-) diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 99b7232d..b20e8aff 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -18,18 +18,25 @@ import ( ) const ( - GatewayPort = 18080 - activityHeartbeatInterval = 60 * time.Second - upstreamCacheTTL = 10 * time.Second + GatewayPort = 18080 + upstreamCacheTTL = 10 * time.Second +) + +var ( + activityFinishDebounceInterval = 30 * time.Second + activityHeartbeatInterval = 60 * time.Second ) type Gateway struct { - client *agenthttp.Client - counter uint64 - server *http.Server - mu sync.Mutex + client controlPlaneClient + counter uint64 + server *http.Server + mu sync.Mutex + activityMu sync.Mutex + upstreamCache map[string]cachedUpstreams wakeCalls map[string]*wakeCall + activities map[string]*activityState } type cachedUpstreams struct { @@ -43,11 +50,25 @@ type wakeCall struct { err error } -func NewGateway(client *agenthttp.Client) *Gateway { +type controlPlaneClient interface { + WakeServerlessService(host string) (*agenthttp.ServerlessWakeResult, error) + RecordServerlessActivity(host, event string) error +} + +type activityState struct { + mu sync.Mutex + activeRequests int + started bool + finishTimer *time.Timer + stopHeartbeat chan struct{} +} + +func NewGateway(client controlPlaneClient) *Gateway { return &Gateway{ client: client, upstreamCache: map[string]cachedUpstreams{}, wakeCalls: map[string]*wakeCall{}, + activities: map[string]*activityState{}, } } @@ -65,6 +86,7 @@ func (g *Gateway) Start(ctx context.Context) error { go func() { <-ctx.Done() + g.stopAllActivities() shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := g.server.Shutdown(shutdownCtx); err != nil { @@ -105,16 +127,11 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := g.client.RecordServerlessActivity(host, "start"); err != nil { - log.Printf("[serverless-gateway] failed to record request start for %s: %v", host, err) + if err := g.beginActivity(host); err != nil { + log.Printf("[serverless-gateway] failed to record activity start for %s: %v", host, err) } - activityDone := make(chan struct{}) - go g.recordActivityHeartbeats(host, activityDone) defer func() { - close(activityDone) - if err := g.client.RecordServerlessActivity(host, "finish"); err != nil { - log.Printf("[serverless-gateway] failed to record request finish for %s: %v", host, err) - } + g.endActivity(host) }() upstream := upstreams[g.nextIndex(len(upstreams))] @@ -225,6 +242,96 @@ func (g *Gateway) evictUpstreams(host string) { delete(g.upstreamCache, host) } +func (g *Gateway) activity(host string) *activityState { + g.activityMu.Lock() + defer g.activityMu.Unlock() + + activity, ok := g.activities[host] + if !ok { + activity = &activityState{} + g.activities[host] = activity + } + return activity +} + +func (g *Gateway) beginActivity(host string) error { + activity := g.activity(host) + activity.mu.Lock() + defer activity.mu.Unlock() + + if activity.finishTimer != nil { + activity.finishTimer.Stop() + activity.finishTimer = nil + } + + activity.activeRequests += 1 + if activity.started { + return nil + } + + activity.started = true + activity.stopHeartbeat = make(chan struct{}) + go g.recordActivityHeartbeats(host, activity.stopHeartbeat) + + return g.client.RecordServerlessActivity(host, "start") +} + +func (g *Gateway) endActivity(host string) { + activity := g.activity(host) + activity.mu.Lock() + defer activity.mu.Unlock() + + if activity.activeRequests > 0 { + activity.activeRequests -= 1 + } + if activity.activeRequests > 0 || !activity.started { + return + } + + if activity.finishTimer != nil { + activity.finishTimer.Stop() + } + activity.finishTimer = time.AfterFunc(activityFinishDebounceInterval, func() { + g.finishActivity(host, activity) + }) +} + +func (g *Gateway) finishActivity(host string, activity *activityState) { + var err error + + activity.mu.Lock() + if activity.activeRequests > 0 || !activity.started { + activity.mu.Unlock() + return + } + + activity.finishTimer = nil + if activity.stopHeartbeat != nil { + close(activity.stopHeartbeat) + activity.stopHeartbeat = nil + } + activity.started = false + err = g.client.RecordServerlessActivity(host, "finish") + activity.mu.Unlock() + + if err != nil { + log.Printf("[serverless-gateway] failed to record activity finish for %s: %v", host, err) + } +} + +func (g *Gateway) stopAllActivities() { + g.activityMu.Lock() + activities := make(map[string]*activityState, len(g.activities)) + for host, activity := range g.activities { + activities[host] = activity + } + g.activityMu.Unlock() + + for host, activity := range activities { + g.finishActivity(host, activity) + } +} + func (g *Gateway) recordActivityHeartbeats(host string, done <-chan struct{}) { ticker := time.NewTicker(activityHeartbeatInterval) defer ticker.Stop() diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index cba1db28..489ab149 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -3,9 +3,35 @@ package serverless import ( "context" "errors" + "sync" "testing" + "time" + + agenthttp "techulus/cloud-agent/internal/http" ) +type fakeControlPlaneClient struct { + mu sync.Mutex + events []string +} + +func (f *fakeControlPlaneClient) WakeServerlessService(host string) (*agenthttp.ServerlessWakeResult, error) { + return nil, nil +} + +func (f *fakeControlPlaneClient) RecordServerlessActivity(host, event string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.events = append(f.events, event) + return nil +} + +func (f *fakeControlPlaneClient) snapshotEvents() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.events...) +} + func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { g := NewGateway(nil) g.wakeCalls["sleepy.example.com"] = &wakeCall{done: make(chan struct{})} @@ -18,3 +44,48 @@ func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { t.Fatalf("expected context.Canceled, got %v", err) } } + +func TestActivityIsDebouncedAcrossBusyWindow(t *testing.T) { + previousDebounce := activityFinishDebounceInterval + activityFinishDebounceInterval = 20 * time.Millisecond + t.Cleanup(func() { + activityFinishDebounceInterval = previousDebounce + }) + + client := &fakeControlPlaneClient{} + gateway := NewGateway(client) + if err := gateway.beginActivity("app.example.com"); err != nil { + t.Fatalf("first beginActivity failed: %v", err) + } + if err := gateway.beginActivity("app.example.com"); err != nil { + t.Fatalf("second beginActivity failed: %v", err) + } + + gateway.endActivity("app.example.com") + time.Sleep(2 * activityFinishDebounceInterval) + expectEvents(t, client.snapshotEvents(), []string{"start"}) + + gateway.endActivity("app.example.com") + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + got := client.snapshotEvents() + if len(got) == 2 { + expectEvents(t, got, []string{"start", "finish"}) + return + } + time.Sleep(5 * time.Millisecond) + } + expectEvents(t, client.snapshotEvents(), []string{"start", "finish"}) +} + +func expectEvents(t *testing.T, got []string, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("events = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("events = %v, want %v", got, want) + } + } +} diff --git a/docs/agents/architecture.mdx b/docs/agents/architecture.mdx index 8ba5cdb1..f568b068 100644 --- a/docs/agents/architecture.mdx +++ b/docs/agents/architecture.mdx @@ -93,6 +93,12 @@ The gateway keeps a short upstream cache and collapses concurrent wake requests for the same host. Request cancellation is respected while waiting for a shared wake result. +The gateway also debounces request activity before reporting it to the control +plane. It records one `start` when a host enters an active traffic window, +heartbeats while the window remains active, and records `finish` only after a +short quiet period. Individual proxied requests do not each write start and +finish activity rows. + Sleep work is intentionally conservative. Before removing a local container, the agent fetches fresh expected state and verifies that the deployment still wants `desiredState: "stopped"`. Cached expected state is not enough to remove a From 9f10008f58f54a9bad86a2513eccd1ac3a6d8bf1 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:03:07 +1000 Subject: [PATCH 09/29] Move serverless lifecycle to proxy agents --- agent/internal/agent/agent.go | 94 +- agent/internal/agent/drift.go | 6 +- agent/internal/agent/handlers.go | 49 - agent/internal/agent/reporting.go | 3 + agent/internal/agent/run.go | 6 +- agent/internal/agent/serverless.go | 143 ++ agent/internal/agent/workqueue.go | 2 - agent/internal/http/client.go | 116 +- agent/internal/serverless/gateway.go | 395 ++++-- agent/internal/serverless/gateway_test.go | 193 ++- docs/agents/architecture.mdx | 32 +- docs/architecture.mdx | 58 +- docs/services/scaling.mdx | 27 +- web/actions/projects.ts | 34 - .../api/v1/agent/serverless/activity/route.ts | 75 -- web/app/api/v1/agent/serverless/wake/route.ts | 95 -- web/app/api/v1/agent/status/route.ts | 13 +- web/db/schema.ts | 27 - web/db/types.ts | 3 - web/lib/agent-status.ts | 248 +++- web/lib/agent/expected-state.ts | 203 ++- web/lib/inngest/functions/crons.ts | 14 - web/lib/inngest/functions/index.ts | 1 - web/lib/scheduler.ts | 15 - web/lib/serverless.ts | 1149 ----------------- web/tests/expected-state.test.ts | 112 ++ 26 files changed, 1309 insertions(+), 1804 deletions(-) create mode 100644 agent/internal/agent/serverless.go delete mode 100644 web/app/api/v1/agent/serverless/activity/route.ts delete mode 100644 web/app/api/v1/agent/serverless/wake/route.ts delete mode 100644 web/lib/serverless.ts diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index 5461c72d..bf8bd72e 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -50,35 +50,40 @@ type ActualState struct { } type Agent struct { - state AgentState - stateMutex sync.RWMutex - reconcileRequested chan struct{} - statusReportRequested chan string - refreshMutex sync.Mutex - pendingExpectedStateRefresh bool - workMutex sync.Mutex - activeWorkItem *agenthttp.WorkQueueItem - pendingWorkResults []agenthttp.CompletedWorkItem - deploymentErrorMutex sync.Mutex - pendingDeploymentErrors []agenthttp.DeploymentError - Client *agenthttp.Client - Reconciler *reconcile.Reconciler - Config *Config - PublicIP string - PrivateIP string - DataDir string - expectedState *agenthttp.ExpectedState - processingStart time.Time - LogCollector *logs.Collector - TraefikLogCollector *logs.TraefikCollector - MetricsSender MetricsSender - Builder *build.Builder - isBuilding bool - buildMutex sync.Mutex - currentBuildID string - IsProxy bool - dnsInSync bool - DisableDNS bool + state AgentState + stateMutex sync.RWMutex + reconcileRequested chan struct{} + statusReportRequested chan string + refreshMutex sync.Mutex + pendingExpectedStateRefresh bool + workMutex sync.Mutex + activeWorkItem *agenthttp.WorkQueueItem + pendingWorkResults []agenthttp.CompletedWorkItem + deploymentErrorMutex sync.Mutex + pendingDeploymentErrors []agenthttp.DeploymentError + serverlessMutex sync.Mutex + pendingServerlessTransitions []agenthttp.ServerlessTransition + pendingServerlessSleep map[string]struct{} + expectedStateMutex sync.RWMutex + latestExpectedState *agenthttp.ExpectedState + Client *agenthttp.Client + Reconciler *reconcile.Reconciler + Config *Config + PublicIP string + PrivateIP string + DataDir string + expectedState *agenthttp.ExpectedState + processingStart time.Time + LogCollector *logs.Collector + TraefikLogCollector *logs.TraefikCollector + MetricsSender MetricsSender + Builder *build.Builder + isBuilding bool + buildMutex sync.Mutex + currentBuildID string + IsProxy bool + dnsInSync bool + DisableDNS bool } func NewAgent( @@ -94,21 +99,22 @@ func NewAgent( disableDNS bool, ) *Agent { return &Agent{ - state: StateIdle, - reconcileRequested: make(chan struct{}, 1), - statusReportRequested: make(chan string, 1), - Client: client, - Reconciler: reconciler, - Config: config, - PublicIP: publicIP, - PrivateIP: privateIP, - DataDir: dataDir, - LogCollector: logCollector, - TraefikLogCollector: traefikLogCollector, - MetricsSender: metricsSender, - Builder: builder, - IsProxy: isProxy, - DisableDNS: disableDNS, + state: StateIdle, + reconcileRequested: make(chan struct{}, 1), + statusReportRequested: make(chan string, 1), + Client: client, + Reconciler: reconciler, + Config: config, + PublicIP: publicIP, + PrivateIP: privateIP, + DataDir: dataDir, + LogCollector: logCollector, + TraefikLogCollector: traefikLogCollector, + MetricsSender: metricsSender, + Builder: builder, + IsProxy: isProxy, + DisableDNS: disableDNS, + pendingServerlessSleep: map[string]struct{}{}, } } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index a0a14c0d..2f59024a 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -103,6 +103,8 @@ func (a *Agent) handleIdle() { if fromCache { log.Printf("[idle] using cached state (CP unreachable)") } + a.SetLatestExpectedState(expected) + a.ReconcilePendingServerlessSleepWithExpected(expected, fromCache) actual, err := a.getActualState() if err != nil { @@ -251,7 +253,7 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS for id, exp := range expectedMap { if _, exists := actualMap[id]; !exists { - if desiredContainerState(exp) == "stopped" { + if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) { continue } expectedContainer := exp @@ -269,7 +271,7 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS expectedContainer := exp actualContainer := act - if desiredContainerState(exp) == "stopped" { + if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) { if shouldStopDesiredStoppedContainer(act.State) { actions = append(actions, reconcileAction{ Kind: actionStopExpectedContainer, diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go index 2a91d5fa..5dd68cb0 100644 --- a/agent/internal/agent/handlers.go +++ b/agent/internal/agent/handlers.go @@ -56,55 +56,6 @@ func (a *Agent) ProcessStop(item agenthttp.WorkQueueItem) error { return nil } -func (a *Agent) ProcessSleep(item agenthttp.WorkQueueItem) error { - var payload struct { - DeploymentID string `json:"deploymentId"` - ContainerID string `json:"containerId"` - } - - if err := json.Unmarshal([]byte(item.Payload), &payload); err != nil { - return fmt.Errorf("failed to parse sleep payload: %w", err) - } - - if payload.ContainerID == "" { - return fmt.Errorf("sleep payload missing containerId") - } - - expectedState, fromCache, err := a.Client.GetExpectedStateWithFallback() - if err != nil { - log.Printf("[sleep] skipping container %s because expected state could not be verified: %v", Truncate(payload.ContainerID, 12), err) - return nil - } - if fromCache { - log.Printf("[sleep] skipping container %s because expected state came from cache", Truncate(payload.ContainerID, 12)) - return nil - } - if !deploymentWantsSleep(expectedState, payload.DeploymentID) { - log.Printf("[sleep] skipping stale sleep item for deployment %s", Truncate(payload.DeploymentID, 8)) - return nil - } - - log.Printf("[sleep] removing container %s for deployment %s", Truncate(payload.ContainerID, 12), Truncate(payload.DeploymentID, 8)) - - if err := container.ForceRemove(payload.ContainerID); err != nil { - return fmt.Errorf("failed to remove sleeping container: %w", err) - } - - return nil -} - -func deploymentWantsSleep(state *agenthttp.ExpectedState, deploymentID string) bool { - if state == nil || deploymentID == "" { - return false - } - for _, expected := range state.Containers { - if expected.DeploymentID == deploymentID { - return expected.DesiredState == "stopped" - } - } - return false -} - func (a *Agent) ProcessForceCleanup(item agenthttp.WorkQueueItem) error { var payload struct { ServiceID string `json:"serviceId"` diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index 2842e01d..8eb4e4c9 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -71,6 +71,9 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport if c.DeploymentID == "" { continue } + if a.ShouldSuppressServerlessContainerReport(c.DeploymentID) { + continue + } status := "stopped" if c.State == "running" { diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index 801c221f..ce0a7700 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -34,7 +34,7 @@ func (a *Agent) Run(ctx context.Context) { } if a.IsProxy { - gateway := serverless.NewGateway(a.Client) + gateway := serverless.NewGateway(a) if err := gateway.Start(ctx); err != nil { log.Printf("[serverless-gateway] failed to start: %v", err) } @@ -117,12 +117,14 @@ func (a *Agent) reportStatus(reason string) { report := a.BuildStatusReport(true) reportedDeploymentErrorCount := len(report.DeploymentErrors) completed, active := a.SnapshotWorkStatus() - response, err := a.Client.ReportStatus(report, completed, active) + serverlessTransitions := a.SnapshotServerlessTransitions() + response, err := a.Client.ReportStatus(report, completed, active, serverlessTransitions) if err != nil { log.Printf("[status] failed to report (%s): %v", reason, err) return } a.ClearReportedDeploymentErrors(reportedDeploymentErrorCount) + a.ClearReportedServerlessTransitions(len(serverlessTransitions)) a.AcknowledgeWorkResults(response.AcceptedWorkItemResults, response.RejectedWorkItemResults) a.LogRejectedActiveWorkItems(response.RejectedActiveWorkItems) a.AcceptLeasedWorkItems(response.WorkItems) diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go new file mode 100644 index 00000000..cbfdb390 --- /dev/null +++ b/agent/internal/agent/serverless.go @@ -0,0 +1,143 @@ +package agent + +import ( + "log" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +func (a *Agent) SetLatestExpectedState(state *agenthttp.ExpectedState) { + a.expectedStateMutex.Lock() + defer a.expectedStateMutex.Unlock() + a.latestExpectedState = state +} + +func (a *Agent) ExpectedState() *agenthttp.ExpectedState { + a.expectedStateMutex.RLock() + defer a.expectedStateMutex.RUnlock() + return a.latestExpectedState +} + +func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { + return a.Reconciler.Deploy(expected) +} + +func (a *Agent) RemoveServerlessContainer(containerID string) error { + return container.ForceRemove(containerID) +} + +func (a *Agent) ListServerlessContainers() ([]container.Container, error) { + return container.List() +} + +func (a *Agent) GetServerlessContainerHealth(containerID string) string { + return container.GetHealthStatus(containerID) +} + +func (a *Agent) QueueServerlessTransition(transition agenthttp.ServerlessTransition) { + if transition.Type == "" || transition.DeploymentID == "" { + return + } + + a.serverlessMutex.Lock() + if transition.Type == "sleep" { + a.pendingServerlessSleep[transition.DeploymentID] = struct{}{} + } else if transition.Type == "wake_started" { + delete(a.pendingServerlessSleep, transition.DeploymentID) + } + a.pendingServerlessTransitions = append( + a.pendingServerlessTransitions, + transition, + ) + a.serverlessMutex.Unlock() + + a.RequestStatusReport("serverless " + transition.Type) +} + +func (a *Agent) SnapshotServerlessTransitions() []agenthttp.ServerlessTransition { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + return append([]agenthttp.ServerlessTransition(nil), a.pendingServerlessTransitions...) +} + +func (a *Agent) ClearReportedServerlessTransitions(count int) { + if count <= 0 { + return + } + + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + if count > len(a.pendingServerlessTransitions) { + count = len(a.pendingServerlessTransitions) + } + a.pendingServerlessTransitions = a.pendingServerlessTransitions[count:] +} + +func (a *Agent) HasPendingServerlessSleep(deploymentID string) bool { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + _, ok := a.pendingServerlessSleep[deploymentID] + return ok +} + +func (a *Agent) ShouldSuppressServerlessContainerReport(deploymentID string) bool { + if a.HasPendingServerlessSleep(deploymentID) { + return true + } + + a.expectedStateMutex.RLock() + defer a.expectedStateMutex.RUnlock() + if a.latestExpectedState == nil { + return false + } + for _, expected := range a.latestExpectedState.Containers { + if expected.DeploymentID == deploymentID { + return expected.DesiredState == "stopped" + } + } + return false +} + +func (a *Agent) ReconcilePendingServerlessSleepWithExpected(state *agenthttp.ExpectedState, fromCache bool) { + if state == nil || fromCache { + return + } + + desiredByDeploymentID := map[string]string{} + for _, expected := range state.Containers { + desiredByDeploymentID[expected.DeploymentID] = expected.DesiredState + } + + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + pendingSleepTransitions := map[string]struct{}{} + for _, transition := range a.pendingServerlessTransitions { + if transition.Type == "sleep" { + pendingSleepTransitions[transition.DeploymentID] = struct{}{} + } + } + + for deploymentID := range a.pendingServerlessSleep { + if _, stillReporting := pendingSleepTransitions[deploymentID]; stillReporting { + continue + } + delete(a.pendingServerlessSleep, deploymentID) + desiredState, ok := desiredByDeploymentID[deploymentID] + if ok && desiredState == "running" { + log.Printf("[serverless] sleep transition for deployment %s is not reflected in expected state; allowing reconcile", Truncate(deploymentID, 8)) + } + } +} + +func (a *Agent) QueueServerlessWakeFailure(deploymentID string, err error) { + if err == nil { + return + } + log.Printf("[serverless] wake failed for deployment %s: %v", Truncate(deploymentID, 8), err) + a.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_failed", + DeploymentID: deploymentID, + Error: err.Error(), + }) +} diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go index d61dd1bf..5302d4af 100644 --- a/agent/internal/agent/workqueue.go +++ b/agent/internal/agent/workqueue.go @@ -125,8 +125,6 @@ func (a *Agent) ProcessWorkItem(item agenthttp.WorkQueueItem) error { return a.ProcessRestart(item) case "stop": return a.ProcessStop(item) - case "sleep": - return a.ProcessSleep(item) case "deploy", "reconcile", "wake": a.RequestReconcile("reconcile work item " + Truncate(item.ID, 8)) return nil diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 381a8a69..ec529e9a 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -22,7 +22,6 @@ type Client struct { keyPair *crypto.KeyPair client *http.Client longClient *http.Client - wakeClient *http.Client dataDir string } @@ -38,9 +37,6 @@ func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string longClient: &http.Client{ Timeout: 40 * time.Second, }, - wakeClient: &http.Client{ - Timeout: 16 * time.Minute, - }, } } @@ -100,16 +96,22 @@ type Upstream struct { } type ServerlessUpstream struct { - Url string `json:"url"` + DeploymentID string `json:"deploymentId"` + ServerID string `json:"serverId"` + Url string `json:"url"` + Local bool `json:"local"` + AlwaysOn bool `json:"alwaysOn"` } -type ServerlessWakeResult struct { - Status string `json:"status"` - ServiceID string `json:"serviceId"` - Upstreams []ServerlessUpstream `json:"upstreams"` - WakingDeployments int `json:"wakingDeployments"` - QueuedWakeServers int `json:"queuedWakeServers"` - MinReadyReplicas int `json:"minReadyReplicas"` +type ServerlessRoute struct { + ServiceID string `json:"serviceId"` + Domain string `json:"domain"` + Port int `json:"port"` + SleepAfterSeconds int `json:"sleepAfterSeconds"` + WakeTimeoutSeconds int `json:"wakeTimeoutSeconds"` + MinReadyReplicas int `json:"minReadyReplicas"` + LocalDeploymentIDs []string `json:"localDeploymentIds"` + Upstreams []ServerlessUpstream `json:"upstreams"` } type TraefikRoute struct { @@ -156,6 +158,9 @@ type ExpectedState struct { Dns struct { Records []DnsRecord `json:"records"` } `json:"dns"` + Serverless struct { + Routes []ServerlessRoute `json:"routes"` + } `json:"serverless"` Traefik struct { HttpRoutes []TraefikRoute `json:"httpRoutes"` TCPRoutes []TraefikTCPRoute `json:"tcpRoutes"` @@ -293,6 +298,13 @@ type ActiveWorkItem struct { Attempt int `json:"attempt"` } +type ServerlessTransition struct { + Type string `json:"type"` + DeploymentID string `json:"deploymentId"` + ContainerID string `json:"containerId,omitempty"` + Error string `json:"error,omitempty"` +} + type BuildDetails struct { Build struct { ID string `json:"id"` @@ -396,7 +408,7 @@ type StatusResponse struct { WorkItems []WorkQueueItem `json:"workItems"` } -func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkItem, active []ActiveWorkItem) (*StatusResponse, error) { +func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkItem, active []ActiveWorkItem, serverlessTransitions []ServerlessTransition) (*StatusResponse, error) { payload := map[string]interface{}{ "statusReport": report, } @@ -406,6 +418,9 @@ func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkIte if len(active) > 0 { payload["activeWorkItems"] = active } + if len(serverlessTransitions) > 0 { + payload["serverlessTransitions"] = serverlessTransitions + } body, err := json.Marshal(payload) if err != nil { @@ -439,81 +454,6 @@ func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkIte return &statusResponse, nil } -func (c *Client) WakeServerlessService(host string) (*ServerlessWakeResult, error) { - payload := map[string]interface{}{ - "host": host, - "wait": true, - } - - body, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("failed to marshal serverless wake request: %w", err) - } - - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/serverless/wake", bytes.NewReader(body)) - if err != nil { - return nil, fmt.Errorf("failed to create serverless wake request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.wakeClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to wake serverless service: %w", err) - } - defer resp.Body.Close() - - var response struct { - OK bool `json:"ok"` - Error string `json:"error"` - Result ServerlessWakeResult `json:"result"` - } - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, fmt.Errorf("failed to decode serverless wake response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - if response.Error != "" { - return nil, fmt.Errorf("serverless wake failed with status %d: %s", resp.StatusCode, response.Error) - } - return nil, fmt.Errorf("serverless wake failed with status %d", resp.StatusCode) - } - - return &response.Result, nil -} - -func (c *Client) RecordServerlessActivity(host, event string) error { - payload := map[string]string{ - "host": host, - "event": event, - } - - body, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal serverless activity request: %w", err) - } - - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/serverless/activity", bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("failed to create serverless activity request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("failed to record serverless activity: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("serverless activity failed with status %d: %s", resp.StatusCode, string(body)) - } - - return nil -} - func (c *Client) GetBuildStatus(buildID string) (string, error) { req, err := http.NewRequest("GET", c.baseURL+"/api/v1/agent/builds/"+buildID+"/status", nil) if err != nil { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index b20e8aff..c2c6de1a 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "techulus/cloud-agent/internal/container" agenthttp "techulus/cloud-agent/internal/http" ) @@ -22,13 +23,10 @@ const ( upstreamCacheTTL = 10 * time.Second ) -var ( - activityFinishDebounceInterval = 30 * time.Second - activityHeartbeatInterval = 60 * time.Second -) +var wakePollInterval = 500 * time.Millisecond type Gateway struct { - client controlPlaneClient + runtime Runtime counter uint64 server *http.Server mu sync.Mutex @@ -39,33 +37,35 @@ type Gateway struct { activities map[string]*activityState } +type Runtime interface { + ExpectedState() *agenthttp.ExpectedState + DeployServerlessContainer(agenthttp.ExpectedContainer) error + RemoveServerlessContainer(containerID string) error + ListServerlessContainers() ([]container.Container, error) + GetServerlessContainerHealth(containerID string) string + QueueServerlessTransition(agenthttp.ServerlessTransition) +} + type cachedUpstreams struct { upstreams []agenthttp.ServerlessUpstream expiresAt time.Time } type wakeCall struct { - done chan struct{} - result *agenthttp.ServerlessWakeResult - err error -} - -type controlPlaneClient interface { - WakeServerlessService(host string) (*agenthttp.ServerlessWakeResult, error) - RecordServerlessActivity(host, event string) error + done chan struct{} + upstreams []agenthttp.ServerlessUpstream + err error } type activityState struct { mu sync.Mutex activeRequests int - started bool - finishTimer *time.Timer - stopHeartbeat chan struct{} + sleepTimer *time.Timer } -func NewGateway(client controlPlaneClient) *Gateway { +func NewGateway(runtime Runtime) *Gateway { return &Gateway{ - client: client, + runtime: runtime, upstreamCache: map[string]cachedUpstreams{}, wakeCalls: map[string]*wakeCall{}, activities: map[string]*activityState{}, @@ -122,17 +122,13 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if len(upstreams) == 0 { - log.Printf("[serverless-gateway] no upstreams for host %s after wake", host) + log.Printf("[serverless-gateway] no upstreams for host %s", host) http.Error(w, "service unavailable", http.StatusServiceUnavailable) return } - if err := g.beginActivity(host); err != nil { - log.Printf("[serverless-gateway] failed to record activity start for %s: %v", host, err) - } - defer func() { - g.endActivity(host) - }() + g.beginActivity(host) + defer g.endActivity(host) upstream := upstreams[g.nextIndex(len(upstreams))] target, err := url.Parse("http://" + upstream.Url) @@ -167,8 +163,8 @@ func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.Se call, owner := g.beginWake(host) if owner { go func() { - result, err := g.client.WakeServerlessService(host) - g.finishWake(host, call, result, err) + upstreams, err := g.resolveUpstreams(host) + g.finishWake(host, call, upstreams, err) }() } @@ -181,15 +177,146 @@ func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.Se if call.err != nil { return nil, call.err } - if call.result == nil || len(call.result.Upstreams) == 0 { - status := "" - if call.result != nil { - status = call.result.Status + return cloneUpstreams(call.upstreams), nil +} + +func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, error) { + state := g.runtime.ExpectedState() + route := findRoute(state, host) + if route == nil { + return nil, fmt.Errorf("no serverless route metadata for host %s", host) + } + + ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + if err != nil { + return nil, err + } + if len(sleepingLocalIDs) == 0 { + if len(ready) > 0 { + return ready, nil + } + return nil, fmt.Errorf("no ready upstreams for host %s", host) + } + + if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && hasAlwaysOnUpstream(ready)) { + go g.wakeLocalDeployments(route, state, sleepingLocalIDs) + return ready, nil + } + + g.wakeLocalDeployments(route, state, sleepingLocalIDs) + return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds) +} + +func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) ([]agenthttp.ServerlessUpstream, []string, error) { + actualContainers, err := g.runtime.ListServerlessContainers() + if err != nil { + return nil, nil, fmt.Errorf("failed to list local containers: %w", err) + } + + expectedByDeploymentID := expectedContainersByDeploymentID(state) + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + localIDs := map[string]struct{}{} + for _, deploymentID := range route.LocalDeploymentIDs { + localIDs[deploymentID] = struct{}{} + } + + upstreamsByURL := map[string]agenthttp.ServerlessUpstream{} + for _, upstream := range route.Upstreams { + if upstream.Local { + continue + } + upstreamsByURL[upstream.Url] = upstream + } + + sleepingLocalIDs := []string{} + for _, deploymentID := range route.LocalDeploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + continue + } + actual, isRunning := actualByDeploymentID[deploymentID] + if isRunning && g.isContainerReady(actual, expected) { + if upstream, ok := localUpstream(route, expected); ok { + upstreamsByURL[upstream.Url] = upstream + } + continue + } + if expected.DesiredState == "stopped" { + sleepingLocalIDs = append(sleepingLocalIDs, deploymentID) + } + } + + upstreams := make([]agenthttp.ServerlessUpstream, 0, len(upstreamsByURL)) + for _, upstream := range upstreamsByURL { + if upstream.Local { + if _, ok := localIDs[upstream.DeploymentID]; !ok { + continue + } } - return nil, fmt.Errorf("no upstreams returned after wake (status=%s)", status) + upstreams = append(upstreams, upstream) } + sortUpstreams(upstreams) + return upstreams, sleepingLocalIDs, nil +} - return cloneUpstreams(call.result.Upstreams), nil +func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState, deploymentIDs []string) { + expectedByDeploymentID := expectedContainersByDeploymentID(state) + for _, deploymentID := range deploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + continue + } + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_started", + DeploymentID: deploymentID, + }) + if err := g.runtime.DeployServerlessContainer(expected); err != nil { + log.Printf("[serverless-gateway] wake failed for deployment %s: %v", deploymentID, err) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_failed", + DeploymentID: deploymentID, + Error: err.Error(), + }) + } + } + g.evictUpstreams(route.Domain) +} + +func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int) ([]agenthttp.ServerlessUpstream, error) { + timeout := time.Duration(wakeTimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = 5 * time.Minute + } + deadline := time.Now().Add(timeout) + + for { + state := g.runtime.ExpectedState() + ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + if err != nil { + return nil, err + } + if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) { + return ready, nil + } + if time.Now().After(deadline) { + if len(ready) > 0 { + return ready, nil + } + return nil, fmt.Errorf("timed out waiting for local serverless wake") + } + time.Sleep(wakePollInterval) + } +} + +func (g *Gateway) isContainerReady(actual container.Container, expected agenthttp.ExpectedContainer) bool { + if actual.State != "running" { + return false + } + if expected.HealthCheck == nil || expected.HealthCheck.Cmd == "" { + return true + } + health := g.runtime.GetServerlessContainerHealth(actual.ID) + return health == "healthy" || health == "none" } func (g *Gateway) cachedUpstreams(host string) ([]agenthttp.ServerlessUpstream, bool) { @@ -220,15 +347,15 @@ func (g *Gateway) beginWake(host string) (*wakeCall, bool) { return call, true } -func (g *Gateway) finishWake(host string, call *wakeCall, result *agenthttp.ServerlessWakeResult, err error) { +func (g *Gateway) finishWake(host string, call *wakeCall, upstreams []agenthttp.ServerlessUpstream, err error) { g.mu.Lock() defer g.mu.Unlock() - call.result = result + call.upstreams = cloneUpstreams(upstreams) call.err = err - if err == nil && result != nil && len(result.Upstreams) > 0 { + if err == nil && len(upstreams) > 0 { g.upstreamCache[host] = cachedUpstreams{ - upstreams: cloneUpstreams(result.Upstreams), + upstreams: cloneUpstreams(upstreams), expiresAt: time.Now().Add(upstreamCacheTTL), } } @@ -254,26 +381,16 @@ func (g *Gateway) activity(host string) *activityState { return activity } -func (g *Gateway) beginActivity(host string) error { +func (g *Gateway) beginActivity(host string) { activity := g.activity(host) activity.mu.Lock() defer activity.mu.Unlock() - if activity.finishTimer != nil { - activity.finishTimer.Stop() - activity.finishTimer = nil + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + activity.sleepTimer = nil } - activity.activeRequests += 1 - if activity.started { - return nil - } - - activity.started = true - activity.stopHeartbeat = make(chan struct{}) - go g.recordActivityHeartbeats(host, activity.stopHeartbeat) - - return g.client.RecordServerlessActivity(host, "start") } func (g *Gateway) endActivity(host string) { @@ -284,70 +401,182 @@ func (g *Gateway) endActivity(host string) { if activity.activeRequests > 0 { activity.activeRequests -= 1 } - if activity.activeRequests > 0 || !activity.started { + if activity.activeRequests > 0 { return } - if activity.finishTimer != nil { - activity.finishTimer.Stop() + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + return } - activity.finishTimer = time.AfterFunc(activityFinishDebounceInterval, func() { - g.finishActivity(host, activity) + delay := time.Duration(route.SleepAfterSeconds) * time.Second + if delay <= 0 { + delay = 5 * time.Minute + } + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + } + activity.sleepTimer = time.AfterFunc(delay, func() { + g.sleepHost(host) }) } -func (g *Gateway) finishActivity(host string, activity *activityState) { - var err error - +func (g *Gateway) sleepHost(host string) { + activity := g.activity(host) activity.mu.Lock() - if activity.activeRequests > 0 || !activity.started { + if activity.activeRequests > 0 { activity.mu.Unlock() return } + activity.sleepTimer = nil + activity.mu.Unlock() - activity.finishTimer = nil - if activity.stopHeartbeat != nil { - close(activity.stopHeartbeat) - activity.stopHeartbeat = nil + state := g.runtime.ExpectedState() + route := findRoute(state, host) + if route == nil { + return } - activity.started = false - err = g.client.RecordServerlessActivity(host, "finish") - activity.mu.Unlock() + actualContainers, err := g.runtime.ListServerlessContainers() if err != nil { - log.Printf("[serverless-gateway] failed to record activity finish for %s: %v", host, err) + log.Printf("[serverless-gateway] failed to list containers before sleep for %s: %v", host, err) + return } + + expectedByDeploymentID := expectedContainersByDeploymentID(state) + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + for _, deploymentID := range route.LocalDeploymentIDs { + expected, ok := expectedByDeploymentID[deploymentID] + if !ok { + continue + } + actual, ok := actualByDeploymentID[deploymentID] + if !ok || actual.State != "running" { + continue + } + if expected.DesiredState != "stopped" { + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "sleep", + DeploymentID: deploymentID, + ContainerID: actual.ID, + }) + } + if err := g.runtime.RemoveServerlessContainer(actual.ID); err != nil { + log.Printf("[serverless-gateway] failed to sleep deployment %s: %v", deploymentID, err) + continue + } + } + g.evictUpstreams(host) } func (g *Gateway) stopAllActivities() { g.activityMu.Lock() - activities := make(map[string]*activityState, len(g.activities)) - for host, activity := range g.activities { - activities[host] = activity + activities := make([]*activityState, 0, len(g.activities)) + for _, activity := range g.activities { + activities = append(activities, activity) } g.activityMu.Unlock() - for host, activity := range activities { - g.finishActivity(host, activity) + for _, activity := range activities { + activity.mu.Lock() + if activity.sleepTimer != nil { + activity.sleepTimer.Stop() + activity.sleepTimer = nil + } + activity.mu.Unlock() + } +} + +func findRoute(state *agenthttp.ExpectedState, host string) *agenthttp.ServerlessRoute { + if state == nil { + return nil + } + normalizedHost := normalizeHost(host) + for i := range state.Serverless.Routes { + if normalizeHost(state.Serverless.Routes[i].Domain) == normalizedHost { + return &state.Serverless.Routes[i] + } + } + return nil +} + +func expectedContainersByDeploymentID(state *agenthttp.ExpectedState) map[string]agenthttp.ExpectedContainer { + containersByDeploymentID := map[string]agenthttp.ExpectedContainer{} + if state == nil { + return containersByDeploymentID } + for _, expected := range state.Containers { + containersByDeploymentID[expected.DeploymentID] = expected + } + return containersByDeploymentID } -func (g *Gateway) recordActivityHeartbeats(host string, done <-chan struct{}) { - ticker := time.NewTicker(activityHeartbeatInterval) - defer ticker.Stop() +func actualContainersByDeploymentID(containers []container.Container) map[string]container.Container { + containersByDeploymentID := map[string]container.Container{} + for _, actual := range containers { + if actual.DeploymentID != "" { + containersByDeploymentID[actual.DeploymentID] = actual + } + } + return containersByDeploymentID +} - for { - select { - case <-ticker.C: - if err := g.client.RecordServerlessActivity(host, "heartbeat"); err != nil { - log.Printf("[serverless-gateway] failed to record request heartbeat for %s: %v", host, err) +func localUpstream(route *agenthttp.ServerlessRoute, expected agenthttp.ExpectedContainer) (agenthttp.ServerlessUpstream, bool) { + if expected.IPAddress != "" { + return agenthttp.ServerlessUpstream{ + DeploymentID: expected.DeploymentID, + Url: fmt.Sprintf("%s:%d", expected.IPAddress, route.Port), + Local: true, + }, true + } + + for _, port := range expected.Ports { + if port.ContainerPort == route.Port && port.HostPort > 0 { + return agenthttp.ServerlessUpstream{ + DeploymentID: expected.DeploymentID, + Url: fmt.Sprintf("127.0.0.1:%d", port.HostPort), + Local: true, + }, true + } + } + return agenthttp.ServerlessUpstream{}, false +} + +func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool { + for _, upstream := range upstreams { + if upstream.AlwaysOn { + return true + } + } + return false +} + +func sortUpstreams(upstreams []agenthttp.ServerlessUpstream) { + for i := 0; i < len(upstreams); i++ { + for j := i + 1; j < len(upstreams); j++ { + if compareUpstream(upstreams[j], upstreams[i]) < 0 { + upstreams[i], upstreams[j] = upstreams[j], upstreams[i] } - case <-done: - return } } } +func compareUpstream(a, b agenthttp.ServerlessUpstream) int { + if a.AlwaysOn != b.AlwaysOn { + if a.AlwaysOn { + return -1 + } + return 1 + } + if a.Local != b.Local { + if a.Local { + return -1 + } + return 1 + } + return strings.Compare(a.Url, b.Url) +} + func cloneUpstreams(upstreams []agenthttp.ServerlessUpstream) []agenthttp.ServerlessUpstream { return append([]agenthttp.ServerlessUpstream(nil), upstreams...) } diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index 489ab149..5f0a2f89 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -7,33 +7,80 @@ import ( "testing" "time" + "techulus/cloud-agent/internal/container" agenthttp "techulus/cloud-agent/internal/http" ) -type fakeControlPlaneClient struct { - mu sync.Mutex - events []string +type fakeRuntime struct { + mu sync.Mutex + state *agenthttp.ExpectedState + containers []container.Container + transitions []agenthttp.ServerlessTransition + removed []string + deployCalls int + deployErr error } -func (f *fakeControlPlaneClient) WakeServerlessService(host string) (*agenthttp.ServerlessWakeResult, error) { - return nil, nil +func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { + f.mu.Lock() + defer f.mu.Unlock() + return f.state } -func (f *fakeControlPlaneClient) RecordServerlessActivity(host, event string) error { +func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { f.mu.Lock() defer f.mu.Unlock() - f.events = append(f.events, event) + f.deployCalls += 1 + if f.deployErr != nil { + return f.deployErr + } + f.containers = append(f.containers, container.Container{ + ID: "ctr-" + expected.DeploymentID, + State: "running", + DeploymentID: expected.DeploymentID, + ServiceID: expected.ServiceID, + }) + return nil +} + +func (f *fakeRuntime) RemoveServerlessContainer(containerID string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, containerID) + next := f.containers[:0] + for _, actual := range f.containers { + if actual.ID != containerID { + next = append(next, actual) + } + } + f.containers = next return nil } -func (f *fakeControlPlaneClient) snapshotEvents() []string { +func (f *fakeRuntime) ListServerlessContainers() ([]container.Container, error) { f.mu.Lock() defer f.mu.Unlock() - return append([]string(nil), f.events...) + return append([]container.Container(nil), f.containers...), nil +} + +func (f *fakeRuntime) GetServerlessContainerHealth(containerID string) string { + return "healthy" +} + +func (f *fakeRuntime) QueueServerlessTransition(transition agenthttp.ServerlessTransition) { + f.mu.Lock() + defer f.mu.Unlock() + f.transitions = append(f.transitions, transition) +} + +func (f *fakeRuntime) snapshot() ([]agenthttp.ServerlessTransition, []string, int) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]agenthttp.ServerlessTransition(nil), f.transitions...), append([]string(nil), f.removed...), f.deployCalls } func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { - g := NewGateway(nil) + g := NewGateway(&fakeRuntime{}) g.wakeCalls["sleepy.example.com"] = &wakeCall{done: make(chan struct{})} ctx, cancel := context.WithCancel(context.Background()) @@ -45,47 +92,115 @@ func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { } } -func TestActivityIsDebouncedAcrossBusyWindow(t *testing.T) { - previousDebounce := activityFinishDebounceInterval - activityFinishDebounceInterval = 20 * time.Millisecond - t.Cleanup(func() { - activityFinishDebounceInterval = previousDebounce - }) +func TestRequestCanUseAlwaysOnWorkerWhileLocalDeploymentWakes(t *testing.T) { + runtime := &fakeRuntime{state: testExpectedState("stopped")} + gateway := NewGateway(runtime) - client := &fakeControlPlaneClient{} - gateway := NewGateway(client) - if err := gateway.beginActivity("app.example.com"); err != nil { - t.Fatalf("first beginActivity failed: %v", err) + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) } - if err := gateway.beginActivity("app.example.com"); err != nil { - t.Fatalf("second beginActivity failed: %v", err) + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.20:3000" { + t.Fatalf("upstreams = %+v, want worker upstream", upstreams) } - gateway.endActivity("app.example.com") - time.Sleep(2 * activityFinishDebounceInterval) - expectEvents(t, client.snapshotEvents(), []string{"start"}) - - gateway.endActivity("app.example.com") deadline := time.Now().Add(time.Second) for time.Now().Before(deadline) { - got := client.snapshotEvents() - if len(got) == 2 { - expectEvents(t, got, []string{"start", "finish"}) + transitions, _, deployCalls := runtime.snapshot() + if deployCalls == 1 && len(transitions) == 1 && transitions[0].Type == "wake_started" { return } time.Sleep(5 * time.Millisecond) } - expectEvents(t, client.snapshotEvents(), []string{"start", "finish"}) + transitions, _, deployCalls := runtime.snapshot() + t.Fatalf("deployCalls=%d transitions=%+v, want one background wake", deployCalls, transitions) } -func expectEvents(t *testing.T, got []string, want []string) { - t.Helper() - if len(got) != len(want) { - t.Fatalf("events = %v, want %v", got, want) +func TestSleepingLocalDeploymentWakesAndReturnsLocalUpstream(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + runtime := &fakeRuntime{state: state} + gateway := NewGateway(runtime) + + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) } - for i := range want { - if got[i] != want[i] { - t.Fatalf("events = %v, want %v", got, want) - } + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.10:3000" { + t.Fatalf("upstreams = %+v, want local upstream", upstreams) + } + + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 1 || transitions[0].Type != "wake_started" { + t.Fatalf("transitions = %+v, want wake_started", transitions) + } +} + +func TestSleepHostRemovesLocalContainerAndReportsSleep(t *testing.T) { + state := testExpectedState("running") + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + + gateway.sleepHost("app.example.com") + + transitions, removed, _ := runtime.snapshot() + if len(removed) != 1 || removed[0] != "ctr-local" { + t.Fatalf("removed = %+v, want ctr-local", removed) + } + if len(transitions) != 1 { + t.Fatalf("transitions = %+v, want one sleep transition", transitions) + } + if transitions[0].Type != "sleep" || transitions[0].DeploymentID != "dep_local" || transitions[0].ContainerID != "ctr-local" { + t.Fatalf("transition = %+v, want sleep for dep_local/ctr-local", transitions[0]) + } +} + +func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { + state := &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_local", + ServiceID: "svc_1", + ServiceName: "api", + Name: "svc_1-dep_local", + DesiredState: localDesiredState, + Image: "nginx", + IPAddress: "10.0.0.10", + }, + }, + } + state.Serverless.Routes = []agenthttp.ServerlessRoute{ + { + ServiceID: "svc_1", + Domain: "app.example.com", + Port: 3000, + SleepAfterSeconds: 300, + WakeTimeoutSeconds: 5, + MinReadyReplicas: 1, + LocalDeploymentIDs: []string{"dep_local"}, + Upstreams: []agenthttp.ServerlessUpstream{ + { + DeploymentID: "dep_worker", + ServerID: "server_worker", + Url: "10.0.0.20:3000", + AlwaysOn: true, + }, + }, + }, } + return state } diff --git a/docs/agents/architecture.mdx b/docs/agents/architecture.mdx index f568b068..d24733a7 100644 --- a/docs/agents/architecture.mdx +++ b/docs/agents/architecture.mdx @@ -75,31 +75,31 @@ reports, the command can be retried up to the fixed attempt limit. | `force_cleanup` | Force remove containers for a service | | `cleanup_volumes` | Remove volume directories for a service | | `deploy` | Handled through expected-state reconciliation | -| `wake` | Reconcile sleeping serverless deployments back to running | -| `sleep` | Remove a stopped serverless container after confirming fresh expected state | ## Serverless Gateway Proxy agents run a local HTTP wake gateway on `127.0.0.1:18080`. The control -plane only emits Traefik routes to this gateway after the proxy reports the -`serverless_gateway` capability in its health status. +plane emits Traefik routes to this gateway only when the service has a +proxy-hosted serverless deployment. Worker-only serverless services keep normal +direct routes because worker deployments do not sleep. For serverless service traffic, Traefik forwards the public request to the local -gateway. The gateway asks the control plane to wake the target service, waits for -ready upstreams, then proxies the original request to the selected container over -the WireGuard mesh. +gateway. The gateway resolves the host from expected-state serverless metadata, +queues `wake_started` as a status transition, starts local proxy-hosted +containers from expected-state config, waits for ready upstreams, then proxies +the original request to the selected container over the WireGuard mesh. The gateway keeps a short upstream cache and collapses concurrent wake requests for the same host. Request cancellation is respected while waiting for a shared wake result. -The gateway also debounces request activity before reporting it to the control -plane. It records one `start` when a host enters an active traffic window, -heartbeats while the window remains active, and records `finish` only after a -short quiet period. Individual proxied requests do not each write start and -finish activity rows. +The gateway tracks request activity in memory. When a host stays idle for the +service's configured sleep timeout, the proxy agent removes only its local +proxy-hosted containers and queues a `sleep` status transition. The control plane +records the transition and updates dashboard/routing state; it does not run the +idle timer. -Sleep work is intentionally conservative. Before removing a local container, the -agent fetches fresh expected state and verifies that the deployment still wants -`desiredState: "stopped"`. Cached expected state is not enough to remove a -container. +If a status report fails, pending serverless transitions stay queued in memory +and are retried. A locally slept deployment is also guarded from immediate +reconcile until a fresh expected-state fetch confirms the control plane recorded +the sleep or rejects it by continuing to advertise `desiredState: "running"`. diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 30a31a15..355643c7 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -205,53 +205,57 @@ Worker nodes do not run Traefik. ### Serverless Containers -Stateless HTTP services can be configured as serverless. Serverless deployments -scale to zero by stopping their containers after an idle period, then wake on the -next public request. +Stateless HTTP services can be configured as serverless. Serverless scale-to-zero +is proxy-local: deployments placed on proxy nodes may sleep after an idle period, +then wake on the next public request handled by that proxy. Deployments placed on +worker nodes stay always on and remain routable. Serverless uses the same declarative expected-state model as normal -deployments: +deployments, but proxy agents own the local lifecycle decision: - A sleeping deployment remains `desired: true`. -- Expected state advertises the container with `desiredState: "stopped"`. -- The agent reconciles that state by stopping the local Podman container. -- A later wake changes the deployment back to `waking`, which makes expected - state advertise `desiredState: "running"` again. +- Expected state advertises proxy-hosted sleeping containers with + `desiredState: "stopped"` so normal reconciliation does not restart them. +- Expected state advertises worker-hosted deployments with + `desiredState: "running"` even when the service is serverless-enabled. +- The proxy gateway wakes local sleeping deployments from expected-state + metadata and reports `wake_started` through the next status report. +- The proxy agent reports local sleep with a `sleep` status transition after it + removes the local container. ```mermaid sequenceDiagram participant User participant Traefik participant Gateway as Agent wake gateway - participant CP as Control plane participant Agent + participant CP as Control plane participant Container User->>Traefik: Request service domain Traefik->>Gateway: Route to 127.0.0.1:18080 - Gateway->>CP: Wake service and wait - CP->>CP: Mark sleeping deployments waking - CP->>Agent: Lease wake/reconcile work - Agent->>Container: Start Podman container - Agent->>CP: Report healthy deployment - CP->>Gateway: Return ready upstreams + Gateway->>Gateway: Resolve host from expected-state metadata + Gateway->>Agent: Queue wake_started status transition + Agent->>Container: Start local Podman container + Agent->>CP: Report wake_started and container status + CP->>CP: Record waking/healthy state Gateway->>Container: Proxy original request ``` -The route to the local wake gateway is emitted only when the proxy agent reports -the `serverless_gateway` capability. Older proxy agents keep normal direct -routes and are not pointed at `127.0.0.1:18080`. +Proxy Traefik routes point to the local wake gateway only when a service has at +least one proxy-hosted serverless deployment. Worker-only serverless services use +normal direct routes because there is nothing local to wake. -The wake gateway keeps the incoming request open while the control plane waits -for enough replicas to become ready. `Min Ready` controls how many ready -deployments are preferred before releasing queued requests. If fewer replicas -become ready and no wake is still in progress, the gateway serves any available -ready upstream instead of returning a 503. +The wake gateway keeps the incoming request open while it starts local proxy +replicas, unless an always-on worker upstream is already ready. `Min Ready` +controls how many ready upstreams are preferred before releasing queued requests. +If fewer replicas become ready and no local wake is still in progress, the +gateway serves any available ready upstream instead of returning a 503. -Sleep is driven by a scheduled control-plane check. A service is eligible to -sleep when it has no fresh active requests, no rollout in progress, no busy -deployments, and its last request or newest deployment is older than the -configured idle timeout. +Sleep is driven by the proxy gateway's in-memory activity timer. The control +plane does not scan request activity and does not enqueue sleep work. It accepts +validated `sleep`, `wake_started`, and `wake_failed` transitions through the +existing agent status endpoint. Wake failures are bounded. Transient failures return a deployment to `sleeping` so a later request can retry. After repeated failures, the deployment is parked diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index bbcab900..e564bb86 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -11,10 +11,16 @@ Replica count ranges from 1 to 10 per service. ## Serverless scaling -Stateless HTTP services can be configured to sleep when idle. A sleeping -deployment keeps its desired replica record, but the container is stopped on the -agent. The next public HTTP request wakes the service and is held until an -upstream is ready or the wake timeout is reached. +Stateless HTTP services can be configured to sleep when idle on proxy nodes. A +sleeping deployment keeps its desired replica record, but the proxy agent removes +the local container. The next public HTTP request wakes local proxy-hosted +deployments and is held until an upstream is ready or the wake timeout is +reached. + +Worker-hosted deployments do not sleep. If a serverless-enabled service has only +worker replicas, it stays always on and routes directly. If a service has both +proxy and worker replicas, proxy replicas may sleep while worker replicas remain +routable. Serverless settings are configured per service: @@ -25,10 +31,10 @@ Serverless settings are configured per service: | Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | | Min Ready | `1` | Preferred number of ready deployments before queued requests resume | -`Min Ready` does not cap how many replicas are started. A cold wake starts all -desired sleeping replicas. `Min Ready` only controls when the held request can -resume. If fewer replicas become ready and no wake is still in progress, the -gateway serves the ready upstreams that exist instead of failing the request. +`Min Ready` does not cap how many local proxy replicas are started. A cold wake +starts the sleeping local proxy replicas for that host. `Min Ready` only controls +when held requests can resume. If a worker upstream is already ready, the gateway +can serve it immediately while local proxy replicas wake in the background. Serverless services require at least one configured replica. `Min Ready` must be between `1` and `10`, and cannot exceed the configured replica count. @@ -52,6 +58,5 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. - Serverless scaling is supported only for stateless HTTP services with a public domain. -- Upgrade agents before enabling serverless. Older container-hosting agents treat - wake and sleep work as unknown commands, so sleep becomes a no-op until the - agent is updated. +- Sleep and wake are proxy-local. Worker replicas are intentionally always on. +- Proxy agents report sleep and wake transitions through normal status reports. diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 74641b76..27ecdac8 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1078,40 +1078,6 @@ export async function updateServiceServerlessSettings( }) .where(eq(services.id, serviceId)); - if (!validated.enabled) { - const now = new Date(); - const wakingDeployments = await tx - .update(deployments) - .set({ - status: "waking", - containerId: null, - healthStatus: null, - unhealthyReportCount: 0, - serverlessWakeStartedAt: now, - }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "sleeping"), - ), - ) - .returning({ serverId: deployments.serverId }); - - const serverIds = new Set( - wakingDeployments.map((deployment) => deployment.serverId), - ); - for (const serverId of serverIds) { - await tx.insert(workQueue).values({ - id: randomUUID(), - serverId, - type: "wake", - payload: JSON.stringify({ - reason: "serverless_disabled", - serviceId, - }), - }); - } - } }); return { success: true }; diff --git a/web/app/api/v1/agent/serverless/activity/route.ts b/web/app/api/v1/agent/serverless/activity/route.ts deleted file mode 100644 index 6a69ff11..00000000 --- a/web/app/api/v1/agent/serverless/activity/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { type NextRequest, NextResponse } from "next/server"; -import { verifyAgentRequest } from "@/lib/agent-auth"; -import { - isProxyServer, - recordServerlessRequestFinish, - recordServerlessRequestHeartbeat, - recordServerlessRequestStart, - resolveServerlessServiceId, -} from "@/lib/serverless"; - -type ActivityRequestBody = { - serviceId?: string; - host?: string; - event?: "start" | "heartbeat" | "finish"; -}; - -export async function POST(request: NextRequest) { - const body = await request.text(); - const auth = await verifyAgentRequest(request, body); - if (!auth.success) { - return NextResponse.json({ error: auth.error }, { status: auth.status }); - } - - if (!(await isProxyServer(auth.serverId))) { - return NextResponse.json( - { error: "Serverless activity is only available to proxy servers" }, - { status: 403 }, - ); - } - - let data: ActivityRequestBody; - try { - data = body ? JSON.parse(body) : {}; - } catch { - return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); - } - - if ( - data.event !== "start" && - data.event !== "heartbeat" && - data.event !== "finish" - ) { - return NextResponse.json( - { error: "event must be 'start', 'heartbeat', or 'finish'" }, - { status: 400 }, - ); - } - - const serviceId = await resolveServerlessServiceId({ - serviceId: data.serviceId, - host: data.host, - }); - if (!serviceId) { - return NextResponse.json({ error: "Service not found" }, { status: 404 }); - } - - if (data.event === "start") { - await recordServerlessRequestStart({ - serviceId, - proxyServerId: auth.serverId, - }); - } else if (data.event === "heartbeat") { - await recordServerlessRequestHeartbeat({ - serviceId, - proxyServerId: auth.serverId, - }); - } else { - await recordServerlessRequestFinish({ - serviceId, - proxyServerId: auth.serverId, - }); - } - - return NextResponse.json({ ok: true, serviceId }); -} diff --git a/web/app/api/v1/agent/serverless/wake/route.ts b/web/app/api/v1/agent/serverless/wake/route.ts deleted file mode 100644 index 75a4206c..00000000 --- a/web/app/api/v1/agent/serverless/wake/route.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { type NextRequest, NextResponse } from "next/server"; -import { verifyAgentRequest } from "@/lib/agent-auth"; -import { - isProxyServer, - resolveServerlessTarget, - wakeAndWaitForServerlessService, - wakeServerlessService, -} from "@/lib/serverless"; - -export const maxDuration = 960; - -type WakeRequestBody = { - serviceId?: string; - host?: string; - wait?: boolean; -}; - -export async function POST(request: NextRequest) { - const body = await request.text(); - const auth = await verifyAgentRequest(request, body); - if (!auth.success) { - return NextResponse.json({ error: auth.error }, { status: auth.status }); - } - - if (!(await isProxyServer(auth.serverId))) { - return NextResponse.json( - { error: "Serverless wake is only available to proxy servers" }, - { status: 403 }, - ); - } - - let data: WakeRequestBody; - try { - data = body ? JSON.parse(body) : {}; - } catch { - return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); - } - - const target = await resolveServerlessTarget({ - serviceId: data.serviceId, - host: data.host, - }); - if (!target) { - return NextResponse.json({ error: "Service not found" }, { status: 404 }); - } - - const result = - data.wait === false - ? await wakeServerlessService({ - serviceId: target.serviceId, - port: target.port, - proxyServerId: auth.serverId, - }) - : await wakeAndWaitForServerlessService({ - serviceId: target.serviceId, - port: target.port, - proxyServerId: auth.serverId, - }); - - if (result.status === "not_found") { - return NextResponse.json( - { error: "Service not found", result }, - { status: 404 }, - ); - } - if (result.status === "not_serverless") { - return NextResponse.json( - { error: "Service is not serverless", result }, - { status: 409 }, - ); - } - if (result.status === "unsupported") { - return NextResponse.json( - { - error: "Serverless wake only supports stateless public HTTP services", - result, - }, - { status: 422 }, - ); - } - if (result.status === "no_deployments") { - return NextResponse.json( - { error: "No sleeping or ready deployments found", result }, - { status: 409 }, - ); - } - if ("timedOut" in result && result.timedOut) { - return NextResponse.json( - { error: "Timed out waiting for serverless service to wake", result }, - { status: 504 }, - ); - } - - return NextResponse.json({ ok: true, result }); -} diff --git a/web/app/api/v1/agent/status/route.ts b/web/app/api/v1/agent/status/route.ts index 2f53b84e..cf1ce4b6 100644 --- a/web/app/api/v1/agent/status/route.ts +++ b/web/app/api/v1/agent/status/route.ts @@ -1,6 +1,10 @@ import { type NextRequest, NextResponse } from "next/server"; import { verifyAgentRequest } from "@/lib/agent-auth"; -import { applyStatusReport, type StatusReport } from "@/lib/agent-status"; +import { + applyStatusReport, + type ServerlessTransition, + type StatusReport, +} from "@/lib/agent-status"; import { type ActiveWorkItem, claimNextWorkItem, @@ -13,6 +17,7 @@ type StatusRequestBody = { statusReport?: StatusReport; completedWorkItems?: WorkItemResult[]; activeWorkItems?: ActiveWorkItem[]; + serverlessTransitions?: ServerlessTransition[]; }; export async function POST(request: NextRequest) { @@ -38,7 +43,11 @@ export async function POST(request: NextRequest) { const { serverId } = auth; - await applyStatusReport(serverId, data.statusReport); + const serverlessTransitions = Array.isArray(data.serverlessTransitions) + ? data.serverlessTransitions + : []; + + await applyStatusReport(serverId, data.statusReport, serverlessTransitions); const completedWorkItems = Array.isArray(data.completedWorkItems) ? data.completedWorkItems.filter(isValidWorkItemResult) diff --git a/web/db/schema.ts b/web/db/schema.ts index e53b8278..a9958c13 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -670,33 +670,6 @@ export const workQueue = pgTable( ], ); -export const serverlessServiceActivity = pgTable( - "serverless_service_activity", - { - id: text("id").primaryKey(), - serviceId: text("service_id") - .notNull() - .references(() => services.id, { onDelete: "cascade" }), - proxyServerId: text("proxy_server_id") - .notNull() - .references(() => servers.id, { onDelete: "cascade" }), - lastRequestAt: timestamp("last_request_at", { withTimezone: true }), - activeRequests: integer("active_requests").notNull().default(0), - updatedAt: timestamp("updated_at", { withTimezone: true }) - .defaultNow() - .notNull() - .$onUpdate(() => new Date()), - }, - (table) => [ - uniqueIndex("serverless_service_activity_service_proxy_idx").on( - table.serviceId, - table.proxyServerId, - ), - index("serverless_service_activity_service_idx").on(table.serviceId), - index("serverless_service_activity_updated_at_idx").on(table.updatedAt), - ], -); - export const githubInstallations = pgTable("github_installations", { id: text("id").primaryKey(), installationId: integer("installation_id").notNull().unique(), diff --git a/web/db/types.ts b/web/db/types.ts index ba95148c..04239040 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -9,7 +9,6 @@ import type { projects, rollouts, secrets, - serverlessServiceActivity, servers, servicePorts, serviceReplicas, @@ -28,8 +27,6 @@ export type ServicePort = typeof servicePorts.$inferSelect; export type ServiceVolume = typeof serviceVolumes.$inferSelect; export type ServiceReplica = typeof serviceReplicas.$inferSelect; export type Secret = typeof secrets.$inferSelect; -export type ServerlessServiceActivity = - typeof serverlessServiceActivity.$inferSelect; export type Deployment = typeof deployments.$inferSelect; export type DeploymentPort = typeof deploymentPorts.$inferSelect; export type Rollout = typeof rollouts.$inferSelect; diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index 956fe713..fb357f4b 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -37,6 +37,11 @@ type DeploymentError = { message: string; }; +export type ServerlessTransition = + | { type: "sleep"; deploymentId: string; containerId: string } + | { type: "wake_started"; deploymentId: string } + | { type: "wake_failed"; deploymentId: string; error: string }; + function isMigrationTargetStarting(status: string | null | undefined) { return status === "deploying_target" || status === "starting"; } @@ -173,6 +178,227 @@ async function applyDeploymentErrors( } } +async function applyServerlessTransitions( + serverId: string, + transitions: ServerlessTransition[], +) { + for (const transition of transitions) { + if (!isValidServerlessTransition(transition)) { + console.log(`[serverless:status] rejected malformed transition`); + continue; + } + + const deployment = await db + .select({ + id: deployments.id, + serviceId: deployments.serviceId, + serverId: deployments.serverId, + containerId: deployments.containerId, + status: deployments.status, + desired: deployments.desired, + serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, + serverlessEnabled: services.serverlessEnabled, + stateful: services.stateful, + serverIsProxy: servers.isProxy, + serverName: servers.name, + }) + .from(deployments) + .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where(eq(deployments.id, transition.deploymentId)) + .then((rows) => rows[0]); + + const invalidReason = getInvalidServerlessTransitionReason({ + serverId, + transition, + deployment, + }); + if (invalidReason || !deployment) { + console.log( + `[serverless:status] rejected ${transition.type} for deployment ${transition.deploymentId}: ${invalidReason}`, + ); + continue; + } + + if (transition.type === "sleep") { + const updated = await db + .update(deployments) + .set({ + status: "sleeping", + containerId: null, + healthStatus: null, + serverlessWakeStartedAt: null, + failedStage: null, + }) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + eq(deployments.containerId, transition.containerId), + eq(deployments.desired, true), + inArray(deployments.status, ["healthy", "running"]), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} slept on proxy ${deployment.serverName}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + continue; + } + + if (transition.type === "wake_started") { + const updated = await db + .update(deployments) + .set({ + status: "waking", + containerId: null, + healthStatus: null, + serverlessWakeStartedAt: new Date(), + failedStage: null, + }) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + eq(deployments.desired, true), + eq(deployments.status, "sleeping"), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} wake started on proxy ${deployment.serverName}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + continue; + } + + const updated = await db + .update(deployments) + .set( + getServerlessWakeFailureUpdate({ + serverlessEnabled: deployment.serverlessEnabled, + currentFailureCount: deployment.serverlessWakeFailureCount, + failedStage: "serverless_wake", + }), + ) + .where( + and( + eq(deployments.id, transition.deploymentId), + eq(deployments.serverId, serverId), + eq(deployments.desired, true), + inArray(deployments.status, ["sleeping", "waking"]), + ), + ) + .returning({ id: deployments.id }); + + if (updated.length > 0) { + console.log( + `[serverless:status] deployment ${transition.deploymentId} wake failed on proxy ${deployment.serverName}: ${formatDeploymentError(transition.error)}`, + ); + await emitDeploymentStatusChanged( + transition.deploymentId, + deployment.serviceId, + ); + } + } +} + +function isValidServerlessTransition( + transition: unknown, +): transition is ServerlessTransition { + if (!transition || typeof transition !== "object") return false; + const candidate = transition as ServerlessTransition; + if (typeof candidate.deploymentId !== "string" || !candidate.deploymentId) { + return false; + } + if (candidate.type === "sleep") { + return typeof candidate.containerId === "string" && !!candidate.containerId; + } + if (candidate.type === "wake_started") { + return true; + } + if (candidate.type === "wake_failed") { + return typeof candidate.error === "string" && !!candidate.error.trim(); + } + return false; +} + +function getInvalidServerlessTransitionReason({ + serverId, + transition, + deployment, +}: { + serverId: string; + transition: ServerlessTransition; + deployment: + | { + serverId: string; + containerId: string | null; + status: string; + desired: boolean; + serverlessEnabled: boolean; + stateful: boolean; + serverIsProxy: boolean; + } + | undefined; +}) { + if (!deployment) return "deployment not found"; + if (deployment.serverId !== serverId) return "deployment belongs to another server"; + if (!deployment.serverIsProxy) return "server is not a proxy"; + if (!deployment.serverlessEnabled) return "service is not serverless"; + if (deployment.stateful) return "service is stateful"; + if (!deployment.desired) return "deployment is not desired"; + + if (transition.type === "sleep") { + if (!["healthy", "running"].includes(deployment.status)) { + return `deployment is not sleepable from ${deployment.status}`; + } + if (deployment.containerId !== transition.containerId) { + return "stale containerId"; + } + } + + if ( + transition.type === "wake_started" && + deployment.status !== "sleeping" + ) { + return `deployment is not sleeping (${deployment.status})`; + } + + if ( + transition.type === "wake_failed" && + !["sleeping", "waking"].includes(deployment.status) + ) { + return `deployment is not waking or sleeping (${deployment.status})`; + } + + return null; +} + +async function emitDeploymentStatusChanged(deploymentId: string, serviceId: string) { + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deploymentId, + parentType: "service", + parentId: serviceId, + }), + ); +} + function formatDeploymentError(message: string): string { return message.trim().replace(/\s+/g, " ").slice(0, 1000); } @@ -197,6 +423,7 @@ export type StatusReport = { export async function applyStatusReport( serverId: string, report: StatusReport, + serverlessTransitions: ServerlessTransition[] = [], ) { const updateData: Record = { lastHeartbeat: new Date(), @@ -276,6 +503,7 @@ export async function applyStatusReport( }; await applyDeploymentErrors(serverId, report.deploymentErrors || []); + await applyServerlessTransitions(serverId, serverlessTransitions); const reportedDeploymentIds = report.containers .map((c) => c.deploymentId) @@ -413,23 +641,6 @@ export async function applyStatusReport( continue; } - if (deployment.status === "sleeping") { - const updateFields: Record = {}; - if (deployment.containerId !== container.containerId) { - updateFields.containerId = container.containerId; - } - if (deployment.healthStatus !== null) { - updateFields.healthStatus = null; - } - if (Object.keys(updateFields).length > 0) { - await db - .update(deployments) - .set(updateFields) - .where(eq(deployments.id, deployment.id)); - } - continue; - } - const updateFields: Record = { healthStatus }; let autohealRestartPayload: Record | null = null; let autohealRecreatePayload: Record | null = null; @@ -442,7 +653,8 @@ export async function applyStatusReport( if ( deployment.status === "pending" || deployment.status === "pulling" || - deployment.status === "waking" + deployment.status === "waking" || + deployment.status === "sleeping" ) { if (container.status !== "running") { continue; diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 1ded78db..ee6d57f2 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -69,10 +69,30 @@ type UdpRoute = { externalPort: number; }; +export type ServerlessRouteUpstream = { + deploymentId: string; + serverId: string; + url: string; + local: boolean; + alwaysOn: boolean; +}; + +export type ServerlessRoute = { + serviceId: string; + domain: string; + port: number; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; + localDeploymentIds: string[]; + upstreams: ServerlessRouteUpstream[]; +}; + export type AgentExpectedState = { serverName: string; containers: ExpectedContainer[]; dns: { records: Array<{ name: string; ips: string[] }> }; + serverless: { routes: ServerlessRoute[] }; traefik: { httpRoutes: HttpRoute[]; tcpRoutes: TcpRoute[]; @@ -108,6 +128,15 @@ type RoutableDeploymentRow = { serverId: string; }; +type ServerlessDeploymentRow = { + id: string; + serviceId: string; + serverId: string; + ipAddress: string | null; + status: Deployment["status"]; + serverIsProxy: boolean; +}; + export async function getServer(serverId: string) { return db .select() @@ -120,15 +149,21 @@ export async function buildAgentExpectedState( server: Server, ): Promise { const allServices = await getActiveServices(); - const containers = await buildExpectedContainers(server.id); + const containers = await buildExpectedContainers(server.id, server.isProxy); const dnsRecords = await buildDnsRecords(allServices); const traefikConfig = await buildTraefikConfig(server, allServices); + const serverless = await buildServerlessExpectedState( + server, + allServices, + containers, + ); const wireguardPeers = await getWireGuardPeers(server.id, server.privateIp); return { serverName: server.name, containers, dns: { records: dnsRecords }, + serverless, traefik: traefikConfig, wireguard: { peers: wireguardPeers }, }; @@ -140,6 +175,7 @@ async function getActiveServices() { async function buildExpectedContainers( serverId: string, + serverIsProxy: boolean, ): Promise { const serverDeployments = await db .select() @@ -189,6 +225,7 @@ async function buildExpectedContainers( deploymentPorts: depPorts, secrets: serviceSecrets, volumes, + serverIsProxy, }); } @@ -212,12 +249,14 @@ export function buildExpectedContainersFromRows({ deploymentPorts: deploymentPortRows, secrets: secretRows, volumes: volumeRows, + serverIsProxy = true, }: { deployments: Deployment[]; services: Service[]; deploymentPorts: DeploymentPortRow[]; secrets: SecretRow[]; volumes: VolumeRow[]; + serverIsProxy?: boolean; }): ExpectedContainer[] { const servicesById = new Map( serviceRows.map((service) => [service.id, service]), @@ -242,7 +281,12 @@ export function buildExpectedContainersFromRows({ serviceId: dep.serviceId, serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, - desiredState: dep.status === "sleeping" ? "stopped" : "running", + desiredState: + serverIsProxy && + service.serverlessEnabled && + dep.status === "sleeping" + ? "stopped" + : "running", image: normalizeImage(service.image), ipAddress: dep.ipAddress, ports: (portsByDeploymentId.get(dep.id) ?? []) @@ -266,6 +310,137 @@ export function buildExpectedContainersFromRows({ resourceMemoryLimitMb: service.resourceMemoryLimitMb, }, ]; + }); +} + +async function buildServerlessExpectedState( + server: Server, + allServices: Service[], + containers: ExpectedContainer[], +): Promise<{ routes: ServerlessRoute[] }> { + if (!server.isProxy || !hasAgentCapability(server, SERVERLESS_GATEWAY_CAPABILITY)) { + return { routes: [] }; + } + + const serverlessServices = allServices.filter( + (service) => service.serverlessEnabled && !service.stateful, + ); + if (serverlessServices.length === 0) return { routes: [] }; + + const serviceIds = serverlessServices.map((service) => service.id); + const [ports, deploymentRows] = await Promise.all([ + db + .select() + .from(servicePorts) + .where(inArray(servicePorts.serviceId, serviceIds)), + db + .select({ + id: deployments.id, + serviceId: deployments.serviceId, + serverId: deployments.serverId, + ipAddress: deployments.ipAddress, + status: deployments.status, + serverIsProxy: servers.isProxy, + }) + .from(deployments) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where( + and( + inArray(deployments.serviceId, serviceIds), + eq(deployments.desired, true), + inArray(deployments.status, expectedDeploymentStatuses), + ), + ), + ]); + + return { + routes: buildServerlessRoutesFromRows({ + serverId: server.id, + services: serverlessServices, + ports, + deployments: deploymentRows, + containers, + }), + }; +} + +export function buildServerlessRoutesFromRows({ + serverId, + services: serviceRows, + ports, + deployments: deploymentRows, + containers, +}: { + serverId: string; + services: Service[]; + ports: ServicePort[]; + deployments: ServerlessDeploymentRow[]; + containers: ExpectedContainer[]; +}): ServerlessRoute[] { + const servicesById = new Map( + serviceRows.map((service) => [service.id, service]), + ); + const deploymentsByServiceId = groupBy( + deploymentRows, + (deployment) => deployment.serviceId, + ); + const expectedDeploymentIds = new Set( + containers.map((container) => container.deploymentId), + ); + + return ports + .slice() + .sort(compareServicePorts) + .flatMap((port) => { + if (!port.isPublic || port.protocol !== "http" || !port.domain) { + return []; + } + + const service = servicesById.get(port.serviceId); + if (!service) return []; + + const serviceDeployments = deploymentsByServiceId.get(port.serviceId) ?? []; + if (!serviceDeployments.some((deployment) => deployment.serverIsProxy)) { + return []; + } + + const localDeploymentIds = serviceDeployments + .filter( + (deployment) => + deployment.serverId === serverId && + deployment.serverIsProxy && + expectedDeploymentIds.has(deployment.id), + ) + .map((deployment) => deployment.id) + .sort(); + + const upstreams = serviceDeployments + .filter( + (deployment) => + deployment.ipAddress && + routableDeploymentStatuses.includes(deployment.status), + ) + .map((deployment) => ({ + deploymentId: deployment.id, + serverId: deployment.serverId, + url: `${deployment.ipAddress}:${port.port}`, + local: deployment.serverId === serverId, + alwaysOn: !deployment.serverIsProxy, + })) + .sort(compareServerlessUpstreams); + + return [ + { + serviceId: service.id, + domain: port.domain, + port: port.port, + sleepAfterSeconds: service.serverlessSleepAfterSeconds, + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds, + minReadyReplicas: Math.max(1, service.serverlessMinReadyReplicas ?? 1), + localDeploymentIds, + upstreams, + }, + ]; }); } @@ -316,7 +491,8 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { server, SERVERLESS_GATEWAY_CAPABILITY, ); - const [ports, routableDeployments, wakeRoutedDeployments] = await Promise.all( + const [ports, routableDeployments, proxyHostedServerlessDeployments] = + await Promise.all( [ serviceIds.length > 0 ? db @@ -344,18 +520,20 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { ? db .select({ serviceId: deployments.serviceId }) .from(deployments) + .innerJoin(servers, eq(deployments.serverId, servers.id)) .where( and( inArray(deployments.serviceId, serviceIds), eq(deployments.desired, true), - inArray(deployments.status, ["sleeping", "waking"]), + eq(servers.isProxy, true), + inArray(deployments.status, expectedDeploymentStatuses), ), ) : Promise.resolve([]), ], ); - const wakeRoutedServiceIds = new Set( - wakeRoutedDeployments.map((deployment) => deployment.serviceId), + const proxyHostedServerlessServiceIds = new Set( + proxyHostedServerlessDeployments.map((deployment) => deployment.serviceId), ); const serverlessServiceIds = supportsServerlessGateway ? new Set( @@ -363,8 +541,8 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .filter( (service) => !service.stateful && - (service.serverlessEnabled || - wakeRoutedServiceIds.has(service.id)), + service.serverlessEnabled && + proxyHostedServerlessServiceIds.has(service.id), ) .map((service) => service.id), ) @@ -521,6 +699,15 @@ function upstreamUrls(deployments: RoutableDeploymentRow[], port: number) { .sort(); } +function compareServerlessUpstreams( + a: ServerlessRouteUpstream, + b: ServerlessRouteUpstream, +) { + if (a.local !== b.local) return a.local ? -1 : 1; + if (a.alwaysOn !== b.alwaysOn) return a.alwaysOn ? -1 : 1; + return a.url.localeCompare(b.url); +} + function normalizeImage(image: string) { if (!image.includes("/")) { return `docker.io/library/${image}`; diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index df26825c..92c3a068 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -8,7 +8,6 @@ import { checkAndPersistControlPlaneUpdate } from "@/lib/control-plane-updates"; import { checkAndRecoverStaleServers, checkAndRunScheduledDeployments, - checkAndSleepIdleServerlessServices, cleanupStaleItems, failTimedOutAgentUpgrades, } from "@/lib/scheduler"; @@ -125,19 +124,6 @@ export const staleItemsCleanup = inngest.createFunction( }, ); -export const serverlessSleepCheck = inngest.createFunction( - { - id: "cron-serverless-sleep-check", - triggers: [cron("* * * * *")], - singleton: { mode: "skip" }, - }, - async ({ step }) => { - await step.run("sleep-idle-serverless-services", async () => { - await checkAndSleepIdleServerlessServices(); - }); - }, -); - export const agentUpgradeTimeoutCheck = inngest.createFunction( { id: "cron-agent-upgrade-timeout-check", diff --git a/web/lib/inngest/functions/index.ts b/web/lib/inngest/functions/index.ts index 63bb58df..ef862f53 100644 --- a/web/lib/inngest/functions/index.ts +++ b/web/lib/inngest/functions/index.ts @@ -9,7 +9,6 @@ export { oldBackupsCleanup, scheduledBackupsCheck, scheduledDeploymentsCheck, - serverlessSleepCheck, staleItemsCleanup, staleServerCheck, } from "./crons"; diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index be8f4313..c7177135 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -18,7 +18,6 @@ import { WORK_QUEUE_LEASE_DURATION_MS, WORK_QUEUE_MAX_ATTEMPTS, } from "@/lib/work-queue"; -import { sleepIdleServerlessServices } from "./serverless"; const STALE_THRESHOLD_MS = 75_000; // 75 seconds @@ -321,17 +320,3 @@ export async function cleanupStaleItems(): Promise { ); } } - -export async function checkAndSleepIdleServerlessServices(): Promise { - const result = await sleepIdleServerlessServices(); - if (result.deploymentsSlept > 0) { - console.log( - `[scheduler] slept ${result.deploymentsSlept} serverless deployment(s) across ${result.servicesSlept} service(s)`, - ); - } - if (result.wakingDeploymentsReset > 0) { - console.log( - `[scheduler] reset ${result.wakingDeploymentsReset} timed-out serverless wake deployment(s)`, - ); - } -} diff --git a/web/lib/serverless.ts b/web/lib/serverless.ts deleted file mode 100644 index 1dce5f09..00000000 --- a/web/lib/serverless.ts +++ /dev/null @@ -1,1149 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { - and, - eq, - gt, - inArray, - isNotNull, - isNull, - lte, - or, - sql, -} from "drizzle-orm"; -import { db } from "@/db"; -import { - deployments, - rollouts, - serverlessServiceActivity, - servers, - servicePorts, - serviceReplicas, - services, - workQueue, -} from "@/db/schema"; -import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; - -const READY_DEPLOYMENT_STATUSES = ["healthy", "running"] as const; -const WAKE_IN_PROGRESS_STATUSES = [ - "pending", - "pulling", - "starting", - "waking", -] as const; -const SLEEPABLE_DEPLOYMENT_STATUSES = ["healthy", "running"] as const; -const SERVERLESS_BUSY_DEPLOYMENT_STATUSES = [ - "pending", - "pulling", - "starting", - "waking", - "draining", - "stopping", -] as const; - -const WAKE_POLL_INTERVAL_MS = 500; -const MIN_ACTIVE_REQUEST_STALE_SECONDS = 120; - -export type ServerlessServiceRef = { - serviceId?: string; - host?: string; -}; - -export type ServerlessDeployment = { - id: string; - serverId: string; - ipAddress: string | null; - status: string; -}; - -export type ServerlessUpstream = { - url: string; -}; - -export type ServerlessWakeResult = { - status: - | "ready" - | "waking" - | "not_found" - | "not_serverless" - | "unsupported" - | "no_deployments"; - serviceId: string; - readyDeployments: ServerlessDeployment[]; - upstreams: ServerlessUpstream[]; - wakingDeployments: number; - queuedWakeServers: number; - minReadyReplicas: number; -}; - -export type ServerlessWaitResult = ServerlessWakeResult & { - timedOut: boolean; -}; - -export type ServerlessSleepResult = { - servicesChecked: number; - servicesSlept: number; - deploymentsSlept: number; - wakingDeploymentsReset: number; -}; - -type ServerlessService = typeof services.$inferSelect; -type DeploymentRow = typeof deployments.$inferSelect; -type DbExecutor = Pick< - typeof db, - "delete" | "execute" | "insert" | "select" | "update" ->; - -type ServerlessTarget = { - serviceId: string; - port: number; -}; - -export async function resolveServerlessServiceId({ - serviceId, - host, -}: ServerlessServiceRef): Promise { - return ( - (await resolveServerlessTarget({ serviceId, host }))?.serviceId ?? null - ); -} - -export async function resolveServerlessTarget({ - serviceId, - host, -}: ServerlessServiceRef): Promise { - if (serviceId) { - const [port] = await db - .select({ port: servicePorts.port }) - .from(servicePorts) - .where( - and( - eq(servicePorts.serviceId, serviceId), - eq(servicePorts.protocol, "http"), - eq(servicePorts.isPublic, true), - isNotNull(servicePorts.domain), - ), - ) - .limit(1); - return port ? { serviceId, port: port.port } : null; - } - - const normalizedHost = normalizeHost(host); - if (!normalizedHost) return null; - - const [port] = await db - .select({ serviceId: servicePorts.serviceId, port: servicePorts.port }) - .from(servicePorts) - .innerJoin(services, eq(services.id, servicePorts.serviceId)) - .where( - and( - eq(servicePorts.domain, normalizedHost), - eq(servicePorts.protocol, "http"), - eq(servicePorts.isPublic, true), - isNull(services.deletedAt), - ), - ) - .limit(1); - - return port ? { serviceId: port.serviceId, port: port.port } : null; -} - -export async function recordServerlessRequestStart({ - serviceId, - proxyServerId, - now = new Date(), -}: { - serviceId: string; - proxyServerId: string; - now?: Date; -}) { - await db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - const service = await getServerlessService(tx, serviceId); - if (service) { - await resetStaleActiveRequests(tx, service, now); - } - await touchServerlessRequestStart(tx, { - serviceId, - proxyServerId, - now, - }); - }); -} - -export async function recordServerlessRequestFinish({ - serviceId, - proxyServerId, - now = new Date(), -}: { - serviceId: string; - proxyServerId: string; - now?: Date; -}) { - await db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - await touchServerlessRequestFinish(tx, { - serviceId, - proxyServerId, - now, - }); - }); -} - -export async function recordServerlessRequestHeartbeat({ - serviceId, - proxyServerId, - now = new Date(), -}: { - serviceId: string; - proxyServerId: string; - now?: Date; -}) { - await db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - await touchServerlessRequestHeartbeat(tx, { - serviceId, - proxyServerId, - now, - }); - }); -} - -async function touchServerlessRequestStart( - executor: DbExecutor, - { - serviceId, - proxyServerId, - now, - }: { - serviceId: string; - proxyServerId: string; - now: Date; - }, -) { - await executor - .insert(serverlessServiceActivity) - .values({ - id: randomUUID(), - serviceId, - proxyServerId, - lastRequestAt: now, - activeRequests: 1, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - serverlessServiceActivity.serviceId, - serverlessServiceActivity.proxyServerId, - ], - set: { - lastRequestAt: now, - activeRequests: sql`${serverlessServiceActivity.activeRequests} + 1`, - updatedAt: now, - }, - }); -} - -async function touchServerlessRequestFinish( - executor: DbExecutor, - { - serviceId, - proxyServerId, - now, - }: { - serviceId: string; - proxyServerId: string; - now: Date; - }, -) { - await executor - .insert(serverlessServiceActivity) - .values({ - id: randomUUID(), - serviceId, - proxyServerId, - lastRequestAt: now, - activeRequests: 0, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - serverlessServiceActivity.serviceId, - serverlessServiceActivity.proxyServerId, - ], - set: { - activeRequests: sql`GREATEST(${serverlessServiceActivity.activeRequests} - 1, 0)`, - lastRequestAt: now, - updatedAt: now, - }, - }); -} - -async function touchServerlessRequestHeartbeat( - executor: DbExecutor, - { - serviceId, - proxyServerId, - now, - }: { - serviceId: string; - proxyServerId: string; - now: Date; - }, -) { - await executor - .insert(serverlessServiceActivity) - .values({ - id: randomUUID(), - serviceId, - proxyServerId, - lastRequestAt: now, - activeRequests: 0, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - serverlessServiceActivity.serviceId, - serverlessServiceActivity.proxyServerId, - ], - set: { - lastRequestAt: now, - updatedAt: now, - }, - }); -} - -export async function wakeServerlessService({ - serviceId, - port, - proxyServerId, - now = new Date(), -}: { - serviceId: string; - port?: number; - proxyServerId: string; - now?: Date; -}): Promise { - return db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - - const [service] = await tx - .select() - .from(services) - .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) - .limit(1); - - if (!service) return emptyWakeResult("not_found", serviceId); - const inServerlessTransition = await hasDesiredWakeTransitionState( - tx, - serviceId, - ); - const unsupported = await getServerlessUnsupportedReason(tx, service, { - allowDisabledTransition: inServerlessTransition, - }); - if (unsupported) return emptyWakeResult(unsupported, serviceId); - - await touchServerlessActivity(tx, { serviceId, proxyServerId, now }); - - const minReadyReplicas = await getEffectiveMinReadyReplicas(tx, service); - const targetPort = port ?? (await getDefaultHttpPort(tx, serviceId)); - if (!targetPort) return emptyWakeResult("unsupported", serviceId); - - const readyDeployments = await getReadyDeployments(tx, serviceId); - if (readyDeployments.length >= minReadyReplicas) { - return { - status: "ready", - serviceId, - readyDeployments, - upstreams: buildUpstreams(readyDeployments, targetPort), - wakingDeployments: 0, - queuedWakeServers: 0, - minReadyReplicas, - }; - } - - const wakeableDeployments = await tx - .select() - .from(deployments) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), - eq(deployments.status, "sleeping"), - ), - ); - const currentWakingDeployments = await countWakeInProgressDeployments( - tx, - serviceId, - ); - - if (wakeableDeployments.length === 0) { - if (readyDeployments.length > 0 && currentWakingDeployments === 0) { - return { - status: "ready", - serviceId, - readyDeployments, - upstreams: buildUpstreams(readyDeployments, targetPort), - wakingDeployments: 0, - queuedWakeServers: 0, - minReadyReplicas, - }; - } - return { - status: currentWakingDeployments > 0 ? "waking" : "no_deployments", - serviceId, - readyDeployments, - upstreams: [], - wakingDeployments: currentWakingDeployments, - queuedWakeServers: 0, - minReadyReplicas, - }; - } - - if (wakeableDeployments.length > 0) { - const wakeableIds = wakeableDeployments.map( - (deployment) => deployment.id, - ); - // Wake all desired replicas; minReadyReplicas controls when requests can resume. - await tx - .update(deployments) - .set({ - status: "waking", - containerId: null, - healthStatus: null, - unhealthyReportCount: 0, - serverlessWakeStartedAt: now, - }) - .where(inArray(deployments.id, wakeableIds)); - await deletePendingSleepWorkItems(tx, wakeableDeployments); - } - - const queuedWakeServers = await enqueueWakeReconcilers( - tx, - serviceId, - wakeableDeployments, - ); - const wakingDeployments = await countWakeInProgressDeployments( - tx, - serviceId, - ); - - return { - status: "waking", - serviceId, - readyDeployments, - upstreams: [], - wakingDeployments, - queuedWakeServers, - minReadyReplicas, - }; - }); -} - -export async function wakeAndWaitForServerlessService({ - serviceId, - port, - proxyServerId, - timeoutSeconds, - now = new Date(), -}: { - serviceId: string; - port?: number; - proxyServerId: string; - timeoutSeconds?: number; - now?: Date; -}): Promise { - const wakeResult = await wakeServerlessService({ - serviceId, - port, - proxyServerId, - now, - }); - - if (wakeResult.status !== "waking") { - return { ...wakeResult, timedOut: false }; - } - - const configuredTimeoutSeconds = - timeoutSeconds ?? (await getWakeTimeoutSeconds(serviceId)) ?? 300; - const targetPort = port ?? (await getDefaultHttpPort(db, serviceId)); - const timeoutMs = Math.max(1, configuredTimeoutSeconds) * 1000; - const deadline = Date.now() + timeoutMs; - - while (Date.now() < deadline) { - await sleep(WAKE_POLL_INTERVAL_MS); - const readyDeployments = await getReadyDeployments(db, serviceId); - if (targetPort && readyDeployments.length >= wakeResult.minReadyReplicas) { - return { - ...wakeResult, - status: "ready", - readyDeployments, - upstreams: buildUpstreams(readyDeployments, targetPort), - timedOut: false, - }; - } - const wakingDeployments = await countWakeInProgressDeployments( - db, - serviceId, - ); - if (targetPort && readyDeployments.length > 0 && wakingDeployments === 0) { - return { - ...wakeResult, - status: "ready", - readyDeployments, - upstreams: buildUpstreams(readyDeployments, targetPort), - wakingDeployments, - timedOut: false, - }; - } - } - - const readyDeployments = await getReadyDeployments(db, serviceId); - if (targetPort && readyDeployments.length > 0) { - return { - ...wakeResult, - status: "ready", - readyDeployments, - upstreams: buildUpstreams(readyDeployments, targetPort), - wakingDeployments: await countWakeInProgressDeployments(db, serviceId), - timedOut: false, - }; - } - - const service = await getServerlessService(db, serviceId); - if (service) { - await resetTimedOutWakingDeployments({ service, now: new Date() }); - } - - return { ...wakeResult, timedOut: true }; -} - -export async function sleepIdleServerlessServices({ - now = new Date(), -}: { - now?: Date; -} = {}): Promise { - const candidates = await getServerlessSleepCandidates(); - - let servicesSlept = 0; - let deploymentsSlept = 0; - let wakingDeploymentsReset = 0; - - for (const service of candidates) { - wakingDeploymentsReset += await resetTimedOutWakingDeployments({ - service, - now, - }); - if (!service.serverlessEnabled) continue; - if (!(await hasPublicHttpEndpoint(db, service.id))) continue; - if (!(await isServiceIdle(db, service, now))) continue; - - const result = await sleepServerlessService({ - serviceId: service.id, - now, - }); - if (result.deploymentsSlept > 0) { - servicesSlept += 1; - deploymentsSlept += result.deploymentsSlept; - } - } - - return { - servicesChecked: candidates.length, - servicesSlept, - deploymentsSlept, - wakingDeploymentsReset, - }; -} - -export async function sleepServerlessService({ - serviceId, - now = new Date(), -}: { - serviceId: string; - now?: Date; -}): Promise<{ deploymentsSlept: number }> { - return db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - - const [service] = await tx - .select() - .from(services) - .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) - .limit(1); - - if (!service) return { deploymentsSlept: 0 }; - const unsupported = await getServerlessUnsupportedReason(tx, service); - if (unsupported) return { deploymentsSlept: 0 }; - if (!(await isServiceIdle(tx, service, now))) { - return { deploymentsSlept: 0 }; - } - - const sleepableDeployments = await tx - .select() - .from(deployments) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), - inArray(deployments.status, SLEEPABLE_DEPLOYMENT_STATUSES), - isNotNull(deployments.containerId), - ), - ); - - if (sleepableDeployments.length === 0) return { deploymentsSlept: 0 }; - - await tx - .update(deployments) - .set({ - status: "sleeping", - containerId: null, - healthStatus: null, - serverlessWakeStartedAt: null, - }) - .where( - inArray( - deployments.id, - sleepableDeployments.map((deployment) => deployment.id), - ), - ); - - for (const deployment of sleepableDeployments) { - await tx.insert(workQueue).values({ - id: randomUUID(), - serverId: deployment.serverId, - type: "sleep", - payload: JSON.stringify({ - reason: "serverless_idle_timeout", - deploymentId: deployment.id, - serviceId, - containerId: deployment.containerId, - }), - }); - } - - return { deploymentsSlept: sleepableDeployments.length }; - }); -} - -async function getServerlessUnsupportedReason( - executor: DbExecutor, - service: ServerlessService, - { - allowDisabledTransition = false, - }: { - allowDisabledTransition?: boolean; - } = {}, -): Promise<"not_serverless" | "unsupported" | null> { - if (!service.serverlessEnabled && !allowDisabledTransition) - return "not_serverless"; - if (service.stateful) return "unsupported"; - if (!(await hasPublicHttpEndpoint(executor, service.id))) - return "unsupported"; - return null; -} - -async function hasPublicHttpEndpoint(executor: DbExecutor, serviceId: string) { - const [port] = await executor - .select({ id: servicePorts.id }) - .from(servicePorts) - .where( - and( - eq(servicePorts.serviceId, serviceId), - eq(servicePorts.protocol, "http"), - eq(servicePorts.isPublic, true), - isNotNull(servicePorts.domain), - ), - ) - .limit(1); - - return Boolean(port); -} - -async function getServerlessSleepCandidates(): Promise { - const [enabledServices, wakingTransitionRows] = await Promise.all([ - db - .select() - .from(services) - .where( - and( - eq(services.serverlessEnabled, true), - eq(services.stateful, false), - isNull(services.deletedAt), - ), - ), - db - .select({ serviceId: deployments.serviceId }) - .from(deployments) - .where( - and(eq(deployments.desired, true), eq(deployments.status, "waking")), - ), - ]); - const wakingServiceIds = unique( - wakingTransitionRows.map((row) => row.serviceId), - ); - const wakingTransitionServices = - wakingServiceIds.length > 0 - ? await db - .select() - .from(services) - .where( - and( - inArray(services.id, wakingServiceIds), - eq(services.stateful, false), - isNull(services.deletedAt), - ), - ) - : []; - - const servicesById = new Map(); - for (const service of enabledServices) { - servicesById.set(service.id, service); - } - for (const service of wakingTransitionServices) { - servicesById.set(service.id, service); - } - return [...servicesById.values()]; -} - -async function hasDesiredWakeTransitionState( - executor: DbExecutor, - serviceId: string, -) { - const [deployment] = await executor - .select({ id: deployments.id }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), - inArray(deployments.status, [ - "sleeping", - "waking", - ...READY_DEPLOYMENT_STATUSES, - ]), - ), - ) - .limit(1); - - return Boolean(deployment); -} - -export async function isProxyServer(serverId: string) { - const [server] = await db - .select({ isProxy: servers.isProxy }) - .from(servers) - .where(eq(servers.id, serverId)) - .limit(1); - return server?.isProxy === true; -} - -async function isServiceIdle( - executor: DbExecutor, - service: ServerlessService, - now: Date, -) { - const [activeRollout] = await executor - .select({ id: rollouts.id }) - .from(rollouts) - .where( - and( - eq(rollouts.serviceId, service.id), - inArray(rollouts.status, ["queued", "in_progress"]), - ), - ) - .limit(1); - if (activeRollout) return false; - - const [busyDeployment] = await executor - .select({ id: deployments.id }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, service.id), - inArray(deployments.status, SERVERLESS_BUSY_DEPLOYMENT_STATUSES), - ), - ) - .limit(1); - if (busyDeployment) return false; - - await resetStaleActiveRequests(executor, service, now); - - const activityRows = await executor - .select({ - lastRequestAt: serverlessServiceActivity.lastRequestAt, - activeRequests: serverlessServiceActivity.activeRequests, - updatedAt: serverlessServiceActivity.updatedAt, - }) - .from(serverlessServiceActivity) - .where(eq(serverlessServiceActivity.serviceId, service.id)); - - const activeRequestStaleCutoff = getActiveRequestStaleCutoff(service, now); - const activeRequests = activityRows.reduce((total, row) => { - if (row.activeRequests <= 0) return total; - if (row.updatedAt <= activeRequestStaleCutoff) return total; - return total + row.activeRequests; - }, 0); - if (activeRequests > 0) return false; - - const lastRequestAt = maxDate( - activityRows - .map((row) => row.lastRequestAt) - .filter((value): value is Date => value !== null), - ); - const sleepableDeployments = await executor - .select({ createdAt: deployments.createdAt }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, service.id), - eq(deployments.desired, true), - inArray(deployments.status, SLEEPABLE_DEPLOYMENT_STATUSES), - ), - ); - - if (sleepableDeployments.length === 0) return false; - - const newestDeploymentCreatedAt = maxDate( - sleepableDeployments.map((deployment) => deployment.createdAt), - ); - const lastActivityAt = maxDate( - [lastRequestAt, newestDeploymentCreatedAt].filter( - (value): value is Date => value !== null, - ), - ); - if (!lastActivityAt) return false; - - const idleMs = now.getTime() - lastActivityAt.getTime(); - return idleMs >= service.serverlessSleepAfterSeconds * 1000; -} - -async function resetTimedOutWakingDeployments({ - service, - now, -}: { - service: ServerlessService; - now: Date; -}) { - return db.transaction(async (tx) => { - await tx.execute( - sql`SELECT pg_advisory_xact_lock(hashtext(${service.id}))`, - ); - - const timeoutCutoff = new Date( - now.getTime() - getWakeTimeoutSecondsForService(service) * 1000, - ); - const timedOutDeployments = await tx - .select({ - id: deployments.id, - serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, - }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, service.id), - eq(deployments.status, "waking"), - eq(deployments.desired, true), - or( - lte(deployments.serverlessWakeStartedAt, timeoutCutoff), - and( - isNull(deployments.serverlessWakeStartedAt), - lte(deployments.createdAt, timeoutCutoff), - ), - ), - ), - ); - - let resetCount = 0; - for (const deployment of timedOutDeployments) { - const updated = await tx - .update(deployments) - .set( - getServerlessWakeFailureUpdate({ - serverlessEnabled: service.serverlessEnabled, - currentFailureCount: deployment.serverlessWakeFailureCount, - failedStage: "serverless_wake_timeout", - }), - ) - .where( - and( - eq(deployments.id, deployment.id), - eq(deployments.status, "waking"), - eq(deployments.desired, true), - ), - ) - .returning({ id: deployments.id }); - resetCount += updated.length; - } - - return resetCount; - }); -} - -async function touchServerlessActivity( - executor: DbExecutor, - { - serviceId, - proxyServerId, - now, - }: { - serviceId: string; - proxyServerId: string; - now: Date; - }, -) { - await executor - .insert(serverlessServiceActivity) - .values({ - id: randomUUID(), - serviceId, - proxyServerId, - lastRequestAt: now, - activeRequests: 0, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - serverlessServiceActivity.serviceId, - serverlessServiceActivity.proxyServerId, - ], - set: { - lastRequestAt: now, - updatedAt: now, - }, - }); -} - -async function getServerlessService( - executor: DbExecutor, - serviceId: string, -): Promise { - const [service] = await executor - .select() - .from(services) - .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) - .limit(1); - return service ?? null; -} - -async function getWakeTimeoutSeconds(serviceId: string) { - const [service] = await db - .select({ wakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds }) - .from(services) - .where(eq(services.id, serviceId)) - .limit(1); - return service?.wakeTimeoutSeconds ?? null; -} - -async function getDefaultHttpPort(executor: DbExecutor, serviceId: string) { - const [port] = await executor - .select({ port: servicePorts.port }) - .from(servicePorts) - .where( - and( - eq(servicePorts.serviceId, serviceId), - eq(servicePorts.protocol, "http"), - eq(servicePorts.isPublic, true), - isNotNull(servicePorts.domain), - ), - ) - .limit(1); - return port?.port ?? null; -} - -async function getReadyDeployments( - executor: DbExecutor, - serviceId: string, -): Promise { - return executor - .select({ - id: deployments.id, - serverId: deployments.serverId, - ipAddress: deployments.ipAddress, - status: deployments.status, - }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), - inArray(deployments.status, READY_DEPLOYMENT_STATUSES), - isNotNull(deployments.ipAddress), - ), - ); -} - -async function countWakeInProgressDeployments( - executor: DbExecutor, - serviceId: string, -) { - const rows = await executor - .select({ id: deployments.id }) - .from(deployments) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), - inArray(deployments.status, WAKE_IN_PROGRESS_STATUSES), - ), - ); - return rows.length; -} - -async function resetStaleActiveRequests( - executor: DbExecutor, - service: ServerlessService, - now: Date, -) { - const staleCutoff = getActiveRequestStaleCutoff(service, now); - await executor - .update(serverlessServiceActivity) - .set({ activeRequests: 0, updatedAt: now }) - .where( - and( - eq(serverlessServiceActivity.serviceId, service.id), - gt(serverlessServiceActivity.activeRequests, 0), - lte(serverlessServiceActivity.updatedAt, staleCutoff), - ), - ); -} - -async function deletePendingSleepWorkItems( - executor: DbExecutor, - wakeableDeployments: DeploymentRow[], -) { - if (wakeableDeployments.length === 0) return; - const serviceId = wakeableDeployments[0]?.serviceId; - const serverIds = unique( - wakeableDeployments.map((deployment) => deployment.serverId), - ); - if (!serviceId || serverIds.length === 0) return; - - await executor - .delete(workQueue) - .where( - and( - eq(workQueue.type, "sleep"), - eq(workQueue.status, "pending"), - inArray(workQueue.serverId, serverIds), - sql`${workQueue.payload}::jsonb ->> 'serviceId' = ${serviceId}`, - ), - ); -} - -function buildUpstreams( - deployments: ServerlessDeployment[], - port: number, -): ServerlessUpstream[] { - return deployments - .filter((deployment) => deployment.ipAddress) - .map((deployment) => ({ url: `${deployment.ipAddress}:${port}` })) - .sort((a, b) => a.url.localeCompare(b.url)); -} - -async function enqueueWakeReconcilers( - executor: DbExecutor, - serviceId: string, - wakeableDeployments: DeploymentRow[], -) { - const serverIds = new Set( - wakeableDeployments.map((deployment) => deployment.serverId), - ); - - for (const serverId of serverIds) { - await executor.insert(workQueue).values({ - id: randomUUID(), - serverId, - type: "wake", - payload: JSON.stringify({ - reason: "serverless_wake", - serviceId, - }), - }); - } - - return serverIds.size; -} - -function emptyWakeResult( - status: ServerlessWakeResult["status"], - serviceId: string, -): ServerlessWakeResult { - return { - status, - serviceId, - readyDeployments: [], - upstreams: [], - wakingDeployments: 0, - queuedWakeServers: 0, - minReadyReplicas: 1, - }; -} - -function getMinReadyReplicas(service: ServerlessService) { - return Math.max(1, service.serverlessMinReadyReplicas ?? 1); -} - -async function getEffectiveMinReadyReplicas( - executor: DbExecutor, - service: ServerlessService, -) { - const [configuredReplicas, desiredDeployments] = await Promise.all([ - getConfiguredReplicaCount(executor, service.id), - countDesiredDeployments(executor, service.id), - ]); - const capacity = Math.max(1, desiredDeployments || configuredReplicas); - return Math.min(getMinReadyReplicas(service), capacity); -} - -async function getConfiguredReplicaCount( - executor: DbExecutor, - serviceId: string, -) { - const rows = await executor - .select({ count: serviceReplicas.count }) - .from(serviceReplicas) - .where(eq(serviceReplicas.serviceId, serviceId)); - return rows.reduce((total, row) => total + row.count, 0); -} - -async function countDesiredDeployments( - executor: DbExecutor, - serviceId: string, -) { - const rows = await executor - .select({ id: deployments.id }) - .from(deployments) - .where( - and(eq(deployments.serviceId, serviceId), eq(deployments.desired, true)), - ); - return rows.length; -} - -function getWakeTimeoutSecondsForService(service: ServerlessService) { - return Math.max(1, service.serverlessWakeTimeoutSeconds ?? 300); -} - -function getActiveRequestStaleCutoff(service: ServerlessService, now: Date) { - const staleSeconds = Math.max( - MIN_ACTIVE_REQUEST_STALE_SECONDS, - getWakeTimeoutSecondsForService(service), - ); - return new Date(now.getTime() - staleSeconds * 1000); -} - -function normalizeHost(host: string | undefined) { - return host?.trim().toLowerCase().replace(/:\d+$/, "") || null; -} - -function maxDate(values: Date[]) { - if (values.length === 0) return null; - return values.reduce((max, value) => (value > max ? value : max), values[0]); -} - -function unique(items: T[]) { - return Array.from(new Set(items)); -} - -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index f09e787b..b5e883bf 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -8,6 +8,7 @@ vi.mock("@/lib/wireguard", () => ({ getWireGuardPeers: vi.fn() })); import { buildExpectedContainersFromRows, + buildServerlessRoutesFromRows, buildTraefikRoutes, } from "@/lib/agent/expected-state"; @@ -41,6 +42,7 @@ describe("expected-state pure builders", () => { healthCheckStartPeriod: null, resourceCpuLimit: 1, resourceMemoryLimitMb: 512, + serverlessEnabled: true, }, ] as any, deploymentPorts: [ @@ -152,4 +154,114 @@ describe("expected-state pure builders", () => { }, ]); }); + + it("builds proxy-local serverless metadata with always-on worker upstreams", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_proxy", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + status: "sleeping", + serverIsProxy: true, + }, + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + status: "healthy", + serverIsProxy: false, + }, + ] as any, + containers: [ + { + deploymentId: "dep_proxy", + desiredState: "stopped", + }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_1", + domain: "app.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_proxy"], + upstreams: [ + { + deploymentId: "dep_worker", + serverId: "worker_1", + url: "10.0.0.20:3000", + local: false, + alwaysOn: true, + }, + ], + }, + ]); + }); + + it("omits serverless metadata for worker-only services", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + status: "healthy", + serverIsProxy: false, + }, + ] as any, + containers: [], + }); + + expect(routes).toEqual([]); + }); }); From 893d506bed60c56ac60813c0f59104c463411cd2 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:09:26 +1000 Subject: [PATCH 10/29] Add serverless settings help text --- web/components/service/details/serverless-section.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index ef417d8a..eb6e7007 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -166,6 +166,11 @@ export const ServerlessSection = memo(function ServerlessSection({
+

+ Containers scale down to zero when idle and wake on traffic. Requests + while sleeping are queued and served after the container is ready. +

+ {validationError && (

{validationError}

)} From 5918ddfe3e89d4ad33fda9a68fe0c73a01387c6a Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:15:34 +1000 Subject: [PATCH 11/29] Add serverless gateway lifecycle logs --- agent/internal/serverless/gateway.go | 143 +++++++++++++++++++++++++-- 1 file changed, 135 insertions(+), 8 deletions(-) diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index c2c6de1a..8dd2263a 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -198,13 +198,27 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, return nil, fmt.Errorf("no ready upstreams for host %s", host) } + wakeStartedAt := time.Now() if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && hasAlwaysOnUpstream(ready)) { + log.Printf( + "[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=background", + host, + len(sleepingLocalIDs), + len(ready), + ) go g.wakeLocalDeployments(route, state, sleepingLocalIDs) return ready, nil } + log.Printf( + "[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=blocking timeout=%s", + host, + len(sleepingLocalIDs), + len(ready), + wakeTimeout(route.WakeTimeoutSeconds), + ) g.wakeLocalDeployments(route, state, sleepingLocalIDs) - return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds) + return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds, wakeStartedAt) } func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) ([]agenthttp.ServerlessUpstream, []string, error) { @@ -264,29 +278,54 @@ func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state * for _, deploymentID := range deploymentIDs { expected, ok := expectedByDeploymentID[deploymentID] if !ok { + log.Printf( + "[serverless-gateway] wake skipped host=%s deployment=%s reason=missing_expected_state", + route.Domain, + deploymentID, + ) continue } + deployStartedAt := time.Now() + log.Printf( + "[serverless-gateway] wake starting host=%s deployment=%s service=%s container=%s", + route.Domain, + deploymentID, + expected.ServiceID, + expected.Name, + ) g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ Type: "wake_started", DeploymentID: deploymentID, }) if err := g.runtime.DeployServerlessContainer(expected); err != nil { - log.Printf("[serverless-gateway] wake failed for deployment %s: %v", deploymentID, err) + log.Printf( + "[serverless-gateway] wake failed host=%s deployment=%s service=%s latency=%s error=%v", + route.Domain, + deploymentID, + expected.ServiceID, + roundDuration(time.Since(deployStartedAt)), + err, + ) g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ Type: "wake_failed", DeploymentID: deploymentID, Error: err.Error(), }) + continue } + log.Printf( + "[serverless-gateway] wake container started host=%s deployment=%s service=%s latency=%s", + route.Domain, + deploymentID, + expected.ServiceID, + roundDuration(time.Since(deployStartedAt)), + ) } g.evictUpstreams(route.Domain) } -func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int) ([]agenthttp.ServerlessUpstream, error) { - timeout := time.Duration(wakeTimeoutSeconds) * time.Second - if timeout <= 0 { - timeout = 5 * time.Minute - } +func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time) ([]agenthttp.ServerlessUpstream, error) { + timeout := wakeTimeout(wakeTimeoutSeconds) deadline := time.Now().Add(timeout) for { @@ -296,12 +335,29 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi return nil, err } if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) { + log.Printf( + "[serverless-gateway] wake ready host=%s upstreams=%d latency=%s", + route.Domain, + len(ready), + roundDuration(time.Since(startedAt)), + ) return ready, nil } if time.Now().After(deadline) { if len(ready) > 0 { + log.Printf( + "[serverless-gateway] wake partially ready host=%s upstreams=%d latency=%s", + route.Domain, + len(ready), + roundDuration(time.Since(startedAt)), + ) return ready, nil } + log.Printf( + "[serverless-gateway] wake timed out host=%s latency=%s", + route.Domain, + roundDuration(time.Since(startedAt)), + ) return nil, fmt.Errorf("timed out waiting for local serverless wake") } time.Sleep(wakePollInterval) @@ -425,7 +481,13 @@ func (g *Gateway) sleepHost(host string) { activity := g.activity(host) activity.mu.Lock() if activity.activeRequests > 0 { + activeRequests := activity.activeRequests activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep skipped host=%s reason=active_requests active=%d", + host, + activeRequests, + ) return } activity.sleepTimer = nil @@ -434,8 +496,14 @@ func (g *Gateway) sleepHost(host string) { state := g.runtime.ExpectedState() route := findRoute(state, host) if route == nil { + log.Printf("[serverless-gateway] sleep skipped host=%s reason=missing_route_metadata", host) return } + log.Printf( + "[serverless-gateway] sleep timer fired host=%s deployments=%d", + host, + len(route.LocalDeploymentIDs), + ) actualContainers, err := g.runtime.ListServerlessContainers() if err != nil { @@ -448,23 +516,70 @@ func (g *Gateway) sleepHost(host string) { for _, deploymentID := range route.LocalDeploymentIDs { expected, ok := expectedByDeploymentID[deploymentID] if !ok { + log.Printf( + "[serverless-gateway] sleep skipped host=%s deployment=%s reason=missing_expected_state", + host, + deploymentID, + ) continue } actual, ok := actualByDeploymentID[deploymentID] if !ok || actual.State != "running" { + reason := "no_local_container" + if ok { + reason = "container_not_running" + } + log.Printf( + "[serverless-gateway] sleep skipped host=%s deployment=%s reason=%s", + host, + deploymentID, + reason, + ) continue } + sleepStartedAt := time.Now() if expected.DesiredState != "stopped" { + log.Printf( + "[serverless-gateway] sleep starting host=%s deployment=%s service=%s container=%s", + host, + deploymentID, + expected.ServiceID, + actual.ID, + ) g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ Type: "sleep", DeploymentID: deploymentID, ContainerID: actual.ID, }) + } else { + log.Printf( + "[serverless-gateway] sleep cleanup starting host=%s deployment=%s service=%s container=%s reason=already_expected_stopped", + host, + deploymentID, + expected.ServiceID, + actual.ID, + ) } if err := g.runtime.RemoveServerlessContainer(actual.ID); err != nil { - log.Printf("[serverless-gateway] failed to sleep deployment %s: %v", deploymentID, err) + log.Printf( + "[serverless-gateway] sleep failed host=%s deployment=%s service=%s container=%s latency=%s error=%v", + host, + deploymentID, + expected.ServiceID, + actual.ID, + roundDuration(time.Since(sleepStartedAt)), + err, + ) continue } + log.Printf( + "[serverless-gateway] sleep complete host=%s deployment=%s service=%s container=%s latency=%s", + host, + deploymentID, + expected.ServiceID, + actual.ID, + roundDuration(time.Since(sleepStartedAt)), + ) } g.evictUpstreams(host) } @@ -577,6 +692,18 @@ func compareUpstream(a, b agenthttp.ServerlessUpstream) int { return strings.Compare(a.Url, b.Url) } +func wakeTimeout(wakeTimeoutSeconds int) time.Duration { + timeout := time.Duration(wakeTimeoutSeconds) * time.Second + if timeout <= 0 { + return 5 * time.Minute + } + return timeout +} + +func roundDuration(duration time.Duration) time.Duration { + return duration.Round(time.Millisecond) +} + func cloneUpstreams(upstreams []agenthttp.ServerlessUpstream) []agenthttp.ServerlessUpstream { return append([]agenthttp.ServerlessUpstream(nil), upstreams...) } From a7a1c89f1bbda9a37ca4cd8589f84f7db9da4227 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:23:58 +1000 Subject: [PATCH 12/29] Require public endpoint for serverless services --- web/actions/projects.ts | 19 ++++++ .../service/details/serverless-section.tsx | 28 +++++++-- web/lib/agent/expected-state.ts | 33 +++++++++- web/tests/expected-state.test.ts | 60 +++++++++++++++++++ 4 files changed, 135 insertions(+), 5 deletions(-) diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 27ecdac8..46776037 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1049,6 +1049,25 @@ export async function updateServiceServerlessSettings( } if (validated.enabled) { + const publicHttpPorts = await tx + .select({ id: servicePorts.id }) + .from(servicePorts) + .where( + and( + eq(servicePorts.serviceId, serviceId), + eq(servicePorts.isPublic, true), + eq(servicePorts.protocol, "http"), + isNotNull(servicePorts.domain), + ), + ) + .limit(1); + + if (publicHttpPorts.length === 0) { + throw new Error( + "Serverless services require a public HTTP port with a domain", + ); + } + const configuredReplicas = await tx .select({ count: serviceReplicas.count }) .from(serviceReplicas) diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index eb6e7007..a3c755c6 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -28,6 +28,19 @@ export const ServerlessSection = memo(function ServerlessSection({ const [minReadyReplicas, setMinReadyReplicas] = useState( String(service.serverlessMinReadyReplicas ?? 1), ); + const hasPublicHttpEndpoint = useMemo( + () => + service.ports.some( + (port) => port.isPublic && port.protocol === "http" && !!port.domain, + ), + [service.ports], + ); + const unavailableReason = service.stateful + ? "Serverless is only supported for stateless services" + : !hasPublicHttpEndpoint + ? "Add a public HTTP port with a domain to enable serverless" + : null; + const optionsDisabled = !!unavailableReason || isSaving; const parsed = useMemo( () => ({ @@ -39,8 +52,8 @@ export const ServerlessSection = memo(function ServerlessSection({ ); const validationError = useMemo(() => { - if (service.stateful && enabled) { - return "Serverless is only supported for stateless services"; + if (unavailableReason && enabled) { + return unavailableReason; } if ( !Number.isInteger(parsed.sleepAfterSeconds) || @@ -64,7 +77,7 @@ export const ServerlessSection = memo(function ServerlessSection({ return "Minimum ready replicas must be between 1 and 10"; } return null; - }, [enabled, parsed, service.stateful]); + }, [enabled, parsed, unavailableReason]); const hasChanges = enabled !== service.serverlessEnabled || @@ -108,7 +121,7 @@ export const ServerlessSection = memo(function ServerlessSection({
@@ -127,6 +140,7 @@ export const ServerlessSection = memo(function ServerlessSection({ max="86400" step="30" value={sleepAfterSeconds} + disabled={optionsDisabled} onChange={(event) => setSleepAfterSeconds(event.target.value)} />
@@ -144,6 +158,7 @@ export const ServerlessSection = memo(function ServerlessSection({ max="900" step="10" value={wakeTimeoutSeconds} + disabled={optionsDisabled} onChange={(event) => setWakeTimeoutSeconds(event.target.value)} /> @@ -161,6 +176,7 @@ export const ServerlessSection = memo(function ServerlessSection({ max="10" step="1" value={minReadyReplicas} + disabled={optionsDisabled} onChange={(event) => setMinReadyReplicas(event.target.value)} /> @@ -171,6 +187,10 @@ export const ServerlessSection = memo(function ServerlessSection({ while sleeping are queued and served after the container is ready.

+ {unavailableReason && !enabled && ( +

{unavailableReason}.

+ )} + {validationError && (

{validationError}

)} diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index ee6d57f2..1943a2a2 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, isNull } from "drizzle-orm"; import { db } from "@/db"; import { deploymentPorts, @@ -218,6 +218,8 @@ async function buildExpectedContainers( .where(inArray(serviceVolumes.serviceId, serviceIds)), ], ); + const serverlessRoutableServiceIds = + await fetchPublicHttpServiceIds(serviceIds); return buildExpectedContainersFromRows({ deployments: serverDeployments, @@ -226,6 +228,7 @@ async function buildExpectedContainers( secrets: serviceSecrets, volumes, serverIsProxy, + serverlessRoutableServiceIds, }); } @@ -243,6 +246,24 @@ async function fetchDeploymentPorts(deploymentIds: string[]) { .where(inArray(deploymentPorts.deploymentId, deploymentIds)); } +async function fetchPublicHttpServiceIds(serviceIds: string[]) { + if (serviceIds.length === 0) return new Set(); + + const rows = await db + .select({ serviceId: servicePorts.serviceId }) + .from(servicePorts) + .where( + and( + inArray(servicePorts.serviceId, serviceIds), + eq(servicePorts.isPublic, true), + eq(servicePorts.protocol, "http"), + isNotNull(servicePorts.domain), + ), + ); + + return new Set(rows.map((row) => row.serviceId)); +} + export function buildExpectedContainersFromRows({ deployments: deploymentRows, services: serviceRows, @@ -250,6 +271,7 @@ export function buildExpectedContainersFromRows({ secrets: secretRows, volumes: volumeRows, serverIsProxy = true, + serverlessRoutableServiceIds, }: { deployments: Deployment[]; services: Service[]; @@ -257,10 +279,18 @@ export function buildExpectedContainersFromRows({ secrets: SecretRow[]; volumes: VolumeRow[]; serverIsProxy?: boolean; + serverlessRoutableServiceIds?: Set; }): ExpectedContainer[] { const servicesById = new Map( serviceRows.map((service) => [service.id, service]), ); + const sleepableServiceIds = + serverlessRoutableServiceIds ?? + new Set( + serviceRows + .filter((service) => service.serverlessEnabled) + .map((service) => service.id), + ); const portsByDeploymentId = groupBy( deploymentPortRows, (port) => port.deploymentId, @@ -284,6 +314,7 @@ export function buildExpectedContainersFromRows({ desiredState: serverIsProxy && service.serverlessEnabled && + sleepableServiceIds.has(service.id) && dep.status === "sleeping" ? "stopped" : "running", diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index b5e883bf..571150cb 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -128,6 +128,66 @@ describe("expected-state pure builders", () => { ]); }); + it("keeps non-public serverless deployments running in expected state", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_private", + ipAddress: "10.0.0.10", + status: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_private", + name: "private-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + serverlessRoutableServiceIds: new Set(), + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "running", + }); + }); + + it("marks public serverless deployments stopped while sleeping", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + status: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + serverlessRoutableServiceIds: new Set(["svc_public"]), + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "stopped", + }); + }); + it("routes serverless HTTP services through the local wake gateway", () => { const routes = buildTraefikRoutes({ serverId: "server_local", From 1dc6422788801f35bca3af9a6e8ed5b598c9ddad Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:23:23 +1000 Subject: [PATCH 13/29] Fix serverless wake and sleep races --- agent/internal/agent/agent.go | 2 + agent/internal/agent/drift.go | 7 ++- agent/internal/agent/serverless.go | 59 +++++++++++++++----- agent/internal/agent/serverless_test.go | 67 +++++++++++++++++++++++ agent/internal/serverless/gateway.go | 33 +++++++---- agent/internal/serverless/gateway_test.go | 34 +++++++++++- 6 files changed, 173 insertions(+), 29 deletions(-) create mode 100644 agent/internal/agent/serverless_test.go diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index bf8bd72e..4742b6db 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -64,6 +64,7 @@ type Agent struct { serverlessMutex sync.Mutex pendingServerlessTransitions []agenthttp.ServerlessTransition pendingServerlessSleep map[string]struct{} + pendingServerlessWake map[string]struct{} expectedStateMutex sync.RWMutex latestExpectedState *agenthttp.ExpectedState Client *agenthttp.Client @@ -115,6 +116,7 @@ func NewAgent( IsProxy: isProxy, DisableDNS: disableDNS, pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{}, } } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 2f59024a..9b5d0218 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -104,7 +104,7 @@ func (a *Agent) handleIdle() { log.Printf("[idle] using cached state (CP unreachable)") } a.SetLatestExpectedState(expected) - a.ReconcilePendingServerlessSleepWithExpected(expected, fromCache) + a.ReconcilePendingServerlessTransitionsWithExpected(expected, fromCache) actual, err := a.getActualState() if err != nil { @@ -253,7 +253,7 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS for id, exp := range expectedMap { if _, exists := actualMap[id]; !exists { - if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) { + if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) || a.HasPendingServerlessWake(id) { continue } expectedContainer := exp @@ -271,6 +271,9 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS expectedContainer := exp actualContainer := act + if a.HasPendingServerlessWake(id) { + continue + } if desiredContainerState(exp) == "stopped" || a.HasPendingServerlessSleep(id) { if shouldStopDesiredStoppedContainer(act.State) { actions = append(actions, reconcileAction{ diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index cbfdb390..341fee5e 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -41,10 +41,21 @@ func (a *Agent) QueueServerlessTransition(transition agenthttp.ServerlessTransit } a.serverlessMutex.Lock() - if transition.Type == "sleep" { + if a.pendingServerlessSleep == nil { + a.pendingServerlessSleep = map[string]struct{}{} + } + if a.pendingServerlessWake == nil { + a.pendingServerlessWake = map[string]struct{}{} + } + switch transition.Type { + case "sleep": a.pendingServerlessSleep[transition.DeploymentID] = struct{}{} - } else if transition.Type == "wake_started" { + delete(a.pendingServerlessWake, transition.DeploymentID) + case "wake_started": delete(a.pendingServerlessSleep, transition.DeploymentID) + a.pendingServerlessWake[transition.DeploymentID] = struct{}{} + case "wake_failed": + delete(a.pendingServerlessWake, transition.DeploymentID) } a.pendingServerlessTransitions = append( a.pendingServerlessTransitions, @@ -81,8 +92,23 @@ func (a *Agent) HasPendingServerlessSleep(deploymentID string) bool { return ok } +func (a *Agent) HasPendingServerlessWake(deploymentID string) bool { + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + _, ok := a.pendingServerlessWake[deploymentID] + return ok +} + func (a *Agent) ShouldSuppressServerlessContainerReport(deploymentID string) bool { - if a.HasPendingServerlessSleep(deploymentID) { + a.serverlessMutex.Lock() + _, pendingSleep := a.pendingServerlessSleep[deploymentID] + _, pendingWake := a.pendingServerlessWake[deploymentID] + a.serverlessMutex.Unlock() + + if pendingWake { + return false + } + if pendingSleep { return true } @@ -99,7 +125,7 @@ func (a *Agent) ShouldSuppressServerlessContainerReport(deploymentID string) boo return false } -func (a *Agent) ReconcilePendingServerlessSleepWithExpected(state *agenthttp.ExpectedState, fromCache bool) { +func (a *Agent) ReconcilePendingServerlessTransitionsWithExpected(state *agenthttp.ExpectedState, fromCache bool) { if state == nil || fromCache { return } @@ -112,9 +138,13 @@ func (a *Agent) ReconcilePendingServerlessSleepWithExpected(state *agenthttp.Exp a.serverlessMutex.Lock() defer a.serverlessMutex.Unlock() pendingSleepTransitions := map[string]struct{}{} + pendingWakeTransitions := map[string]struct{}{} for _, transition := range a.pendingServerlessTransitions { - if transition.Type == "sleep" { + switch transition.Type { + case "sleep": pendingSleepTransitions[transition.DeploymentID] = struct{}{} + case "wake_started": + pendingWakeTransitions[transition.DeploymentID] = struct{}{} } } @@ -128,16 +158,15 @@ func (a *Agent) ReconcilePendingServerlessSleepWithExpected(state *agenthttp.Exp log.Printf("[serverless] sleep transition for deployment %s is not reflected in expected state; allowing reconcile", Truncate(deploymentID, 8)) } } -} -func (a *Agent) QueueServerlessWakeFailure(deploymentID string, err error) { - if err == nil { - return + for deploymentID := range a.pendingServerlessWake { + if _, stillReporting := pendingWakeTransitions[deploymentID]; stillReporting { + continue + } + delete(a.pendingServerlessWake, deploymentID) + desiredState, ok := desiredByDeploymentID[deploymentID] + if !ok || desiredState != "running" { + log.Printf("[serverless] wake transition for deployment %s is not reflected in expected state; allowing reconcile", Truncate(deploymentID, 8)) + } } - log.Printf("[serverless] wake failed for deployment %s: %v", Truncate(deploymentID, 8), err) - a.QueueServerlessTransition(agenthttp.ServerlessTransition{ - Type: "wake_failed", - DeploymentID: deploymentID, - Error: err.Error(), - }) } diff --git a/agent/internal/agent/serverless_test.go b/agent/internal/agent/serverless_test.go new file mode 100644 index 00000000..3662c9e8 --- /dev/null +++ b/agent/internal/agent/serverless_test.go @@ -0,0 +1,67 @@ +package agent + +import ( + "testing" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +func TestPendingServerlessWakeDoesNotStopStaleStoppedExpectedContainer(t *testing.T) { + agent := &Agent{ + DisableDNS: true, + pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{ + "dep_serverless": {}, + }, + } + expected := &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_serverless", + ServiceID: "svc_1", + Name: "svc_1-dep_serverless", + DesiredState: "stopped", + Image: "nginx", + }, + }, + } + actual := &ActualState{ + Containers: []container.Container{ + { + ID: "ctr_serverless", + Name: "svc_1-dep_serverless", + Image: "nginx", + State: "running", + DeploymentID: "dep_serverless", + }, + }, + } + + for _, action := range agent.planReconcile(expected, actual) { + if action.DeploymentID == "dep_serverless" { + t.Fatalf("planReconcile returned action for pending wake: %+v", action) + } + } +} + +func TestPendingServerlessWakeDoesNotSuppressContainerReport(t *testing.T) { + agent := &Agent{ + pendingServerlessSleep: map[string]struct{}{}, + pendingServerlessWake: map[string]struct{}{ + "dep_serverless": {}, + }, + latestExpectedState: &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "dep_serverless", + DesiredState: "stopped", + }, + }, + }, + } + + if agent.ShouldSuppressServerlessContainerReport("dep_serverless") { + t.Fatal("container report was suppressed while wake transition is pending") + } +} diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 8dd2263a..8683a8c7 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "sort" "strings" "sync" "sync/atomic" @@ -111,6 +112,9 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + g.beginActivity(host) + defer g.endActivity(host) + upstreams, err := g.getUpstreams(r.Context(), host) if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { @@ -127,9 +131,6 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - g.beginActivity(host) - defer g.endActivity(host) - upstream := upstreams[g.nextIndex(len(upstreams))] target, err := url.Parse("http://" + upstream.Url) if err != nil { @@ -537,6 +538,19 @@ func (g *Gateway) sleepHost(host string) { ) continue } + + activity.mu.Lock() + if activity.activeRequests > 0 { + activeRequests := activity.activeRequests + activity.mu.Unlock() + log.Printf( + "[serverless-gateway] sleep skipped host=%s deployment=%s reason=active_requests active=%d", + host, + deploymentID, + activeRequests, + ) + return + } sleepStartedAt := time.Now() if expected.DesiredState != "stopped" { log.Printf( @@ -561,6 +575,7 @@ func (g *Gateway) sleepHost(host string) { ) } if err := g.runtime.RemoveServerlessContainer(actual.ID); err != nil { + activity.mu.Unlock() log.Printf( "[serverless-gateway] sleep failed host=%s deployment=%s service=%s container=%s latency=%s error=%v", host, @@ -572,6 +587,8 @@ func (g *Gateway) sleepHost(host string) { ) continue } + g.evictUpstreams(host) + activity.mu.Unlock() log.Printf( "[serverless-gateway] sleep complete host=%s deployment=%s service=%s container=%s latency=%s", host, @@ -667,13 +684,9 @@ func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool { } func sortUpstreams(upstreams []agenthttp.ServerlessUpstream) { - for i := 0; i < len(upstreams); i++ { - for j := i + 1; j < len(upstreams); j++ { - if compareUpstream(upstreams[j], upstreams[i]) < 0 { - upstreams[i], upstreams[j] = upstreams[j], upstreams[i] - } - } - } + sort.Slice(upstreams, func(i, j int) bool { + return compareUpstream(upstreams[i], upstreams[j]) < 0 + }) } func compareUpstream(a, b agenthttp.ServerlessUpstream) int { diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index 5f0a2f89..2aa2e143 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -19,6 +19,7 @@ type fakeRuntime struct { removed []string deployCalls int deployErr error + afterList func() } func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { @@ -59,8 +60,13 @@ func (f *fakeRuntime) RemoveServerlessContainer(containerID string) error { func (f *fakeRuntime) ListServerlessContainers() ([]container.Container, error) { f.mu.Lock() - defer f.mu.Unlock() - return append([]container.Container(nil), f.containers...), nil + containers := append([]container.Container(nil), f.containers...) + afterList := f.afterList + f.mu.Unlock() + if afterList != nil { + afterList() + } + return containers, nil } func (f *fakeRuntime) GetServerlessContainerHealth(containerID string) string { @@ -169,6 +175,30 @@ func TestSleepHostRemovesLocalContainerAndReportsSleep(t *testing.T) { } } +func TestSleepHostRechecksActivityBeforeRemovingContainer(t *testing.T) { + state := testExpectedState("running") + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + runtime.afterList = func() { + gateway.beginActivity("app.example.com") + } + + gateway.sleepHost("app.example.com") + + transitions, removed, _ := runtime.snapshot() + if len(removed) != 0 { + t.Fatalf("removed = %+v, want no removals while request is active", removed) + } + if len(transitions) != 0 { + t.Fatalf("transitions = %+v, want no sleep transition while request is active", transitions) + } +} + func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { state := &agenthttp.ExpectedState{ Containers: []agenthttp.ExpectedContainer{ From 8b4be3d7c9f5fa67b5c7d15de5ff1732cdbd9079 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:32:43 +1000 Subject: [PATCH 14/29] Harden proxy-local serverless lifecycle --- agent/internal/agent/agent.go | 3 + agent/internal/agent/drift.go | 6 +- agent/internal/agent/serverless.go | 37 ++- agent/internal/serverless/gateway.go | 329 ++++++++++++++++++---- agent/internal/serverless/gateway_test.go | 89 +++++- 5 files changed, 396 insertions(+), 68 deletions(-) diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index 4742b6db..5871549c 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -61,6 +61,8 @@ type Agent struct { pendingWorkResults []agenthttp.CompletedWorkItem deploymentErrorMutex sync.Mutex pendingDeploymentErrors []agenthttp.DeploymentError + deployLockMutex sync.Mutex + deploymentDeployLocks map[string]*sync.Mutex serverlessMutex sync.Mutex pendingServerlessTransitions []agenthttp.ServerlessTransition pendingServerlessSleep map[string]struct{} @@ -115,6 +117,7 @@ func NewAgent( Builder: builder, IsProxy: isProxy, DisableDNS: disableDNS, + deploymentDeployLocks: map[string]*sync.Mutex{}, pendingServerlessSleep: map[string]struct{}{}, pendingServerlessWake: map[string]struct{}{}, } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 9b5d0218..6d083884 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -472,7 +472,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if action.Expected == nil { return fmt.Errorf("missing expected container for %s", action.Kind) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to deploy container: %w", err) } return nil @@ -486,7 +486,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if err := container.Stop(action.Actual.ID); err != nil { log.Printf("[reconcile] warning: failed to stop old container: %v", err) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } } @@ -499,7 +499,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if err := container.Stop(action.Actual.ID); err != nil { log.Printf("[reconcile] warning: failed to stop old container: %v", err) } - if err := a.Reconciler.Deploy(*action.Expected); err != nil { + if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } return nil diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index 341fee5e..9d777eb8 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -2,6 +2,7 @@ package agent import ( "log" + "sync" "techulus/cloud-agent/internal/container" agenthttp "techulus/cloud-agent/internal/http" @@ -20,7 +21,41 @@ func (a *Agent) ExpectedState() *agenthttp.ExpectedState { } func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { - return a.Reconciler.Deploy(expected) + return a.DeployExpectedContainer(expected) +} + +func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) error { + return a.withDeploymentDeployLock(expected.DeploymentID, func() error { + containers, err := container.List() + if err == nil { + for _, actual := range containers { + if actual.DeploymentID != expected.DeploymentID || actual.State != "running" { + continue + } + if normalizeImage(actual.Image) == normalizeImage(expected.Image) { + return nil + } + } + } + return a.Reconciler.Deploy(expected) + }) +} + +func (a *Agent) withDeploymentDeployLock(deploymentID string, fn func() error) error { + a.deployLockMutex.Lock() + if a.deploymentDeployLocks == nil { + a.deploymentDeployLocks = map[string]*sync.Mutex{} + } + lock, ok := a.deploymentDeployLocks[deploymentID] + if !ok { + lock = &sync.Mutex{} + a.deploymentDeployLocks[deploymentID] = lock + } + a.deployLockMutex.Unlock() + + lock.Lock() + defer lock.Unlock() + return fn() } func (a *Agent) RemoveServerlessContainer(containerID string) error { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 8683a8c7..07b97a2d 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -24,7 +24,10 @@ const ( upstreamCacheTTL = 10 * time.Second ) -var wakePollInterval = 500 * time.Millisecond +var ( + wakePollInterval = 500 * time.Millisecond + idleTimerSeedInterval = 15 * time.Second +) type Gateway struct { runtime Runtime @@ -102,6 +105,20 @@ func (g *Gateway) Start(ctx context.Context) error { } }() + go func() { + g.seedIdleTimers() + ticker := time.NewTicker(idleTimerSeedInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + g.seedIdleTimers() + } + } + }() + return nil } @@ -112,8 +129,15 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - g.beginActivity(host) - defer g.endActivity(host) + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + log.Printf("[serverless-gateway] no route metadata for host %s", host) + http.Error(w, "service unavailable", http.StatusServiceUnavailable) + return + } + activityKey := serviceActivityKey(route.ServiceID) + g.beginActivity(activityKey) + defer g.endActivity(activityKey, route.ServiceID, route.SleepAfterSeconds) upstreams, err := g.getUpstreams(r.Context(), host) if err != nil { @@ -207,7 +231,12 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, len(sleepingLocalIDs), len(ready), ) - go g.wakeLocalDeployments(route, state, sleepingLocalIDs) + go func() { + startedIDs := g.wakeLocalDeployments(route, state, sleepingLocalIDs) + if len(startedIDs) > 0 { + g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, wakeStartedAt, startedIDs) + } + }() return ready, nil } @@ -218,8 +247,11 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, len(ready), wakeTimeout(route.WakeTimeoutSeconds), ) - g.wakeLocalDeployments(route, state, sleepingLocalIDs) - return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds, wakeStartedAt) + startedIDs := g.wakeLocalDeployments(route, state, sleepingLocalIDs) + if len(startedIDs) == 0 { + return nil, fmt.Errorf("failed to start local serverless wake") + } + return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds, wakeStartedAt, startedIDs) } func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) ([]agenthttp.ServerlessUpstream, []string, error) { @@ -274,8 +306,9 @@ func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenth return upstreams, sleepingLocalIDs, nil } -func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState, deploymentIDs []string) { +func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState, deploymentIDs []string) []string { expectedByDeploymentID := expectedContainersByDeploymentID(state) + startedIDs := []string{} for _, deploymentID := range deploymentIDs { expected, ok := expectedByDeploymentID[deploymentID] if !ok { @@ -314,6 +347,7 @@ func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state * }) continue } + startedIDs = append(startedIDs, deploymentID) log.Printf( "[serverless-gateway] wake container started host=%s deployment=%s service=%s latency=%s", route.Domain, @@ -323,11 +357,12 @@ func (g *Gateway) wakeLocalDeployments(route *agenthttp.ServerlessRoute, state * ) } g.evictUpstreams(route.Domain) + return startedIDs } -func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time) ([]agenthttp.ServerlessUpstream, error) { +func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time, wokenDeploymentIDs []string) ([]agenthttp.ServerlessUpstream, error) { timeout := wakeTimeout(wakeTimeoutSeconds) - deadline := time.Now().Add(timeout) + deadline := startedAt.Add(timeout) for { state := g.runtime.ExpectedState() @@ -336,6 +371,10 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi return nil, err } if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) { + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + if len(pendingIDs) > 0 { + go g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, startedAt, pendingIDs) + } log.Printf( "[serverless-gateway] wake ready host=%s upstreams=%d latency=%s", route.Domain, @@ -345,6 +384,8 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi return ready, nil } if time.Now().After(deadline) { + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + g.queueWakeTimeouts(route, pendingIDs, startedAt) if len(ready) > 0 { log.Printf( "[serverless-gateway] wake partially ready host=%s upstreams=%d latency=%s", @@ -365,6 +406,70 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi } } +func (g *Gateway) waitForWokenDeployments(route *agenthttp.ServerlessRoute, wakeTimeoutSeconds int, startedAt time.Time, wokenDeploymentIDs []string) { + timeout := wakeTimeout(wakeTimeoutSeconds) + deadline := startedAt.Add(timeout) + + for { + state := g.runtime.ExpectedState() + ready, _, err := g.readyUpstreams(route, state) + if err != nil { + log.Printf("[serverless-gateway] wake monitor failed host=%s error=%v", route.Domain, err) + return + } + pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) + if len(pendingIDs) == 0 { + log.Printf( + "[serverless-gateway] wake ready host=%s deployments=%d latency=%s", + route.Domain, + len(wokenDeploymentIDs), + roundDuration(time.Since(startedAt)), + ) + return + } + if time.Now().After(deadline) { + g.queueWakeTimeouts(route, pendingIDs, startedAt) + return + } + time.Sleep(wakePollInterval) + } +} + +func pendingWakeDeploymentIDs(wokenDeploymentIDs []string, ready []agenthttp.ServerlessUpstream) []string { + readyIDs := map[string]struct{}{} + for _, upstream := range ready { + if upstream.Local { + readyIDs[upstream.DeploymentID] = struct{}{} + } + } + + pendingIDs := []string{} + for _, deploymentID := range wokenDeploymentIDs { + if _, ok := readyIDs[deploymentID]; !ok { + pendingIDs = append(pendingIDs, deploymentID) + } + } + return pendingIDs +} + +func (g *Gateway) queueWakeTimeouts(route *agenthttp.ServerlessRoute, deploymentIDs []string, startedAt time.Time) { + for _, deploymentID := range deploymentIDs { + errMessage := fmt.Sprintf("timed out waiting %s for local serverless wake", roundDuration(time.Since(startedAt))) + log.Printf( + "[serverless-gateway] wake timed out host=%s deployment=%s service=%s latency=%s", + route.Domain, + deploymentID, + route.ServiceID, + roundDuration(time.Since(startedAt)), + ) + g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ + Type: "wake_failed", + DeploymentID: deploymentID, + Error: errMessage, + }) + } +} + func (g *Gateway) isContainerReady(actual container.Container, expected agenthttp.ExpectedContainer) bool { if actual.State != "running" { return false @@ -426,20 +531,72 @@ func (g *Gateway) evictUpstreams(host string) { delete(g.upstreamCache, host) } -func (g *Gateway) activity(host string) *activityState { +func (g *Gateway) evictServiceUpstreams(serviceID string) { + state := g.runtime.ExpectedState() + g.mu.Lock() + defer g.mu.Unlock() + if state == nil { + g.upstreamCache = map[string]cachedUpstreams{} + return + } + for _, route := range state.Serverless.Routes { + if route.ServiceID == serviceID { + delete(g.upstreamCache, normalizeHost(route.Domain)) + } + } +} + +func (g *Gateway) seedIdleTimers() { + state := g.runtime.ExpectedState() + if state == nil { + return + } + actualContainers, err := g.runtime.ListServerlessContainers() + if err != nil { + log.Printf("[serverless-gateway] failed to seed idle timers: %v", err) + return + } + actualByDeploymentID := actualContainersByDeploymentID(actualContainers) + seenServices := map[string]struct{}{} + for _, route := range state.Serverless.Routes { + if _, seen := seenServices[route.ServiceID]; seen { + continue + } + seenServices[route.ServiceID] = struct{}{} + if !hasRunningLocalDeployment(route.LocalDeploymentIDs, actualByDeploymentID) { + continue + } + g.scheduleSleepTimer(serviceActivityKey(route.ServiceID), route.ServiceID, route.SleepAfterSeconds) + } +} + +func (g *Gateway) scheduleSleepTimer(key string, serviceID string, sleepAfterSeconds int) { + activity := g.activity(key) + activity.mu.Lock() + defer activity.mu.Unlock() + if activity.activeRequests > 0 || activity.sleepTimer != nil { + return + } + delay := sleepDelay(sleepAfterSeconds) + activity.sleepTimer = time.AfterFunc(delay, func() { + g.sleepService(serviceID) + }) +} + +func (g *Gateway) activity(key string) *activityState { g.activityMu.Lock() defer g.activityMu.Unlock() - activity, ok := g.activities[host] + activity, ok := g.activities[key] if !ok { activity = &activityState{} - g.activities[host] = activity + g.activities[key] = activity } return activity } -func (g *Gateway) beginActivity(host string) { - activity := g.activity(host) +func (g *Gateway) beginActivity(key string) { + activity := g.activity(key) activity.mu.Lock() defer activity.mu.Unlock() @@ -450,8 +607,8 @@ func (g *Gateway) beginActivity(host string) { activity.activeRequests += 1 } -func (g *Gateway) endActivity(host string) { - activity := g.activity(host) +func (g *Gateway) endActivity(key string, serviceID string, sleepAfterSeconds int) { + activity := g.activity(key) activity.mu.Lock() defer activity.mu.Unlock() @@ -462,31 +619,34 @@ func (g *Gateway) endActivity(host string) { return } - route := findRoute(g.runtime.ExpectedState(), host) - if route == nil { - return - } - delay := time.Duration(route.SleepAfterSeconds) * time.Second - if delay <= 0 { - delay = 5 * time.Minute - } if activity.sleepTimer != nil { activity.sleepTimer.Stop() } + delay := sleepDelay(sleepAfterSeconds) activity.sleepTimer = time.AfterFunc(delay, func() { - g.sleepHost(host) + g.sleepService(serviceID) }) } func (g *Gateway) sleepHost(host string) { - activity := g.activity(host) + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + log.Printf("[serverless-gateway] sleep skipped host=%s reason=missing_route_metadata", host) + return + } + g.sleepService(route.ServiceID) +} + +func (g *Gateway) sleepService(serviceID string) { + activityKey := serviceActivityKey(serviceID) + activity := g.activity(activityKey) activity.mu.Lock() if activity.activeRequests > 0 { activeRequests := activity.activeRequests activity.mu.Unlock() log.Printf( - "[serverless-gateway] sleep skipped host=%s reason=active_requests active=%d", - host, + "[serverless-gateway] sleep skipped service=%s reason=active_requests active=%d", + serviceID, activeRequests, ) return @@ -495,31 +655,32 @@ func (g *Gateway) sleepHost(host string) { activity.mu.Unlock() state := g.runtime.ExpectedState() - route := findRoute(state, host) + route := findRouteByServiceID(state, serviceID) if route == nil { - log.Printf("[serverless-gateway] sleep skipped host=%s reason=missing_route_metadata", host) + log.Printf("[serverless-gateway] sleep skipped service=%s reason=missing_route_metadata", serviceID) return } + localDeploymentIDs := localDeploymentIDsForService(state, serviceID) log.Printf( - "[serverless-gateway] sleep timer fired host=%s deployments=%d", - host, - len(route.LocalDeploymentIDs), + "[serverless-gateway] sleep timer fired service=%s deployments=%d", + serviceID, + len(localDeploymentIDs), ) actualContainers, err := g.runtime.ListServerlessContainers() if err != nil { - log.Printf("[serverless-gateway] failed to list containers before sleep for %s: %v", host, err) + log.Printf("[serverless-gateway] failed to list containers before sleep for service %s: %v", serviceID, err) return } expectedByDeploymentID := expectedContainersByDeploymentID(state) actualByDeploymentID := actualContainersByDeploymentID(actualContainers) - for _, deploymentID := range route.LocalDeploymentIDs { + for _, deploymentID := range localDeploymentIDs { expected, ok := expectedByDeploymentID[deploymentID] if !ok { log.Printf( - "[serverless-gateway] sleep skipped host=%s deployment=%s reason=missing_expected_state", - host, + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=missing_expected_state", + serviceID, deploymentID, ) continue @@ -531,21 +692,29 @@ func (g *Gateway) sleepHost(host string) { reason = "container_not_running" } log.Printf( - "[serverless-gateway] sleep skipped host=%s deployment=%s reason=%s", - host, + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=%s", + serviceID, deploymentID, reason, ) continue } + if expected.DesiredState != "stopped" && !g.isContainerReady(actual, expected) { + log.Printf( + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=container_not_ready", + serviceID, + deploymentID, + ) + continue + } activity.mu.Lock() if activity.activeRequests > 0 { activeRequests := activity.activeRequests activity.mu.Unlock() log.Printf( - "[serverless-gateway] sleep skipped host=%s deployment=%s reason=active_requests active=%d", - host, + "[serverless-gateway] sleep skipped service=%s deployment=%s reason=active_requests active=%d", + serviceID, deploymentID, activeRequests, ) @@ -554,10 +723,9 @@ func (g *Gateway) sleepHost(host string) { sleepStartedAt := time.Now() if expected.DesiredState != "stopped" { log.Printf( - "[serverless-gateway] sleep starting host=%s deployment=%s service=%s container=%s", - host, + "[serverless-gateway] sleep starting service=%s deployment=%s container=%s", + serviceID, deploymentID, - expected.ServiceID, actual.ID, ) g.runtime.QueueServerlessTransition(agenthttp.ServerlessTransition{ @@ -567,38 +735,35 @@ func (g *Gateway) sleepHost(host string) { }) } else { log.Printf( - "[serverless-gateway] sleep cleanup starting host=%s deployment=%s service=%s container=%s reason=already_expected_stopped", - host, + "[serverless-gateway] sleep cleanup starting service=%s deployment=%s container=%s reason=already_expected_stopped", + serviceID, deploymentID, - expected.ServiceID, actual.ID, ) } if err := g.runtime.RemoveServerlessContainer(actual.ID); err != nil { activity.mu.Unlock() log.Printf( - "[serverless-gateway] sleep failed host=%s deployment=%s service=%s container=%s latency=%s error=%v", - host, + "[serverless-gateway] sleep failed service=%s deployment=%s container=%s latency=%s error=%v", + serviceID, deploymentID, - expected.ServiceID, actual.ID, roundDuration(time.Since(sleepStartedAt)), err, ) continue } - g.evictUpstreams(host) + g.evictServiceUpstreams(serviceID) activity.mu.Unlock() log.Printf( - "[serverless-gateway] sleep complete host=%s deployment=%s service=%s container=%s latency=%s", - host, + "[serverless-gateway] sleep complete service=%s deployment=%s container=%s latency=%s", + serviceID, deploymentID, - expected.ServiceID, actual.ID, roundDuration(time.Since(sleepStartedAt)), ) } - g.evictUpstreams(host) + g.evictServiceUpstreams(serviceID) } func (g *Gateway) stopAllActivities() { @@ -632,6 +797,39 @@ func findRoute(state *agenthttp.ExpectedState, host string) *agenthttp.Serverles return nil } +func findRouteByServiceID(state *agenthttp.ExpectedState, serviceID string) *agenthttp.ServerlessRoute { + if state == nil { + return nil + } + for i := range state.Serverless.Routes { + if state.Serverless.Routes[i].ServiceID == serviceID { + return &state.Serverless.Routes[i] + } + } + return nil +} + +func localDeploymentIDsForService(state *agenthttp.ExpectedState, serviceID string) []string { + if state == nil { + return nil + } + ids := map[string]struct{}{} + for _, route := range state.Serverless.Routes { + if route.ServiceID != serviceID { + continue + } + for _, deploymentID := range route.LocalDeploymentIDs { + ids[deploymentID] = struct{}{} + } + } + deploymentIDs := make([]string, 0, len(ids)) + for deploymentID := range ids { + deploymentIDs = append(deploymentIDs, deploymentID) + } + sort.Strings(deploymentIDs) + return deploymentIDs +} + func expectedContainersByDeploymentID(state *agenthttp.ExpectedState) map[string]agenthttp.ExpectedContainer { containersByDeploymentID := map[string]agenthttp.ExpectedContainer{} if state == nil { @@ -683,6 +881,15 @@ func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool { return false } +func hasRunningLocalDeployment(deploymentIDs []string, actualByDeploymentID map[string]container.Container) bool { + for _, deploymentID := range deploymentIDs { + if actual, ok := actualByDeploymentID[deploymentID]; ok && actual.State == "running" { + return true + } + } + return false +} + func sortUpstreams(upstreams []agenthttp.ServerlessUpstream) { sort.Slice(upstreams, func(i, j int) bool { return compareUpstream(upstreams[i], upstreams[j]) < 0 @@ -713,6 +920,18 @@ func wakeTimeout(wakeTimeoutSeconds int) time.Duration { return timeout } +func sleepDelay(sleepAfterSeconds int) time.Duration { + delay := time.Duration(sleepAfterSeconds) * time.Second + if delay <= 0 { + return 5 * time.Minute + } + return delay +} + +func serviceActivityKey(serviceID string) string { + return "service:" + serviceID +} + func roundDuration(duration time.Duration) time.Duration { return duration.Round(time.Millisecond) } diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index 2aa2e143..3b2a9fcb 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -12,14 +12,15 @@ import ( ) type fakeRuntime struct { - mu sync.Mutex - state *agenthttp.ExpectedState - containers []container.Container - transitions []agenthttp.ServerlessTransition - removed []string - deployCalls int - deployErr error - afterList func() + mu sync.Mutex + state *agenthttp.ExpectedState + containers []container.Container + transitions []agenthttp.ServerlessTransition + removed []string + deployCalls int + deployErr error + afterList func() + healthStatus string } func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { @@ -70,6 +71,9 @@ func (f *fakeRuntime) ListServerlessContainers() ([]container.Container, error) } func (f *fakeRuntime) GetServerlessContainerHealth(containerID string) string { + if f.healthStatus != "" { + return f.healthStatus + } return "healthy" } @@ -185,7 +189,7 @@ func TestSleepHostRechecksActivityBeforeRemovingContainer(t *testing.T) { } gateway := NewGateway(runtime) runtime.afterList = func() { - gateway.beginActivity("app.example.com") + gateway.beginActivity(serviceActivityKey("svc_1")) } gateway.sleepHost("app.example.com") @@ -199,6 +203,73 @@ func TestSleepHostRechecksActivityBeforeRemovingContainer(t *testing.T) { } } +func TestSleepHostUsesServiceActivityAcrossDomains(t *testing.T) { + state := testExpectedState("running") + secondRoute := state.Serverless.Routes[0] + secondRoute.Domain = "api.example.com" + state.Serverless.Routes = append(state.Serverless.Routes, secondRoute) + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + {ID: "ctr-local", State: "running", DeploymentID: "dep_local", ServiceID: "svc_1"}, + }, + } + gateway := NewGateway(runtime) + gateway.beginActivity(serviceActivityKey("svc_1")) + + gateway.sleepHost("api.example.com") + + transitions, removed, _ := runtime.snapshot() + if len(removed) != 0 { + t.Fatalf("removed = %+v, want no removals while sibling domain is active", removed) + } + if len(transitions) != 0 { + t.Fatalf("transitions = %+v, want no sleep transition while sibling domain is active", transitions) + } +} + +func TestWakeTimeoutQueuesWakeFailedTransition(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Containers[0].HealthCheck = &agenthttp.HealthCheck{ + Cmd: "curl http://localhost:3000/health", + Interval: 1, + Timeout: 1, + Retries: 1, + } + state.Serverless.Routes[0].Upstreams = nil + state.Serverless.Routes[0].WakeTimeoutSeconds = 1 + runtime := &fakeRuntime{ + state: state, + healthStatus: "unhealthy", + } + gateway := NewGateway(runtime) + + _, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err == nil { + t.Fatal("getUpstreams succeeded, want timeout error") + } + + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 2 { + t.Fatalf("transitions = %+v, want wake_started and wake_failed", transitions) + } + if transitions[0].Type != "wake_started" || transitions[1].Type != "wake_failed" { + t.Fatalf("transitions = %+v, want wake_started then wake_failed", transitions) + } + if transitions[1].DeploymentID != "dep_local" || transitions[1].Error == "" { + t.Fatalf("wake_failed transition = %+v, want dep_local with error", transitions[1]) + } +} + func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { state := &agenthttp.ExpectedState{ Containers: []agenthttp.ExpectedContainer{ From 8988ab55fb92a3f8997abb5653057ac9c5f91d7d Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:15:24 +1000 Subject: [PATCH 15/29] Restrict serverless routes to owner proxies --- docs/agents/architecture.mdx | 12 +++- docs/architecture.mdx | 12 +++- docs/services/scaling.mdx | 6 ++ web/lib/agent/expected-state.ts | 50 ++++++++++---- web/tests/expected-state.test.ts | 113 ++++++++++++++++++++++++++++++- 5 files changed, 173 insertions(+), 20 deletions(-) diff --git a/docs/agents/architecture.mdx b/docs/agents/architecture.mdx index d24733a7..e9eef776 100644 --- a/docs/agents/architecture.mdx +++ b/docs/agents/architecture.mdx @@ -79,9 +79,15 @@ reports, the command can be retried up to the fixed attempt limit. ## Serverless Gateway Proxy agents run a local HTTP wake gateway on `127.0.0.1:18080`. The control -plane emits Traefik routes to this gateway only when the service has a -proxy-hosted serverless deployment. Worker-only serverless services keep normal -direct routes because worker deployments do not sleep. +plane emits Traefik routes to this gateway only on proxy nodes that host a local +proxy deployment for that serverless service. Proxies without a local +proxy-hosted deployment do not emit a public HTTP route for that service. +Worker-only serverless services keep normal direct routes because worker +deployments do not sleep. + +This means public DNS or external load balancers must route a serverless +service's traffic only to proxy nodes that own a local proxy replica for that +service. For serverless service traffic, Traefik forwards the public request to the local gateway. The gateway resolves the host from expected-state serverless metadata, diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 355643c7..569a7bba 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -242,9 +242,15 @@ sequenceDiagram Gateway->>Container: Proxy original request ``` -Proxy Traefik routes point to the local wake gateway only when a service has at -least one proxy-hosted serverless deployment. Worker-only serverless services use -normal direct routes because there is nothing local to wake. +Proxy Traefik routes point to the local wake gateway only on proxy nodes that +host a local proxy deployment for that serverless service. Non-owner proxies do +not emit a public HTTP route for that service, even when always-on worker +upstreams exist. Worker-only serverless services use normal direct routes because +there is nothing local to wake. + +Public DNS or external load balancers must therefore send serverless traffic only +to proxy nodes that own a local proxy replica for that service. Cross-proxy wake +coordination is intentionally out of scope. The wake gateway keeps the incoming request open while it starts local proxy replicas, unless an always-on worker upstream is already ready. `Min Ready` diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index e564bb86..8f4e4432 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -22,6 +22,11 @@ worker replicas, it stays always on and routes directly. If a service has both proxy and worker replicas, proxy replicas may sleep while worker replicas remain routable. +For services with proxy-hosted serverless replicas, public traffic must be sent +only to proxy nodes that host a local proxy replica for that service. Non-owner +proxies do not emit a public HTTP route for the service, so DNS or the external +load balancer must avoid those proxies for that service's domains. + Serverless settings are configured per service: | Setting | Default | Description | @@ -59,4 +64,5 @@ You can also manually lock any service to a specific server by setting the locke - Maximum 10 replicas per service. - Serverless scaling is supported only for stateless HTTP services with a public domain. - Sleep and wake are proxy-local. Worker replicas are intentionally always on. +- Serverless traffic must be routed only to proxy nodes that own a local proxy replica for that service. - Proxy agents report sleep and wake transitions through normal status reports. diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 1943a2a2..6e7abc94 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -547,9 +547,12 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { ), ) : Promise.resolve([]), - supportsServerlessGateway && serviceIds.length > 0 + serviceIds.length > 0 ? db - .select({ serviceId: deployments.serviceId }) + .select({ + serviceId: deployments.serviceId, + serverId: deployments.serverId, + }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) .where( @@ -566,24 +569,39 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { const proxyHostedServerlessServiceIds = new Set( proxyHostedServerlessDeployments.map((deployment) => deployment.serviceId), ); - const serverlessServiceIds = supportsServerlessGateway - ? new Set( - allServices - .filter( - (service) => - !service.stateful && - service.serverlessEnabled && - proxyHostedServerlessServiceIds.has(service.id), - ) - .map((service) => service.id), + const localProxyHostedServerlessServiceIds = new Set( + proxyHostedServerlessDeployments + .filter((deployment) => deployment.serverId === server.id) + .map((deployment) => deployment.serviceId), + ); + const serverlessProxyRoutedServiceIds = new Set( + allServices + .filter( + (service) => + !service.stateful && + service.serverlessEnabled && + proxyHostedServerlessServiceIds.has(service.id), ) - : new Set(); + .map((service) => service.id), + ); + const serverlessServiceIds = new Set( + [...serverlessProxyRoutedServiceIds].filter((serviceId) => + supportsServerlessGateway && + localProxyHostedServerlessServiceIds.has(serviceId), + ), + ); + const serverlessRouteSuppressedServiceIds = new Set( + [...serverlessProxyRoutedServiceIds].filter( + (serviceId) => !serverlessServiceIds.has(serviceId), + ), + ); const routes = buildTraefikRoutes({ serverId: server.id, ports, routableDeployments, serverlessServiceIds, + serverlessRouteSuppressedServiceIds, }); const routedDomains = routes.httpRoutes.map((r) => r.domain); const certificates = await getAllCertificatesForDomains(routedDomains); @@ -611,11 +629,13 @@ export function buildTraefikRoutes({ ports, routableDeployments, serverlessServiceIds = new Set(), + serverlessRouteSuppressedServiceIds = new Set(), }: { serverId: string; ports: ServicePort[]; routableDeployments: RoutableDeploymentRow[]; serverlessServiceIds?: Set; + serverlessRouteSuppressedServiceIds?: Set; }) { const httpRoutes: HttpRoute[] = []; const tcpRoutes: TcpRoute[] = []; @@ -641,6 +661,10 @@ export function buildTraefikRoutes({ continue; } + if (serverlessRouteSuppressedServiceIds.has(port.serviceId)) { + continue; + } + const localDeployments = serviceDeployments.filter( (d) => d.serverId === serverId && d.ipAddress, ); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 571150cb..2277406f 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -188,7 +188,7 @@ describe("expected-state pure builders", () => { }); }); - it("routes serverless HTTP services through the local wake gateway", () => { + it("routes owner proxy serverless HTTP services through the local wake gateway", () => { const routes = buildTraefikRoutes({ serverId: "server_local", ports: [ @@ -215,6 +215,117 @@ describe("expected-state pure builders", () => { ]); }); + it("omits serverless HTTP routes on non-owner proxies", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_2", + ports: [ + { + id: "port_1", + serviceId: "svc_serverless", + port: 3000, + isPublic: true, + protocol: "http", + domain: "sleepy.example.com", + }, + ] as any, + routableDeployments: [], + serverlessRouteSuppressedServiceIds: new Set(["svc_serverless"]), + }); + + expect(routes.httpRoutes).toEqual([]); + }); + + it("keeps worker-only serverless services on direct routes", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports: [ + { + id: "port_1", + serviceId: "svc_worker_only", + port: 3000, + isPublic: true, + protocol: "http", + domain: "worker-only.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_worker_only", + serverId: "worker_1", + ipAddress: "10.0.0.20", + }, + ] as any, + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "worker-only.example.com", + domain: "worker-only.example.com", + serviceId: "svc_worker_only", + upstreams: [{ url: "10.0.0.20:3000", weight: 1 }], + }, + ]); + }); + + it("routes mixed serverless services through the gateway on owner proxies", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports: [ + { + id: "port_1", + serviceId: "svc_mixed", + port: 3000, + isPublic: true, + protocol: "http", + domain: "mixed.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_mixed", + serverId: "worker_1", + ipAddress: "10.0.0.30", + }, + ] as any, + serverlessServiceIds: new Set(["svc_mixed"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "mixed.example.com", + domain: "mixed.example.com", + serviceId: "svc_mixed", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); + + it("does not emit worker-direct fallback routes for non-owner mixed serverless services", () => { + const routes = buildTraefikRoutes({ + serverId: "proxy_2", + ports: [ + { + id: "port_1", + serviceId: "svc_mixed", + port: 3000, + isPublic: true, + protocol: "http", + domain: "mixed.example.com", + }, + ] as any, + routableDeployments: [ + { + serviceId: "svc_mixed", + serverId: "worker_1", + ipAddress: "10.0.0.30", + }, + ] as any, + serverlessRouteSuppressedServiceIds: new Set(["svc_mixed"]), + }); + + expect(routes.httpRoutes).toEqual([]); + }); + it("builds proxy-local serverless metadata with always-on worker upstreams", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", From 1d2d4b0b719abf78c0c257422a89b019a4fc103c Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:21:16 +1000 Subject: [PATCH 16/29] Fix serverless wake coalescing and stop --- agent/internal/serverless/gateway.go | 51 +++++++++++---- agent/internal/serverless/gateway_test.go | 80 +++++++++++++++++++---- web/actions/projects.ts | 14 ++-- 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 07b97a2d..4b1b217b 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -185,11 +185,16 @@ func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.Se return upstreams, nil } - call, owner := g.beginWake(host) + route := findRoute(g.runtime.ExpectedState(), host) + if route == nil { + return nil, fmt.Errorf("no serverless route metadata for host %s", host) + } + wakeKey := routeWakeKey(route) + call, owner := g.beginWake(wakeKey) if owner { go func() { upstreams, err := g.resolveUpstreams(host) - g.finishWake(host, call, upstreams, err) + g.finishWake(wakeKey, call, upstreams, err) }() } @@ -202,7 +207,11 @@ func (g *Gateway) getUpstreams(ctx context.Context, host string) ([]agenthttp.Se if call.err != nil { return nil, call.err } - return cloneUpstreams(call.upstreams), nil + upstreams := cloneUpstreams(call.upstreams) + if len(upstreams) > 0 { + g.cacheUpstreams(host, upstreams) + } + return upstreams, nil } func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, error) { @@ -496,35 +505,38 @@ func (g *Gateway) cachedUpstreams(host string) ([]agenthttp.ServerlessUpstream, return cloneUpstreams(cached.upstreams), true } -func (g *Gateway) beginWake(host string) (*wakeCall, bool) { +func (g *Gateway) beginWake(wakeKey string) (*wakeCall, bool) { g.mu.Lock() defer g.mu.Unlock() - if call, ok := g.wakeCalls[host]; ok { + if call, ok := g.wakeCalls[wakeKey]; ok { return call, false } call := &wakeCall{done: make(chan struct{})} - g.wakeCalls[host] = call + g.wakeCalls[wakeKey] = call return call, true } -func (g *Gateway) finishWake(host string, call *wakeCall, upstreams []agenthttp.ServerlessUpstream, err error) { +func (g *Gateway) finishWake(wakeKey string, call *wakeCall, upstreams []agenthttp.ServerlessUpstream, err error) { g.mu.Lock() defer g.mu.Unlock() call.upstreams = cloneUpstreams(upstreams) call.err = err - if err == nil && len(upstreams) > 0 { - g.upstreamCache[host] = cachedUpstreams{ - upstreams: cloneUpstreams(upstreams), - expiresAt: time.Now().Add(upstreamCacheTTL), - } - } - delete(g.wakeCalls, host) + delete(g.wakeCalls, wakeKey) close(call.done) } +func (g *Gateway) cacheUpstreams(host string, upstreams []agenthttp.ServerlessUpstream) { + g.mu.Lock() + defer g.mu.Unlock() + g.upstreamCache[host] = cachedUpstreams{ + upstreams: cloneUpstreams(upstreams), + expiresAt: time.Now().Add(upstreamCacheTTL), + } +} + func (g *Gateway) evictUpstreams(host string) { g.mu.Lock() defer g.mu.Unlock() @@ -830,6 +842,17 @@ func localDeploymentIDsForService(state *agenthttp.ExpectedState, serviceID stri return deploymentIDs } +func routeWakeKey(route *agenthttp.ServerlessRoute) string { + deploymentIDs := append([]string(nil), route.LocalDeploymentIDs...) + sort.Strings(deploymentIDs) + return fmt.Sprintf( + "%s:%d:%s", + route.ServiceID, + route.Port, + strings.Join(deploymentIDs, ","), + ) +} + func expectedContainersByDeploymentID(state *agenthttp.ExpectedState) map[string]agenthttp.ExpectedContainer { containersByDeploymentID := map[string]agenthttp.ExpectedContainer{} if state == nil { diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index 3b2a9fcb..976087c5 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -12,15 +12,18 @@ import ( ) type fakeRuntime struct { - mu sync.Mutex - state *agenthttp.ExpectedState - containers []container.Container - transitions []agenthttp.ServerlessTransition - removed []string - deployCalls int - deployErr error - afterList func() - healthStatus string + mu sync.Mutex + state *agenthttp.ExpectedState + containers []container.Container + transitions []agenthttp.ServerlessTransition + removed []string + deployCalls int + deployErr error + afterList func() + healthStatus string + deployStarted chan struct{} + deployStartedOnce sync.Once + allowDeploy chan struct{} } func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { @@ -30,6 +33,14 @@ func (f *fakeRuntime) ExpectedState() *agenthttp.ExpectedState { } func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { + if f.deployStarted != nil { + f.deployStartedOnce.Do(func() { + close(f.deployStarted) + }) + } + if f.allowDeploy != nil { + <-f.allowDeploy + } f.mu.Lock() defer f.mu.Unlock() f.deployCalls += 1 @@ -90,18 +101,63 @@ func (f *fakeRuntime) snapshot() ([]agenthttp.ServerlessTransition, []string, in } func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { - g := NewGateway(&fakeRuntime{}) - g.wakeCalls["sleepy.example.com"] = &wakeCall{done: make(chan struct{})} + state := testExpectedState("stopped") + g := NewGateway(&fakeRuntime{state: state}) + g.wakeCalls[routeWakeKey(&state.Serverless.Routes[0])] = &wakeCall{done: make(chan struct{})} ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := g.getUpstreams(ctx, "sleepy.example.com") + _, err := g.getUpstreams(ctx, "app.example.com") if !errors.Is(err, context.Canceled) { t.Fatalf("expected context.Canceled, got %v", err) } } +func TestConcurrentDomainsShareOneWakeAttempt(t *testing.T) { + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + secondRoute := state.Serverless.Routes[0] + secondRoute.Domain = "api.example.com" + state.Serverless.Routes = append(state.Serverless.Routes, secondRoute) + runtime := &fakeRuntime{ + state: state, + deployStarted: make(chan struct{}), + allowDeploy: make(chan struct{}), + } + gateway := NewGateway(runtime) + + errs := make(chan error, 2) + go func() { + _, err := gateway.getUpstreams(context.Background(), "app.example.com") + errs <- err + }() + select { + case <-runtime.deployStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first wake to start") + } + go func() { + _, err := gateway.getUpstreams(context.Background(), "api.example.com") + errs <- err + }() + time.Sleep(10 * time.Millisecond) + close(runtime.allowDeploy) + + for i := 0; i < 2; i++ { + if err := <-errs; err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + } + transitions, _, deployCalls := runtime.snapshot() + if deployCalls != 1 { + t.Fatalf("deployCalls = %d, want 1", deployCalls) + } + if len(transitions) != 1 || transitions[0].Type != "wake_started" { + t.Fatalf("transitions = %+v, want one wake_started", transitions) + } +} + func TestRequestCanUseAlwaysOnWorkerWhileLocalDeploymentWakes(t *testing.T) { runtime := &fakeRuntime{state: testExpectedState("stopped")} gateway := NewGateway(runtime) diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 46776037..c82faa1a 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1264,31 +1264,31 @@ export async function updateServiceConfig( export async function stopService(serviceId: string) { await requireDeveloperRole(); - const runningDeployments = await db + const desiredDeployments = await db .select() .from(deployments) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + eq(deployments.desired, true), ), ); - for (const dep of runningDeployments) { - if (!dep.containerId) continue; - + for (const dep of desiredDeployments) { + const nextStatus = dep.containerId ? "stopping" : "stopped"; await db .update(deployments) - .set(markDeploymentUndesired("stopping")) + .set(markDeploymentUndesired(nextStatus)) .where(eq(deployments.id, dep.id)); + if (!dep.containerId) continue; await enqueueWork(dep.serverId, "stop", { deploymentId: dep.id, containerId: dep.containerId, }); } - return { success: true, count: runningDeployments.length }; + return { success: true, count: desiredDeployments.length }; } export async function restartService(serviceId: string) { From 3fbd9a48d521221638773791c3f5278b20710d5a Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:27:39 +1000 Subject: [PATCH 17/29] Prevent wake failure from reattaching containers --- web/lib/agent-status.ts | 11 +++++------ web/tests/agent-status.test.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 web/tests/agent-status.test.ts diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index fb357f4b..873b1184 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -42,6 +42,10 @@ export type ServerlessTransition = | { type: "wake_started"; deploymentId: string } | { type: "wake_failed"; deploymentId: string; error: string }; +export function shouldAttachReportedContainer(status: string) { + return status === "pending" || status === "pulling" || status === "waking"; +} + function isMigrationTargetStarting(status: string | null | undefined) { return status === "deploying_target" || status === "starting"; } @@ -650,12 +654,7 @@ export async function applyStatusReport( updateFields.containerId = container.containerId; } - if ( - deployment.status === "pending" || - deployment.status === "pulling" || - deployment.status === "waking" || - deployment.status === "sleeping" - ) { + if (shouldAttachReportedContainer(deployment.status)) { if (container.status !== "running") { continue; } diff --git a/web/tests/agent-status.test.ts b/web/tests/agent-status.test.ts new file mode 100644 index 00000000..16da17e6 --- /dev/null +++ b/web/tests/agent-status.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/lib/inngest/client", () => ({ + inngest: { send: vi.fn() }, +})); +vi.mock("@/lib/inngest/events", () => ({ + inngestEvents: { + resourceStatusChanged: { create: vi.fn((payload) => payload) }, + serverDnsSynced: { create: vi.fn((payload) => payload) }, + }, +})); +vi.mock("@/lib/victoria-logs", () => ({ + ingestRolloutLog: vi.fn(), +})); +vi.mock("@/lib/work-queue", () => ({ + enqueueWork: vi.fn(), +})); + +import { shouldAttachReportedContainer } from "@/lib/agent-status"; + +describe("agent status serverless attachment", () => { + it("does not attach reported containers to sleeping deployments", () => { + expect(shouldAttachReportedContainer("pending")).toBe(true); + expect(shouldAttachReportedContainer("pulling")).toBe(true); + expect(shouldAttachReportedContainer("waking")).toBe(true); + expect(shouldAttachReportedContainer("sleeping")).toBe(false); + expect(shouldAttachReportedContainer("failed")).toBe(false); + }); +}); From fa7bf81407ba1ef7c2434a5f325e3b12c4b9babd Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:30:08 +1000 Subject: [PATCH 18/29] Remove stale serverless wake plumbing --- agent/internal/agent/workqueue.go | 2 +- web/db/schema.ts | 8 -------- web/lib/agent-status.ts | 4 ---- web/lib/serverless-wake-failures.ts | 1 - 4 files changed, 1 insertion(+), 14 deletions(-) diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go index 5302d4af..77aead2b 100644 --- a/agent/internal/agent/workqueue.go +++ b/agent/internal/agent/workqueue.go @@ -125,7 +125,7 @@ func (a *Agent) ProcessWorkItem(item agenthttp.WorkQueueItem) error { return a.ProcessRestart(item) case "stop": return a.ProcessStop(item) - case "deploy", "reconcile", "wake": + case "deploy", "reconcile": a.RequestReconcile("reconcile work item " + Truncate(item.ID, 8)) return nil case "force_cleanup": diff --git a/web/db/schema.ts b/web/db/schema.ts index a9958c13..29f96e8c 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -567,9 +567,6 @@ export const deployments = pgTable( rolloutId: text("rollout_id"), previousDeploymentId: text("previous_deployment_id"), failedStage: text("failed_stage"), - serverlessWakeStartedAt: timestamp("serverless_wake_started_at", { - withTimezone: true, - }), serverlessWakeFailureCount: integer("serverless_wake_failure_count") .notNull() .default(0), @@ -583,9 +580,6 @@ export const deployments = pgTable( index("deployments_service_id_idx").on(table.serviceId), index("deployments_server_id_idx").on(table.serverId), index("deployments_status_idx").on(table.status), - index("deployments_serverless_wake_started_at_idx").on( - table.serverlessWakeStartedAt, - ), ], ); @@ -643,8 +637,6 @@ export const workQueue = pgTable( "backup_volume", "restore_volume", "create_manifest", - "sleep", - "wake", "upgrade_agent", ], }).notNull(), diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index 873b1184..a54ae398 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -231,7 +231,6 @@ async function applyServerlessTransitions( status: "sleeping", containerId: null, healthStatus: null, - serverlessWakeStartedAt: null, failedStage: null, }) .where( @@ -264,7 +263,6 @@ async function applyServerlessTransitions( status: "waking", containerId: null, healthStatus: null, - serverlessWakeStartedAt: new Date(), failedStage: null, }) .where( @@ -607,7 +605,6 @@ export async function applyStatusReport( containerId: container.containerId, status: newStatus, healthStatus: hasHealthCheck ? "starting" : "none", - serverlessWakeStartedAt: null, serverlessWakeFailureCount: 0, }) .where(eq(deployments.id, stuckDeployment.id)); @@ -669,7 +666,6 @@ export async function applyStatusReport( const newStatus = hasHealthCheck ? "starting" : "healthy"; updateFields.status = newStatus; if (deployment.status === "waking") { - updateFields.serverlessWakeStartedAt = null; updateFields.serverlessWakeFailureCount = 0; } if (hasHealthCheck) { diff --git a/web/lib/serverless-wake-failures.ts b/web/lib/serverless-wake-failures.ts index 4f38947f..646066d0 100644 --- a/web/lib/serverless-wake-failures.ts +++ b/web/lib/serverless-wake-failures.ts @@ -15,7 +15,6 @@ export function getServerlessWakeFailureUpdate({ const baseUpdate = { containerId: null, healthStatus: null, - serverlessWakeStartedAt: null, serverlessWakeFailureCount: nextFailureCount, }; From 991837b37a45def26a56e53a5075d5403fb2aaa5 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:42:13 +1000 Subject: [PATCH 19/29] Preserve sleeping serverless rollouts --- web/lib/agent/expected-state.ts | 4 +- web/lib/inngest/functions/rollout-helpers.ts | 22 ++++- web/lib/inngest/functions/rollout-workflow.ts | 31 ++++++- web/tests/expected-state.test.ts | 84 +++++++++++++++++++ web/tests/rollout-helpers.test.ts | 48 +++++++++++ 5 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 web/tests/rollout-helpers.test.ts diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 6e7abc94..1ff9c456 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -315,7 +315,8 @@ export function buildExpectedContainersFromRows({ serverIsProxy && service.serverlessEnabled && sleepableServiceIds.has(service.id) && - dep.status === "sleeping" + (dep.status === "sleeping" || + (dep.status === "draining" && !dep.containerId)) ? "stopped" : "running", image: normalizeImage(service.image), @@ -440,6 +441,7 @@ export function buildServerlessRoutesFromRows({ (deployment) => deployment.serverId === serverId && deployment.serverIsProxy && + deployment.status !== "draining" && expectedDeploymentIds.has(deployment.id), ) .map((deployment) => deployment.id) diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 055c7dc9..ac314ca2 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -33,6 +33,19 @@ export type DeploymentContext = { isRollingUpdate: boolean; }; +export function isActiveDeploymentForRollout( + deployment: { status: string }, + service: { serverlessEnabled?: boolean | null }, +) { + if (deployment.status === "running" || deployment.status === "healthy") { + return true; + } + return ( + service.serverlessEnabled === true && + (deployment.status === "sleeping" || deployment.status === "waking") + ); +} + export function normalizeImage(image: string): string { if (!image.includes("/")) { return `docker.io/library/${image}`; @@ -167,13 +180,18 @@ export async function validateServers( export async function prepareRollingUpdate( serviceId: string, ): Promise<{ deploymentIds: string[] }> { + const service = await getService(serviceId); + if (!service) { + return { deploymentIds: [] }; + } + const existingDeployments = await db .select() .from(deployments) .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter( - (d) => d.status === "running" || d.status === "healthy", + (d) => isActiveDeploymentForRollout(d, service), ); return { deploymentIds: runningDeployments.map((d) => d.id) }; @@ -464,7 +482,7 @@ export async function checkForRollingUpdate( .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter( - (d) => d.status === "running" || d.status === "healthy", + (d) => isActiveDeploymentForRollout(d, service), ); return runningDeployments.length > 0; diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index b57cc455..6697b9a1 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -505,6 +505,14 @@ export const rolloutWorkflow = inngest.createFunction( } await step.run("start-dns-sync", async () => { + const service = await getService(serviceId); + if (!service) { + throw new Error("Service not found"); + } + const oldActiveStatuses = service.serverlessEnabled + ? (["running", "healthy", "sleeping", "waking"] as const) + : (["running", "healthy"] as const); + await db .update(rollouts) .set({ currentStage: "dns_sync" }) @@ -526,7 +534,7 @@ export const rolloutWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), + inArray(deployments.status, oldActiveStatuses), or( ne(deployments.rolloutId, rolloutId), isNull(deployments.rolloutId), @@ -598,7 +606,19 @@ export const rolloutWorkflow = inngest.createFunction( if (isRollingUpdate) { await step.run("stop-old-deployments", async () => { - const stoppedDeployments = await db + const stoppedDeploymentsWithoutContainers = await db + .update(deployments) + .set({ status: "stopped", desired: false }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "draining"), + isNull(deployments.containerId), + ), + ) + .returning({ id: deployments.id }); + + const stoppingDeployments = await db .update(deployments) .set({ status: "stopping", desired: false }) .where( @@ -609,12 +629,15 @@ export const rolloutWorkflow = inngest.createFunction( ) .returning({ id: deployments.id }); - if (stoppedDeployments.length > 0) { + const stoppedCount = + stoppedDeploymentsWithoutContainers.length + + stoppingDeployments.length; + if (stoppedCount > 0) { await ingestRolloutLog( rolloutId, serviceId, "dns_sync", - `Stopping ${stoppedDeployments.length} old deployment(s) after DNS sync`, + `Stopping ${stoppedCount} old deployment(s) after DNS sync`, ); } }); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 2277406f..3b84f632 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -188,6 +188,37 @@ describe("expected-state pure builders", () => { }); }); + it("keeps drained serverless deployments without containers stopped", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_draining", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + status: "draining", + containerId: null, + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: true, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + serverlessRoutableServiceIds: new Set(["svc_public"]), + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_draining", + desiredState: "stopped", + }); + }); + it("routes owner proxy serverless HTTP services through the local wake gateway", () => { const routes = buildTraefikRoutes({ serverId: "server_local", @@ -397,6 +428,59 @@ describe("expected-state pure builders", () => { ]); }); + it("does not include draining serverless deployments as wakeable local deployments", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: true, + stateful: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_1", + port: 3000, + isPublic: true, + protocol: "http", + domain: "app.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_old", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + status: "draining", + serverIsProxy: true, + }, + { + id: "dep_new", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.11", + status: "running", + serverIsProxy: true, + }, + ] as any, + containers: [ + { deploymentId: "dep_old", desiredState: "stopped" }, + { deploymentId: "dep_new", desiredState: "running" }, + ] as any, + }); + + expect(routes[0]?.localDeploymentIds).toEqual(["dep_new"]); + expect(routes[0]?.upstreams.map((upstream) => upstream.deploymentId)).toEqual([ + "dep_new", + ]); + }); + it("omits serverless metadata for worker-only services", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts new file mode 100644 index 00000000..8d0bae47 --- /dev/null +++ b/web/tests/rollout-helpers.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/db/queries", () => ({ getService: vi.fn() })); +vi.mock("@/lib/acme-manager", () => ({ + getCertificate: vi.fn(), + issueCertificate: vi.fn(), +})); +vi.mock("@/lib/service-config", () => ({ + buildCurrentConfig: vi.fn(), +})); +vi.mock("@/lib/wireguard", () => ({ + assignContainerIp: vi.fn(), +})); +vi.mock("@/lib/work-queue", () => ({ + enqueueWork: vi.fn(), +})); + +import { isActiveDeploymentForRollout } from "@/lib/inngest/functions/rollout-helpers"; + +describe("rollout helpers", () => { + it("treats sleeping and waking serverless deployments as active rollout versions", () => { + expect( + isActiveDeploymentForRollout( + { status: "sleeping" }, + { serverlessEnabled: true }, + ), + ).toBe(true); + expect( + isActiveDeploymentForRollout( + { status: "waking" }, + { serverlessEnabled: true }, + ), + ).toBe(true); + expect( + isActiveDeploymentForRollout( + { status: "sleeping" }, + { serverlessEnabled: false }, + ), + ).toBe(false); + expect( + isActiveDeploymentForRollout( + { status: "healthy" }, + { serverlessEnabled: false }, + ), + ).toBe(true); + }); +}); From ebba5df92eaebbedb9ef8d365e959e4372ab390e Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:08:25 +1000 Subject: [PATCH 20/29] Make serverless settings deploy-time config --- .../service/details/serverless-section.tsx | 2 +- .../service/service-layout-client.tsx | 2 + web/lib/agent-status.ts | 28 ++- web/lib/agent/expected-state.ts | 165 ++++++++++++----- web/lib/inngest/functions/rollout-helpers.ts | 21 ++- web/lib/inngest/functions/rollout-workflow.ts | 37 ++-- web/lib/service-config.ts | 172 +++++++++++++++++- web/tests/expected-state.test.ts | 127 +++++++++++++ web/tests/rollout-helpers.test.ts | 43 ++++- web/tests/service-config.test.ts | 68 +++++++ 10 files changed, 588 insertions(+), 77 deletions(-) create mode 100644 web/tests/service-config.test.ts diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index a3c755c6..47919fd1 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -95,7 +95,7 @@ export const ServerlessSection = memo(function ServerlessSection({ minReadyReplicas: parsed.minReadyReplicas, }); onUpdate(); - toast.success("Serverless settings updated"); + toast.success("Serverless settings saved. Deploy to apply."); } catch (error) { toast.error( error instanceof Error diff --git a/web/components/service/service-layout-client.tsx b/web/components/service/service-layout-client.tsx index e81946e6..7af810ba 100644 --- a/web/components/service/service-layout-client.tsx +++ b/web/components/service/service-layout-client.tsx @@ -87,6 +87,8 @@ export function ServiceLayoutClient({ port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol, + tlsPassthrough: p.tlsPassthrough, })); const current = buildCurrentConfig( service, diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index a54ae398..bd492f2b 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -22,6 +22,10 @@ import { markDeploymentUndesired } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; +import { + getDeployedServerlessConfig, + getDeployedStateful, +} from "@/lib/service-config"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueWork } from "@/lib/work-queue"; @@ -92,6 +96,11 @@ async function applyDeploymentErrors( status: deployments.status, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, serverlessEnabled: services.serverlessEnabled, + serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, + serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, + serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, + stateful: services.stateful, + deployedConfig: services.deployedConfig, rolloutStatus: rollouts.status, serverName: servers.name, }) @@ -126,7 +135,8 @@ async function applyDeploymentErrors( .set( isServerlessWakeDeployment ? getServerlessWakeFailureUpdate({ - serverlessEnabled: deployment.serverlessEnabled, + serverlessEnabled: + getDeployedServerlessConfig(deployment).enabled, currentFailureCount: deployment.serverlessWakeFailureCount, failedStage: "serverless_wake", }) @@ -202,7 +212,11 @@ async function applyServerlessTransitions( desired: deployments.desired, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, serverlessEnabled: services.serverlessEnabled, + serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, + serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, + serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, stateful: services.stateful, + deployedConfig: services.deployedConfig, serverIsProxy: servers.isProxy, serverName: servers.name, }) @@ -291,7 +305,7 @@ async function applyServerlessTransitions( .update(deployments) .set( getServerlessWakeFailureUpdate({ - serverlessEnabled: deployment.serverlessEnabled, + serverlessEnabled: getDeployedServerlessConfig(deployment).enabled, currentFailureCount: deployment.serverlessWakeFailureCount, failedStage: "serverless_wake", }), @@ -352,7 +366,11 @@ function getInvalidServerlessTransitionReason({ status: string; desired: boolean; serverlessEnabled: boolean; + serverlessSleepAfterSeconds: number; + serverlessWakeTimeoutSeconds: number; + serverlessMinReadyReplicas: number; stateful: boolean; + deployedConfig: string | null; serverIsProxy: boolean; } | undefined; @@ -360,8 +378,10 @@ function getInvalidServerlessTransitionReason({ if (!deployment) return "deployment not found"; if (deployment.serverId !== serverId) return "deployment belongs to another server"; if (!deployment.serverIsProxy) return "server is not a proxy"; - if (!deployment.serverlessEnabled) return "service is not serverless"; - if (deployment.stateful) return "service is stateful"; + if (!getDeployedServerlessConfig(deployment).enabled) { + return "service is not serverless"; + } + if (getDeployedStateful(deployment)) return "service is stateful"; if (!deployment.desired) return "deployment is not desired"; if (transition.type === "sleep") { diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 1ff9c456..e6eb40eb 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNotNull, isNull } from "drizzle-orm"; +import { and, eq, inArray, isNull } from "drizzle-orm"; import { db } from "@/db"; import { deploymentPorts, @@ -15,6 +15,11 @@ import { expectedDeploymentStatuses, routableDeploymentStatuses, } from "@/lib/deployment-status"; +import { + getDeployedServerlessConfig, + getDeployedServicePorts, + isDeployedServerlessService, +} from "@/lib/service-config"; import { slugify } from "@/lib/utils"; import { getWireGuardPeers } from "@/lib/wireguard"; @@ -24,6 +29,18 @@ type Deployment = typeof deployments.$inferSelect; type ServicePort = typeof servicePorts.$inferSelect; const SERVERLESS_GATEWAY_PORT = 18080; +type RouteServicePort = Pick< + ServicePort, + | "id" + | "serviceId" + | "port" + | "isPublic" + | "domain" + | "protocol" + | "externalPort" + | "tlsPassthrough" +>; + export type ExpectedContainer = { deploymentId: string; serviceId: string; @@ -191,8 +208,8 @@ async function buildExpectedContainers( const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); if (serviceIds.length === 0) return []; - const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all( - [ + const [activeServices, depPorts, serviceSecrets, volumes, servicePortRows] = + await Promise.all([ db .select() .from(services) @@ -216,10 +233,12 @@ async function buildExpectedContainers( }) .from(serviceVolumes) .where(inArray(serviceVolumes.serviceId, serviceIds)), - ], + fetchServicePorts(serviceIds), + ]); + const serverlessRoutableServiceIds = getServerlessRoutableServiceIds( + activeServices, + servicePortRows, ); - const serverlessRoutableServiceIds = - await fetchPublicHttpServiceIds(serviceIds); return buildExpectedContainersFromRows({ deployments: serverDeployments, @@ -246,22 +265,13 @@ async function fetchDeploymentPorts(deploymentIds: string[]) { .where(inArray(deploymentPorts.deploymentId, deploymentIds)); } -async function fetchPublicHttpServiceIds(serviceIds: string[]) { - if (serviceIds.length === 0) return new Set(); +async function fetchServicePorts(serviceIds: string[]) { + if (serviceIds.length === 0) return []; - const rows = await db - .select({ serviceId: servicePorts.serviceId }) + return db + .select() .from(servicePorts) - .where( - and( - inArray(servicePorts.serviceId, serviceIds), - eq(servicePorts.isPublic, true), - eq(servicePorts.protocol, "http"), - isNotNull(servicePorts.domain), - ), - ); - - return new Set(rows.map((row) => row.serviceId)); + .where(inArray(servicePorts.serviceId, serviceIds)); } export function buildExpectedContainersFromRows({ @@ -288,7 +298,7 @@ export function buildExpectedContainersFromRows({ serverlessRoutableServiceIds ?? new Set( serviceRows - .filter((service) => service.serverlessEnabled) + .filter((service) => isDeployedServerlessService(service)) .map((service) => service.id), ); const portsByDeploymentId = groupBy( @@ -313,7 +323,7 @@ export function buildExpectedContainersFromRows({ name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, desiredState: serverIsProxy && - service.serverlessEnabled && + isDeployedServerlessService(service) && sleepableServiceIds.has(service.id) && (dep.status === "sleeping" || (dep.status === "draining" && !dep.containerId)) @@ -354,9 +364,7 @@ async function buildServerlessExpectedState( return { routes: [] }; } - const serverlessServices = allServices.filter( - (service) => service.serverlessEnabled && !service.stateful, - ); + const serverlessServices = allServices.filter(isDeployedServerlessService); if (serverlessServices.length === 0) return { routes: [] }; const serviceIds = serverlessServices.map((service) => service.id); @@ -405,33 +413,33 @@ export function buildServerlessRoutesFromRows({ }: { serverId: string; services: Service[]; - ports: ServicePort[]; + ports: RouteServicePort[]; deployments: ServerlessDeploymentRow[]; containers: ExpectedContainer[]; }): ServerlessRoute[] { - const servicesById = new Map( - serviceRows.map((service) => [service.id, service]), - ); const deploymentsByServiceId = groupBy( deploymentRows, (deployment) => deployment.serviceId, ); + const portsByServiceId = groupBy(ports, (port) => port.serviceId); const expectedDeploymentIds = new Set( containers.map((container) => container.deploymentId), ); - return ports - .slice() - .sort(compareServicePorts) - .flatMap((port) => { + return serviceRows + .flatMap((service) => + getRuntimeServicePortsForRoutes( + service, + portsByServiceId.get(service.id) ?? [], + ).map((port) => ({ service, port })), + ) + .sort((a, b) => compareServicePorts(a.port, b.port)) + .flatMap(({ service, port }) => { if (!port.isPublic || port.protocol !== "http" || !port.domain) { return []; } - const service = servicesById.get(port.serviceId); - if (!service) return []; - - const serviceDeployments = deploymentsByServiceId.get(port.serviceId) ?? []; + const serviceDeployments = deploymentsByServiceId.get(service.id) ?? []; if (!serviceDeployments.some((deployment) => deployment.serverIsProxy)) { return []; } @@ -467,9 +475,14 @@ export function buildServerlessRoutesFromRows({ serviceId: service.id, domain: port.domain, port: port.port, - sleepAfterSeconds: service.serverlessSleepAfterSeconds, - wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds, - minReadyReplicas: Math.max(1, service.serverlessMinReadyReplicas ?? 1), + sleepAfterSeconds: + getDeployedServerlessConfig(service).sleepAfterSeconds, + wakeTimeoutSeconds: + getDeployedServerlessConfig(service).wakeTimeoutSeconds, + minReadyReplicas: Math.max( + 1, + getDeployedServerlessConfig(service).minReadyReplicas, + ), localDeploymentIds, upstreams, }, @@ -580,8 +593,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { allServices .filter( (service) => - !service.stateful && - service.serverlessEnabled && + isDeployedServerlessService(service) && proxyHostedServerlessServiceIds.has(service.id), ) .map((service) => service.id), @@ -600,7 +612,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { const routes = buildTraefikRoutes({ serverId: server.id, - ports, + ports: buildRuntimeRoutePorts(allServices, ports), routableDeployments, serverlessServiceIds, serverlessRouteSuppressedServiceIds, @@ -634,7 +646,7 @@ export function buildTraefikRoutes({ serverlessRouteSuppressedServiceIds = new Set(), }: { serverId: string; - ports: ServicePort[]; + ports: RouteServicePort[]; routableDeployments: RoutableDeploymentRow[]; serverlessServiceIds?: Set; serverlessRouteSuppressedServiceIds?: Set; @@ -789,7 +801,70 @@ function groupBy(items: T[], keyFn: (item: T) => K) { return groups; } -function compareServicePorts(a: ServicePort, b: ServicePort) { +export function buildRuntimeRoutePorts( + serviceRows: Service[], + portRows: ServicePort[], +): RouteServicePort[] { + const portsByServiceId = groupBy(portRows, (port) => port.serviceId); + return serviceRows.flatMap((service) => + getRuntimeServicePortsForRoutes( + service, + portsByServiceId.get(service.id) ?? [], + ), + ); +} + +function getRuntimeServicePortsForRoutes( + service: Service, + livePorts: RouteServicePort[], +): RouteServicePort[] { + const ports = isDeployedServerlessService(service) + ? getDeployedServicePorts(service, livePorts) + : livePorts; + return ports.map((port, index) => { + const routePort = port as Partial; + return { + id: + typeof routePort.id === "string" + ? routePort.id + : `${service.id}:deployed:${index}`, + serviceId: service.id, + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? "http", + externalPort: + typeof routePort.externalPort === "number" + ? routePort.externalPort + : null, + tlsPassthrough: routePort.tlsPassthrough ?? false, + }; + }); +} + +function getServerlessRoutableServiceIds( + serviceRows: Service[], + portRows: RouteServicePort[], +) { + return new Set( + serviceRows + .filter((service) => hasDeployedPublicHttpPort(service, portRows)) + .map((service) => service.id), + ); +} + +function hasDeployedPublicHttpPort( + service: Service, + portRows: RouteServicePort[], +) { + if (!isDeployedServerlessService(service)) return false; + const livePorts = portRows.filter((port) => port.serviceId === service.id); + return getRuntimeServicePortsForRoutes(service, livePorts).some( + (port) => port.isPublic && port.protocol === "http" && !!port.domain, + ); +} + +function compareServicePorts(a: RouteServicePort, b: RouteServicePort) { return ( a.serviceId.localeCompare(b.serviceId) || a.protocol.localeCompare(b.protocol) || diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index ac314ca2..a7ef27cf 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -13,7 +13,11 @@ import { serviceVolumes, } from "@/db/schema"; import { getCertificate, issueCertificate } from "@/lib/acme-manager"; -import { buildCurrentConfig } from "@/lib/service-config"; +import { + buildCurrentConfig, + getDeployedStateful, + isDeployedServerlessService, +} from "@/lib/service-config"; import { assignContainerIp } from "@/lib/wireguard"; import { enqueueWork } from "@/lib/work-queue"; @@ -35,13 +39,20 @@ export type DeploymentContext = { export function isActiveDeploymentForRollout( deployment: { status: string }, - service: { serverlessEnabled?: boolean | null }, + service: { + deployedConfig?: string | null; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; + }, ) { if (deployment.status === "running" || deployment.status === "healthy") { return true; } return ( - service.serverlessEnabled === true && + isDeployedServerlessService(service) && (deployment.status === "sleeping" || deployment.status === "waking") ); } @@ -449,6 +460,8 @@ export async function saveDeployedConfig( port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol, + tlsPassthrough: p.tlsPassthrough, })); const updatedService = await getService(serviceId); @@ -472,7 +485,7 @@ export async function checkForRollingUpdate( serviceId: string, ): Promise { const service = await getService(serviceId); - if (!service || service.stateful) { + if (!service || getDeployedStateful(service)) { return false; } diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 6697b9a1..3143f9c6 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -2,6 +2,7 @@ import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; import { deployments, rollouts, servers } from "@/db/schema"; +import { isDeployedServerlessService } from "@/lib/service-config"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -385,23 +386,6 @@ export const rolloutWorkflow = inngest.createFunction( return result; }); - await step.run("save-deployed-config", async () => { - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - - const serverMap = await validateServers(placements); - - await saveDeployedConfig(serviceId, { - service, - placements, - serverMap, - totalReplicas, - isRollingUpdate, - }); - }); - await step.run("start-health-check", async () => { const service = await getService(serviceId); const hasHealthCheck = service?.healthCheckCmd != null; @@ -509,7 +493,7 @@ export const rolloutWorkflow = inngest.createFunction( if (!service) { throw new Error("Service not found"); } - const oldActiveStatuses = service.serverlessEnabled + const oldActiveStatuses = isDeployedServerlessService(service) ? (["running", "healthy", "sleeping", "waking"] as const) : (["running", "healthy"] as const); @@ -643,6 +627,23 @@ export const rolloutWorkflow = inngest.createFunction( }); } + await step.run("save-deployed-config", async () => { + const service = await getService(serviceId); + if (!service) { + throw new Error("Service not found"); + } + + const serverMap = await validateServers(placements); + + await saveDeployedConfig(serviceId, { + service, + placements, + serverMap, + totalReplicas, + isRollingUpdate, + }); + }); + await step.run("complete-rollout", async () => { await db .update(rollouts) diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 7e6aec83..eddc8307 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -44,15 +44,24 @@ export type PlacementConfig = { replicas: number; }; +export type ServerlessConfig = { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + minReadyReplicas: number; +}; + export type DeployedConfig = { source: SourceConfig; hostname?: string; + stateful?: boolean; placement?: PlacementConfig; replicas: ReplicaConfig[]; healthCheck: HealthCheckConfig | null; startCommand?: string | null; resourceLimits?: ResourceLimitsConfig; ports: PortConfig[]; + serverless?: ServerlessConfig; secretKeys?: string[]; secrets?: SecretConfig[]; volumes?: VolumeConfig[]; @@ -77,9 +86,20 @@ export function buildCurrentConfig( resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; replicas: number; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; }, replicas: { serverId: string; serverName: string; count: number }[], - ports: { port: number; isPublic: boolean; domain: string | null }[], + ports: { + port: number; + isPublic: boolean; + domain: string | null; + protocol?: "http" | "tcp" | "udp" | null; + tlsPassthrough?: boolean | null; + }[], secrets?: { key: string; updatedAt: Date | string }[], volumes?: { name: string; containerPath: string }[], ): DeployedConfig { @@ -92,6 +112,7 @@ export function buildCurrentConfig( image: service.image, }, hostname: service.hostname ?? undefined, + stateful: service.stateful ?? false, placement: { replicas: replicas.reduce((sum, r) => sum + r.count, 0), }, @@ -120,7 +141,15 @@ export function buildCurrentConfig( port: p.port, isPublic: p.isPublic, domain: p.domain, + protocol: p.protocol ?? "http", + tlsPassthrough: p.tlsPassthrough ?? undefined, })), + serverless: { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, + minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, + }, secrets: (secrets ?? []).map((s) => ({ key: s.key, updatedAt: @@ -182,6 +211,21 @@ export function diffConfigs( to: `${current.resourceLimits.memoryMb} MB`, }); } + if (current.stateful) { + changes.push({ + field: "Service type", + from: "(not deployed)", + to: "Stateful", + }); + } + const currentServerless = normalizeServerlessConfig(current.serverless); + if (currentServerless.enabled) { + changes.push({ + field: "Serverless", + from: "(not deployed)", + to: "Enabled", + }); + } for (const port of current.ports) { const portType = port.isPublic ? "public" : "internal"; changes.push({ @@ -207,6 +251,56 @@ export function diffConfigs( return changes; } + if ((deployed.stateful ?? false) !== (current.stateful ?? false)) { + changes.push({ + field: "Service type", + from: deployed.stateful ? "Stateful" : "Stateless", + to: current.stateful ? "Stateful" : "Stateless", + }); + } + + const deployedServerless = normalizeServerlessConfig(deployed.serverless); + const currentServerless = normalizeServerlessConfig(current.serverless); + if (deployedServerless.enabled !== currentServerless.enabled) { + changes.push({ + field: "Serverless", + from: deployedServerless.enabled ? "Enabled" : "Disabled", + to: currentServerless.enabled ? "Enabled" : "Disabled", + }); + } + if (deployedServerless.enabled && currentServerless.enabled) { + if ( + deployedServerless.sleepAfterSeconds !== + currentServerless.sleepAfterSeconds + ) { + changes.push({ + field: "Serverless sleep timeout", + from: `${deployedServerless.sleepAfterSeconds}s`, + to: `${currentServerless.sleepAfterSeconds}s`, + }); + } + if ( + deployedServerless.wakeTimeoutSeconds !== + currentServerless.wakeTimeoutSeconds + ) { + changes.push({ + field: "Serverless wake timeout", + from: `${deployedServerless.wakeTimeoutSeconds}s`, + to: `${currentServerless.wakeTimeoutSeconds}s`, + }); + } + if ( + deployedServerless.minReadyReplicas !== + currentServerless.minReadyReplicas + ) { + changes.push({ + field: "Serverless min ready", + from: String(deployedServerless.minReadyReplicas), + to: String(currentServerless.minReadyReplicas), + }); + } + } + if (deployed.source.image !== current.source.image) { changes.push({ field: "Image", @@ -466,3 +560,79 @@ export function parseDeployedConfig( return null; } } + +export function normalizeServerlessConfig( + config: ServerlessConfig | undefined, +): ServerlessConfig { + return { + enabled: config?.enabled ?? false, + sleepAfterSeconds: config?.sleepAfterSeconds ?? 300, + wakeTimeoutSeconds: config?.wakeTimeoutSeconds ?? 300, + minReadyReplicas: config?.minReadyReplicas ?? 1, + }; +} + +export function getCurrentServerlessConfig(service: { + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}): ServerlessConfig { + return { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, + minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, + }; +} + +export function getDeployedServerlessConfig(service: { + deployedConfig?: string | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}): ServerlessConfig { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + return deployed?.serverless + ? normalizeServerlessConfig(deployed.serverless) + : getCurrentServerlessConfig(service); +} + +export function getDeployedStateful(service: { + deployedConfig?: string | null; + stateful?: boolean | null; +}) { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + return deployed?.stateful ?? service.stateful ?? false; +} + +export function isDeployedServerlessService(service: { + deployedConfig?: string | null; + stateful?: boolean | null; + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; + serverlessMinReadyReplicas?: number | null; +}) { + return ( + getDeployedServerlessConfig(service).enabled && !getDeployedStateful(service) + ); +} + +export function getDeployedServicePorts( + service: { id: string; deployedConfig?: string | null }, + livePorts: PortConfig[], +): PortConfig[] { + const deployed = parseDeployedConfig(service.deployedConfig ?? null); + if (!deployed?.ports) { + return livePorts; + } + return deployed.ports.map((port) => ({ + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? "http", + tlsPassthrough: port.tlsPassthrough, + })); +} diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 3b84f632..47b2b80a 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -8,11 +8,33 @@ vi.mock("@/lib/wireguard", () => ({ getWireGuardPeers: vi.fn() })); import { buildExpectedContainersFromRows, + buildRuntimeRoutePorts, buildServerlessRoutesFromRows, buildTraefikRoutes, } from "@/lib/agent/expected-state"; describe("expected-state pure builders", () => { + const deployedServerlessConfig = JSON.stringify({ + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [ + { + port: 3000, + isPublic: true, + domain: "sleepy.example.com", + protocol: "http", + }, + ], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }); + it("groups container inputs by deployment and service deterministically", () => { const containers = buildExpectedContainersFromRows({ deployments: [ @@ -188,6 +210,37 @@ describe("expected-state pure builders", () => { }); }); + it("keeps deployed serverless sleeping while live serverless settings are disabled", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_public", + ipAddress: "10.0.0.10", + status: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_public", + name: "public-api", + image: "nginx", + serverlessEnabled: false, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + serverlessRoutableServiceIds: new Set(["svc_public"]), + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_sleeping", + desiredState: "stopped", + }); + }); + it("keeps drained serverless deployments without containers stopped", () => { const containers = buildExpectedContainersFromRows({ deployments: [ @@ -481,6 +534,80 @@ describe("expected-state pure builders", () => { ]); }); + it("keeps wake metadata from deployed serverless config while live settings are disabled", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_1", + serverlessEnabled: false, + stateful: false, + serverlessSleepAfterSeconds: 60, + serverlessWakeTimeoutSeconds: 60, + serverlessMinReadyReplicas: 1, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + ports: [], + deployments: [ + { + id: "dep_sleeping", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + status: "sleeping", + serverIsProxy: true, + }, + ] as any, + containers: [ + { deploymentId: "dep_sleeping", desiredState: "stopped" }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_1", + domain: "sleepy.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_sleeping"], + upstreams: [], + }, + ]); + }); + + it("keeps Traefik gateway route from deployed serverless ports while live ports are removed", () => { + const ports = buildRuntimeRoutePorts( + [ + { + id: "svc_public", + serverlessEnabled: false, + stateful: false, + deployedConfig: deployedServerlessConfig, + }, + ] as any, + [], + ); + + const routes = buildTraefikRoutes({ + serverId: "proxy_1", + ports, + routableDeployments: [], + serverlessServiceIds: new Set(["svc_public"]), + }); + + expect(routes.httpRoutes).toEqual([ + { + id: "sleepy.example.com", + domain: "sleepy.example.com", + serviceId: "svc_public", + upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], + }, + ]); + }); + it("omits serverless metadata for worker-only services", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts index 8d0bae47..5557060b 100644 --- a/web/tests/rollout-helpers.test.ts +++ b/web/tests/rollout-helpers.test.ts @@ -6,9 +6,6 @@ vi.mock("@/lib/acme-manager", () => ({ getCertificate: vi.fn(), issueCertificate: vi.fn(), })); -vi.mock("@/lib/service-config", () => ({ - buildCurrentConfig: vi.fn(), -})); vi.mock("@/lib/wireguard", () => ({ assignContainerIp: vi.fn(), })); @@ -20,6 +17,20 @@ import { isActiveDeploymentForRollout } from "@/lib/inngest/functions/rollout-he describe("rollout helpers", () => { it("treats sleeping and waking serverless deployments as active rollout versions", () => { + const deployedServerlessConfig = JSON.stringify({ + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }); + expect( isActiveDeploymentForRollout( { status: "sleeping" }, @@ -35,7 +46,31 @@ describe("rollout helpers", () => { expect( isActiveDeploymentForRollout( { status: "sleeping" }, - { serverlessEnabled: false }, + { + serverlessEnabled: false, + deployedConfig: deployedServerlessConfig, + }, + ), + ).toBe(true); + expect( + isActiveDeploymentForRollout( + { status: "sleeping" }, + { + serverlessEnabled: false, + deployedConfig: JSON.stringify({ + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }), + }, ), ).toBe(false); expect( diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts new file mode 100644 index 00000000..991bd109 --- /dev/null +++ b/web/tests/service-config.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + diffConfigs, + getDeployedServerlessConfig, + isDeployedServerlessService, + type DeployedConfig, +} from "@/lib/service-config"; + +function deployedConfig( + overrides: Partial = {}, +): DeployedConfig { + return { + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + ...overrides, + }; +} + +describe("service config", () => { + it("uses deployed serverless settings as the runtime mode", () => { + const service = { + serverlessEnabled: false, + serverlessSleepAfterSeconds: 60, + serverlessWakeTimeoutSeconds: 60, + serverlessMinReadyReplicas: 1, + stateful: true, + deployedConfig: JSON.stringify(deployedConfig()), + }; + + expect(getDeployedServerlessConfig(service)).toMatchObject({ + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }); + expect(isDeployedServerlessService(service)).toBe(true); + }); + + it("reports serverless changes as pending config", () => { + const changes = diffConfigs(deployedConfig(), { + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, + }); + + expect(changes).toContainEqual({ + field: "Serverless", + from: "Enabled", + to: "Disabled", + }); + }); +}); From a184f9d4f7f902240086fb711c4c70228f456467 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:38:28 +1000 Subject: [PATCH 21/29] Fix serverless rollout rollback cleanup --- web/lib/inngest/functions/rollout-utils.ts | 64 +++++++++++++++++---- web/tests/rollout-utils.test.ts | 66 ++++++++++++++++++++++ 2 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 web/tests/rollout-utils.test.ts diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 955bb089..e7a4cb69 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -1,8 +1,35 @@ -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, isNull } from "drizzle-orm"; import { db } from "@/db"; -import { deployments, rollouts } from "@/db/schema"; +import { deployments, rollouts, services } from "@/db/schema"; import { markDeploymentUndesired } from "@/lib/deployment-status"; import { sendDeploymentFailureAlert } from "@/lib/email"; +import { isDeployedServerlessService } from "@/lib/service-config"; + +const ROLLOUT_FAILURE_CLEANUP_STATUSES = [ + "pending", + "pulling", + "starting", + "healthy", + "running", + "sleeping", + "waking", + "failed", +] as const; + +export function shouldRollBackDeploymentStatus(status: string) { + return ROLLOUT_FAILURE_CLEANUP_STATUSES.includes( + status as (typeof ROLLOUT_FAILURE_CLEANUP_STATUSES)[number], + ); +} + +export function shouldRestoreDrainingDeploymentAsSleeping( + deployment: { containerId: string | null }, + service: Parameters[0] | null | undefined, +) { + return ( + !deployment.containerId && !!service && isDeployedServerlessService(service) + ); +} export async function handleRolloutFailure( rolloutId: string, @@ -41,6 +68,30 @@ export async function handleRolloutFailure( const serverId = rolloutDeployments[0].serverId; if (isRollingUpdate) { + const service = await db + .select() + .from(services) + .where(eq(services.id, serviceId)) + .then((rows) => rows[0]); + + if ( + shouldRestoreDrainingDeploymentAsSleeping( + { containerId: null }, + service, + ) + ) { + await db + .update(deployments) + .set({ status: "sleeping", healthStatus: null }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "draining"), + isNull(deployments.containerId), + ), + ); + } + await db .update(deployments) .set({ status: "running" }) @@ -58,14 +109,7 @@ export async function handleRolloutFailure( .where( and( eq(deployments.rolloutId, rolloutId), - inArray(deployments.status, [ - "pending", - "pulling", - "starting", - "healthy", - "running", - "failed", - ]), + inArray(deployments.status, ROLLOUT_FAILURE_CLEANUP_STATUSES), ), ); diff --git a/web/tests/rollout-utils.test.ts b/web/tests/rollout-utils.test.ts new file mode 100644 index 00000000..cc3132d4 --- /dev/null +++ b/web/tests/rollout-utils.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/db", () => ({ db: {} })); +vi.mock("@/lib/email", () => ({ + sendDeploymentFailureAlert: vi.fn(), +})); + +import { + shouldRestoreDrainingDeploymentAsSleeping, + shouldRollBackDeploymentStatus, +} from "@/lib/inngest/functions/rollout-utils"; + +const deployedServerlessConfig = JSON.stringify({ + source: { type: "image", image: "nginx" }, + stateful: false, + replicas: [], + healthCheck: null, + ports: [], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + }, +}); + +describe("rollout failure helpers", () => { + it("cleans up sleeping and waking rollout deployments", () => { + expect(shouldRollBackDeploymentStatus("sleeping")).toBe(true); + expect(shouldRollBackDeploymentStatus("waking")).toBe(true); + expect(shouldRollBackDeploymentStatus("running")).toBe(true); + expect(shouldRollBackDeploymentStatus("draining")).toBe(false); + expect(shouldRollBackDeploymentStatus("stopped")).toBe(false); + }); + + it("restores no-container deployed-serverless drains as sleeping", () => { + const service = { + serverlessEnabled: false, + stateful: true, + deployedConfig: deployedServerlessConfig, + }; + + expect( + shouldRestoreDrainingDeploymentAsSleeping( + { containerId: null }, + service, + ), + ).toBe(true); + expect( + shouldRestoreDrainingDeploymentAsSleeping( + { containerId: "ctr_old" }, + service, + ), + ).toBe(false); + expect( + shouldRestoreDrainingDeploymentAsSleeping( + { containerId: null }, + { + serverlessEnabled: false, + stateful: false, + deployedConfig: null, + }, + ), + ).toBe(false); + }); +}); From 200e51398ef9990d02ed7f63026b2a1e30edd89f Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:08:13 +1000 Subject: [PATCH 22/29] Reuse serverless drain restore on abort --- web/actions/projects.ts | 11 +--- web/lib/inngest/functions/rollout-utils.ts | 67 +++++++++++----------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/web/actions/projects.ts b/web/actions/projects.ts index c82faa1a..4b04c2ac 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -34,6 +34,7 @@ import { deployServiceInternal } from "@/lib/deploy-service"; import { markDeploymentUndesired } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; import { containerPathSchema, @@ -1356,15 +1357,7 @@ export async function abortRollout(serviceId: string) { ); } - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - ), - ); + await restoreDrainingDeploymentsForRollback(serviceId); const rolloutDeployments = activeRolloutIds.length > 0 diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index e7a4cb69..b876661c 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -31,6 +31,39 @@ export function shouldRestoreDrainingDeploymentAsSleeping( ); } +export async function restoreDrainingDeploymentsForRollback(serviceId: string) { + const service = await db + .select() + .from(services) + .where(eq(services.id, serviceId)) + .then((rows) => rows[0]); + + if ( + shouldRestoreDrainingDeploymentAsSleeping({ containerId: null }, service) + ) { + await db + .update(deployments) + .set({ status: "sleeping", healthStatus: null }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "draining"), + isNull(deployments.containerId), + ), + ); + } + + await db + .update(deployments) + .set({ status: "running" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.status, "draining"), + ), + ); +} + export async function handleRolloutFailure( rolloutId: string, serviceId: string, @@ -68,39 +101,7 @@ export async function handleRolloutFailure( const serverId = rolloutDeployments[0].serverId; if (isRollingUpdate) { - const service = await db - .select() - .from(services) - .where(eq(services.id, serviceId)) - .then((rows) => rows[0]); - - if ( - shouldRestoreDrainingDeploymentAsSleeping( - { containerId: null }, - service, - ) - ) { - await db - .update(deployments) - .set({ status: "sleeping", healthStatus: null }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - isNull(deployments.containerId), - ), - ); - } - - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - ), - ); + await restoreDrainingDeploymentsForRollback(serviceId); } await db From 31373ab76cd2a967aad1a189713973a854b49a88 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:46:25 +1000 Subject: [PATCH 23/29] Use stop-start for serverless sleep --- agent/internal/agent/serverless.go | 42 ++++++++++- agent/internal/serverless/gateway.go | 6 +- agent/internal/serverless/gateway_test.go | 89 ++++++++++++++++++----- web/lib/agent-status.ts | 2 - 4 files changed, 111 insertions(+), 28 deletions(-) diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index 9d777eb8..9b033cc5 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -21,7 +21,43 @@ func (a *Agent) ExpectedState() *agenthttp.ExpectedState { } func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) error { - return a.DeployExpectedContainer(expected) + return a.withDeploymentDeployLock(expected.DeploymentID, func() error { + containers, err := container.List() + if err == nil { + for _, actual := range containers { + if actual.DeploymentID != expected.DeploymentID { + continue + } + if normalizeImage(actual.Image) != normalizeImage(expected.Image) { + log.Printf( + "[serverless] recreate deployment %s because image changed (%s -> %s)", + Truncate(expected.DeploymentID, 8), + actual.Image, + expected.Image, + ) + return a.Reconciler.Deploy(expected) + } + if actual.State == "running" { + return nil + } + log.Printf( + "[serverless] starting stopped container %s for deployment %s", + Truncate(actual.ID, 12), + Truncate(expected.DeploymentID, 8), + ) + if err := container.Start(actual.ID); err != nil { + log.Printf( + "[serverless] start failed for deployment %s, recreating: %v", + Truncate(expected.DeploymentID, 8), + err, + ) + return a.Reconciler.Deploy(expected) + } + return nil + } + } + return a.Reconciler.Deploy(expected) + }) } func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) error { @@ -58,8 +94,8 @@ func (a *Agent) withDeploymentDeployLock(deploymentID string, fn func() error) e return fn() } -func (a *Agent) RemoveServerlessContainer(containerID string) error { - return container.ForceRemove(containerID) +func (a *Agent) StopServerlessContainer(containerID string) error { + return container.Stop(containerID) } func (a *Agent) ListServerlessContainers() ([]container.Container, error) { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 4b1b217b..163ddbcc 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -44,7 +44,7 @@ type Gateway struct { type Runtime interface { ExpectedState() *agenthttp.ExpectedState DeployServerlessContainer(agenthttp.ExpectedContainer) error - RemoveServerlessContainer(containerID string) error + StopServerlessContainer(containerID string) error ListServerlessContainers() ([]container.Container, error) GetServerlessContainerHealth(containerID string) string QueueServerlessTransition(agenthttp.ServerlessTransition) @@ -747,13 +747,13 @@ func (g *Gateway) sleepService(serviceID string) { }) } else { log.Printf( - "[serverless-gateway] sleep cleanup starting service=%s deployment=%s container=%s reason=already_expected_stopped", + "[serverless-gateway] sleep stop starting service=%s deployment=%s container=%s reason=already_expected_stopped", serviceID, deploymentID, actual.ID, ) } - if err := g.runtime.RemoveServerlessContainer(actual.ID); err != nil { + if err := g.runtime.StopServerlessContainer(actual.ID); err != nil { activity.mu.Unlock() log.Printf( "[serverless-gateway] sleep failed service=%s deployment=%s container=%s latency=%s error=%v", diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index 976087c5..5bf165b2 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -16,7 +16,7 @@ type fakeRuntime struct { state *agenthttp.ExpectedState containers []container.Container transitions []agenthttp.ServerlessTransition - removed []string + stopped []string deployCalls int deployErr error afterList func() @@ -47,6 +47,12 @@ func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedConta if f.deployErr != nil { return f.deployErr } + for i, actual := range f.containers { + if actual.DeploymentID == expected.DeploymentID { + f.containers[i].State = "running" + return nil + } + } f.containers = append(f.containers, container.Container{ ID: "ctr-" + expected.DeploymentID, State: "running", @@ -56,17 +62,15 @@ func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedConta return nil } -func (f *fakeRuntime) RemoveServerlessContainer(containerID string) error { +func (f *fakeRuntime) StopServerlessContainer(containerID string) error { f.mu.Lock() defer f.mu.Unlock() - f.removed = append(f.removed, containerID) - next := f.containers[:0] - for _, actual := range f.containers { - if actual.ID != containerID { - next = append(next, actual) + f.stopped = append(f.stopped, containerID) + for i, actual := range f.containers { + if actual.ID == containerID { + f.containers[i].State = "exited" } } - f.containers = next return nil } @@ -97,7 +101,13 @@ func (f *fakeRuntime) QueueServerlessTransition(transition agenthttp.ServerlessT func (f *fakeRuntime) snapshot() ([]agenthttp.ServerlessTransition, []string, int) { f.mu.Lock() defer f.mu.Unlock() - return append([]agenthttp.ServerlessTransition(nil), f.transitions...), append([]string(nil), f.removed...), f.deployCalls + return append([]agenthttp.ServerlessTransition(nil), f.transitions...), append([]string(nil), f.stopped...), f.deployCalls +} + +func (f *fakeRuntime) snapshotContainers() []container.Container { + f.mu.Lock() + defer f.mu.Unlock() + return append([]container.Container(nil), f.containers...) } func TestGetUpstreamsReturnsWhenFollowerContextIsCancelled(t *testing.T) { @@ -211,7 +221,46 @@ func TestSleepingLocalDeploymentWakesAndReturnsLocalUpstream(t *testing.T) { } } -func TestSleepHostRemovesLocalContainerAndReportsSleep(t *testing.T) { +func TestSleepingLocalDeploymentStartsExistingStoppedContainer(t *testing.T) { + previousPoll := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { + wakePollInterval = previousPoll + }) + + state := testExpectedState("stopped") + state.Serverless.Routes[0].Upstreams = nil + runtime := &fakeRuntime{ + state: state, + containers: []container.Container{ + { + ID: "ctr-existing", + State: "exited", + DeploymentID: "dep_local", + ServiceID: "svc_1", + }, + }, + } + gateway := NewGateway(runtime) + + upstreams, err := gateway.getUpstreams(context.Background(), "app.example.com") + if err != nil { + t.Fatalf("getUpstreams failed: %v", err) + } + if len(upstreams) != 1 || upstreams[0].Url != "10.0.0.10:3000" { + t.Fatalf("upstreams = %+v, want local upstream", upstreams) + } + + containers := runtime.snapshotContainers() + if len(containers) != 1 { + t.Fatalf("containers = %+v, want existing container only", containers) + } + if containers[0].ID != "ctr-existing" || containers[0].State != "running" { + t.Fatalf("container = %+v, want existing container running", containers[0]) + } +} + +func TestSleepHostStopsLocalContainerAndReportsSleep(t *testing.T) { state := testExpectedState("running") runtime := &fakeRuntime{ state: state, @@ -223,9 +272,9 @@ func TestSleepHostRemovesLocalContainerAndReportsSleep(t *testing.T) { gateway.sleepHost("app.example.com") - transitions, removed, _ := runtime.snapshot() - if len(removed) != 1 || removed[0] != "ctr-local" { - t.Fatalf("removed = %+v, want ctr-local", removed) + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 1 || stopped[0] != "ctr-local" { + t.Fatalf("stopped = %+v, want ctr-local", stopped) } if len(transitions) != 1 { t.Fatalf("transitions = %+v, want one sleep transition", transitions) @@ -235,7 +284,7 @@ func TestSleepHostRemovesLocalContainerAndReportsSleep(t *testing.T) { } } -func TestSleepHostRechecksActivityBeforeRemovingContainer(t *testing.T) { +func TestSleepHostRechecksActivityBeforeStoppingContainer(t *testing.T) { state := testExpectedState("running") runtime := &fakeRuntime{ state: state, @@ -250,9 +299,9 @@ func TestSleepHostRechecksActivityBeforeRemovingContainer(t *testing.T) { gateway.sleepHost("app.example.com") - transitions, removed, _ := runtime.snapshot() - if len(removed) != 0 { - t.Fatalf("removed = %+v, want no removals while request is active", removed) + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 0 { + t.Fatalf("stopped = %+v, want no stops while request is active", stopped) } if len(transitions) != 0 { t.Fatalf("transitions = %+v, want no sleep transition while request is active", transitions) @@ -275,9 +324,9 @@ func TestSleepHostUsesServiceActivityAcrossDomains(t *testing.T) { gateway.sleepHost("api.example.com") - transitions, removed, _ := runtime.snapshot() - if len(removed) != 0 { - t.Fatalf("removed = %+v, want no removals while sibling domain is active", removed) + transitions, stopped, _ := runtime.snapshot() + if len(stopped) != 0 { + t.Fatalf("stopped = %+v, want no stops while sibling domain is active", stopped) } if len(transitions) != 0 { t.Fatalf("transitions = %+v, want no sleep transition while sibling domain is active", transitions) diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index bd492f2b..c97e3149 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -243,7 +243,6 @@ async function applyServerlessTransitions( .update(deployments) .set({ status: "sleeping", - containerId: null, healthStatus: null, failedStage: null, }) @@ -275,7 +274,6 @@ async function applyServerlessTransitions( .update(deployments) .set({ status: "waking", - containerId: null, healthStatus: null, failedStage: null, }) From 2ecc9e019c8b8f8f4f7ee85077f0ed4aee73f41e Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:36:07 +1000 Subject: [PATCH 24/29] Fix serverless gateway capability reporting --- agent/internal/agent/agent.go | 2 ++ agent/internal/agent/reporting.go | 2 +- agent/internal/agent/run.go | 2 ++ agent/internal/agent/serverless_test.go | 19 +++++++++++++++++++ web/actions/projects.ts | 14 ++------------ web/lib/project-deletion.ts | 10 ++++++++++ web/tests/project-deletion.test.ts | 9 +++++++++ 7 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 web/lib/project-deletion.ts create mode 100644 web/tests/project-deletion.test.ts diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index 5871549c..c28e88fa 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "sync" + "sync/atomic" "time" "techulus/cloud-agent/internal/build" @@ -85,6 +86,7 @@ type Agent struct { buildMutex sync.Mutex currentBuildID string IsProxy bool + serverlessGatewayRunning atomic.Bool dnsInSync bool DisableDNS bool } diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index 8eb4e4c9..98c5169b 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -102,7 +102,7 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport } func (a *Agent) agentCapabilities() []string { - if !a.IsProxy { + if !a.IsProxy || !a.serverlessGatewayRunning.Load() { return nil } return []string{serverlessGatewayCapability} diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index ce0a7700..58ead8ae 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -37,6 +37,8 @@ func (a *Agent) Run(ctx context.Context) { gateway := serverless.NewGateway(a) if err := gateway.Start(ctx); err != nil { log.Printf("[serverless-gateway] failed to start: %v", err) + } else { + a.serverlessGatewayRunning.Store(true) } } diff --git a/agent/internal/agent/serverless_test.go b/agent/internal/agent/serverless_test.go index 3662c9e8..683570ec 100644 --- a/agent/internal/agent/serverless_test.go +++ b/agent/internal/agent/serverless_test.go @@ -1,6 +1,7 @@ package agent import ( + "slices" "testing" "techulus/cloud-agent/internal/container" @@ -45,6 +46,24 @@ func TestPendingServerlessWakeDoesNotStopStaleStoppedExpectedContainer(t *testin } } +func TestServerlessGatewayCapabilityRequiresStartedGateway(t *testing.T) { + agent := &Agent{IsProxy: true} + + if slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("proxy reported serverless gateway capability before gateway start") + } + + agent.serverlessGatewayRunning.Store(true) + if !slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("proxy did not report serverless gateway capability after gateway start") + } + + agent.IsProxy = false + if slices.Contains(agent.agentCapabilities(), serverlessGatewayCapability) { + t.Fatal("worker reported serverless gateway capability") + } +} + func TestPendingServerlessWakeDoesNotSuppressContainerReport(t *testing.T) { agent := &Agent{ pendingServerlessSleep: map[string]struct{}{}, diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 4b04c2ac..70eb239e 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -36,6 +36,7 @@ import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; +import { blocksProjectDeletion } from "@/lib/project-deletion"; import { containerPathSchema, githubRepoUrlSchema, @@ -263,24 +264,13 @@ export async function deleteProject(id: string) { .from(services) .where(eq(services.projectId, id)); - const activeStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - "stopping", - ]; - for (const service of projectServices) { const activeDeployments = await db .select() .from(deployments) .where(eq(deployments.serviceId, service.id)); - const hasActiveDeployments = activeDeployments.some((d) => - activeStatuses.includes(d.status), - ); + const hasActiveDeployments = activeDeployments.some(blocksProjectDeletion); if (hasActiveDeployments) { throw new Error( diff --git a/web/lib/project-deletion.ts b/web/lib/project-deletion.ts new file mode 100644 index 00000000..2aa154e8 --- /dev/null +++ b/web/lib/project-deletion.ts @@ -0,0 +1,10 @@ +import type { deployments } from "@/db/schema"; + +type ProjectDeletionDeployment = Pick< + typeof deployments.$inferSelect, + "desired" +>; + +export function blocksProjectDeletion(deployment: ProjectDeletionDeployment) { + return deployment.desired; +} diff --git a/web/tests/project-deletion.test.ts b/web/tests/project-deletion.test.ts new file mode 100644 index 00000000..be247088 --- /dev/null +++ b/web/tests/project-deletion.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from "vitest"; +import { blocksProjectDeletion } from "@/lib/project-deletion"; + +describe("project deletion guard", () => { + it("blocks deletion for desired deployments regardless of lifecycle status", () => { + expect(blocksProjectDeletion({ desired: true })).toBe(true); + expect(blocksProjectDeletion({ desired: false })).toBe(false); + }); +}); From 5b1b83472fd42d65b80d063c989c4c78e0fd9d8c Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:05:51 +1000 Subject: [PATCH 25/29] Allow stateful services to use serverless --- docs/services/scaling.mdx | 6 +- docs/services/volumes.mdx | 2 + web/actions/projects.ts | 3 - .../service/details/serverless-section.tsx | 8 +-- web/lib/agent-status.ts | 7 +-- web/lib/service-config.ts | 4 +- web/tests/expected-state.test.ts | 55 +++++++++++++++++++ web/tests/service-config.test.ts | 13 +++++ 8 files changed, 78 insertions(+), 20 deletions(-) diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 8f4e4432..0c858ffa 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -11,7 +11,7 @@ Replica count ranges from 1 to 10 per service. ## Serverless scaling -Stateless HTTP services can be configured to sleep when idle on proxy nodes. A +Public HTTP services can be configured to sleep when idle on proxy nodes. A sleeping deployment keeps its desired replica record, but the proxy agent removes the local container. The next public HTTP request wakes local proxy-hosted deployments and is held until an upstream is ready or the wake timeout is @@ -31,7 +31,7 @@ Serverless settings are configured per service: | Setting | Default | Description | | --- | --- | --- | -| Enable serverless | Off | Allows stateless HTTP deployments to sleep and wake on request | +| Enable serverless | Off | Allows proxy-hosted HTTP deployments to sleep and wake on request | | Sleep after | `300s` | Idle period before running containers are stopped | | Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | | Min Ready | `1` | Preferred number of ready deployments before queued requests resume | @@ -62,7 +62,7 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services are always pinned to their locked server. - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. -- Serverless scaling is supported only for stateless HTTP services with a public domain. +- Serverless scaling requires a public HTTP service domain. - Sleep and wake are proxy-local. Worker replicas are intentionally always on. - Serverless traffic must be routed only to proxy nodes that own a local proxy replica for that service. - Proxy agents report sleep and wake transitions through normal status reports. diff --git a/docs/services/volumes.mdx b/docs/services/volumes.mdx index ea13e4a5..d516e02e 100644 --- a/docs/services/volumes.mdx +++ b/docs/services/volumes.mdx @@ -18,6 +18,8 @@ Each volume has a name and a container path: When you add a volume, the service automatically becomes **stateful**. Stateful services are locked to a single server and limited to 1 replica so the container always mounts the same local data path. When the last volume is removed, the service reverts to stateless. +Stateful services can use serverless scaling when they have a public HTTP domain. Because volumes are local to one server, only a proxy-hosted stateful replica can sleep and wake on request. A stateful replica placed on a worker node stays always on. + ## Volume Backups Volumes can be backed up to S3-compatible storage on a schedule or on demand. diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 70eb239e..d17858b3 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1035,9 +1035,6 @@ export async function updateServiceServerlessSettings( if (!service || service.deletedAt) { throw new Error("Service not found"); } - if (service.stateful && validated.enabled) { - throw new Error("Serverless is only supported for stateless services"); - } if (validated.enabled) { const publicHttpPorts = await tx diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index 47919fd1..4e2691ee 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -35,11 +35,9 @@ export const ServerlessSection = memo(function ServerlessSection({ ), [service.ports], ); - const unavailableReason = service.stateful - ? "Serverless is only supported for stateless services" - : !hasPublicHttpEndpoint - ? "Add a public HTTP port with a domain to enable serverless" - : null; + const unavailableReason = !hasPublicHttpEndpoint + ? "Add a public HTTP port with a domain to enable serverless" + : null; const optionsDisabled = !!unavailableReason || isSaving; const parsed = useMemo( diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index c97e3149..bf6e483e 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -22,10 +22,7 @@ import { markDeploymentUndesired } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; -import { - getDeployedServerlessConfig, - getDeployedStateful, -} from "@/lib/service-config"; +import { getDeployedServerlessConfig } from "@/lib/service-config"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueWork } from "@/lib/work-queue"; @@ -367,7 +364,6 @@ function getInvalidServerlessTransitionReason({ serverlessSleepAfterSeconds: number; serverlessWakeTimeoutSeconds: number; serverlessMinReadyReplicas: number; - stateful: boolean; deployedConfig: string | null; serverIsProxy: boolean; } @@ -379,7 +375,6 @@ function getInvalidServerlessTransitionReason({ if (!getDeployedServerlessConfig(deployment).enabled) { return "service is not serverless"; } - if (getDeployedStateful(deployment)) return "service is stateful"; if (!deployment.desired) return "deployment is not desired"; if (transition.type === "sleep") { diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index eddc8307..263e34c9 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -615,9 +615,7 @@ export function isDeployedServerlessService(service: { serverlessWakeTimeoutSeconds?: number | null; serverlessMinReadyReplicas?: number | null; }) { - return ( - getDeployedServerlessConfig(service).enabled && !getDeployedStateful(service) - ); + return getDeployedServerlessConfig(service).enabled; } export function getDeployedServicePorts( diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 47b2b80a..ff004bba 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -481,6 +481,61 @@ describe("expected-state pure builders", () => { ]); }); + it("builds proxy-local serverless metadata for stateful services", () => { + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [ + { + id: "svc_stateful", + serverlessEnabled: true, + stateful: true, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + }, + ] as any, + ports: [ + { + id: "port_1", + serviceId: "svc_stateful", + port: 3000, + isPublic: true, + protocol: "http", + domain: "db.example.com", + }, + ] as any, + deployments: [ + { + id: "dep_stateful", + serviceId: "svc_stateful", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + status: "sleeping", + serverIsProxy: true, + }, + ] as any, + containers: [ + { + deploymentId: "dep_stateful", + desiredState: "stopped", + }, + ] as any, + }); + + expect(routes).toEqual([ + { + serviceId: "svc_stateful", + domain: "db.example.com", + port: 3000, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + minReadyReplicas: 1, + localDeploymentIds: ["dep_stateful"], + upstreams: [], + }, + ]); + }); + it("does not include draining serverless deployments as wakeable local deployments", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index 991bd109..5564eabf 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -44,6 +44,19 @@ describe("service config", () => { expect(isDeployedServerlessService(service)).toBe(true); }); + it("allows deployed stateful services to be serverless", () => { + const service = { + serverlessEnabled: true, + stateful: true, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 120, + serverlessMinReadyReplicas: 1, + deployedConfig: JSON.stringify(deployedConfig({ stateful: true })), + }; + + expect(isDeployedServerlessService(service)).toBe(true); + }); + it("reports serverless changes as pending config", () => { const changes = diffConfigs(deployedConfig(), { source: { type: "image", image: "nginx" }, From b79541243888ead89b3dc272267e1ec01fe70e93 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:26:42 +1000 Subject: [PATCH 26/29] Refactor deployments into explicit state fields --- docs/architecture.mdx | 7 +- docs/services/scaling.mdx | 8 +- web/actions/projects.ts | 36 ++-- .../[serviceId]/configuration/page.tsx | 2 +- .../[env]/services/[serviceId]/page.tsx | 6 +- .../service/details/deployment-progress.tsx | 8 +- .../details/service-details-overview.tsx | 7 +- web/components/service/service-canvas.tsx | 4 +- .../service/service-layout-client.tsx | 2 +- web/components/ui/canvas-wrapper.tsx | 23 ++- web/db/queries.ts | 2 +- web/db/schema.ts | 22 ++- web/db/types.ts | 2 +- web/lib/agent-status.ts | 158 ++++++++++++------ web/lib/agent/expected-state.ts | 48 +++--- web/lib/autoheal-policy.ts | 21 ++- web/lib/backups/trigger-backup.ts | 4 +- web/lib/cli-service.ts | 2 +- web/lib/deployment-status.ts | 129 +++++++++----- .../inngest/functions/migration-workflow.ts | 4 +- .../inngest/functions/on-deployment-failed.ts | 4 +- .../functions/restore-trigger-workflow.ts | 4 +- web/lib/inngest/functions/rollout-helpers.ts | 23 +-- web/lib/inngest/functions/rollout-utils.ts | 68 +++----- web/lib/inngest/functions/rollout-workflow.ts | 83 ++++----- .../functions/service-deletion-workflow.ts | 26 ++- web/lib/migrations.ts | 4 +- web/lib/project-deletion.ts | 4 +- web/lib/scheduler.ts | 11 +- web/lib/serverless-wake-failures.ts | 9 +- web/lib/work-queue.ts | 6 +- web/tests/autoheal-policy.test.ts | 20 ++- web/tests/deployment-status.test.ts | 52 ++++-- web/tests/expected-state.test.ts | 142 +++++++++------- web/tests/project-deletion.test.ts | 9 +- web/tests/rollout-helpers.test.ts | 54 +----- web/tests/rollout-utils.test.ts | 81 ++++----- web/tests/serverless-wake-failures.test.ts | 14 +- 38 files changed, 598 insertions(+), 511 deletions(-) diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 569a7bba..52ea851e 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -205,15 +205,16 @@ Worker nodes do not run Traefik. ### Serverless Containers -Stateless HTTP services can be configured as serverless. Serverless scale-to-zero -is proxy-local: deployments placed on proxy nodes may sleep after an idle period, +Public HTTP services can be configured as serverless. Serverless scale-to-zero is +proxy-local: deployments placed on proxy nodes may sleep after an idle period, then wake on the next public request handled by that proxy. Deployments placed on worker nodes stay always on and remain routable. Serverless uses the same declarative expected-state model as normal deployments, but proxy agents own the local lifecycle decision: -- A sleeping deployment remains `desired: true`. +- A sleeping deployment keeps `trafficState: "active"` but records + `runtimeDesiredState: "stopped"` and `observedPhase: "sleeping"`. - Expected state advertises proxy-hosted sleeping containers with `desiredState: "stopped"` so normal reconciliation does not restart them. - Expected state advertises worker-hosted deployments with diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 0c858ffa..46aa96ec 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -12,10 +12,10 @@ Replica count ranges from 1 to 10 per service. ## Serverless scaling Public HTTP services can be configured to sleep when idle on proxy nodes. A -sleeping deployment keeps its desired replica record, but the proxy agent removes -the local container. The next public HTTP request wakes local proxy-hosted -deployments and is held until an upstream is ready or the wake timeout is -reached. +sleeping deployment keeps active traffic intent, records stopped runtime intent, +and stops the local container. The next public HTTP request wakes local +proxy-hosted deployments and is held until an upstream is ready or the wake +timeout is reached. Worker-hosted deployments do not sleep. If a serverless-enabled service has only worker replicas, it stays always on and routes directly. If a service has both diff --git a/web/actions/projects.ts b/web/actions/projects.ts index d17858b3..ed695da2 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -31,7 +31,11 @@ import { import { requireDeveloperRole } from "@/lib/auth"; import { DEFAULT_RESOURCE_LIMITS } from "@/lib/constants"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { + isObservedReady, + markDeploymentRemoved, + runtimeExpectedStates, +} from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; @@ -527,15 +531,7 @@ async function hardDeleteService(serviceId: string) { .where(eq(deployments.serviceId, serviceId)); for (const dep of allDeployments) { - if ( - (dep.status === "running" || dep.status === "healthy") && - dep.containerId - ) { - await db - .update(deployments) - .set(markDeploymentUndesired("stopping")) - .where(eq(deployments.id, dep.id)); - + if (isObservedReady(dep.observedPhase) && dep.containerId) { await enqueueWork(dep.serverId, "stop", { deploymentId: dep.id, containerId: dep.containerId, @@ -618,7 +614,7 @@ export async function deleteService(serviceId: string) { .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), + inArray(deployments.observedPhase, ["running", "healthy"]), ), ) .then((r) => r[0]); @@ -1258,15 +1254,14 @@ export async function stopService(serviceId: string) { .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.desired, true), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), ); for (const dep of desiredDeployments) { - const nextStatus = dep.containerId ? "stopping" : "stopped"; await db .update(deployments) - .set(markDeploymentUndesired(nextStatus)) + .set(markDeploymentRemoved()) .where(eq(deployments.id, dep.id)); if (!dep.containerId) continue; @@ -1292,7 +1287,7 @@ export async function restartService(serviceId: string) { .where(eq(deployments.serviceId, serviceId)); const deploymentsToRestart = runningDeployments.filter( - (d) => d.status === "running" && d.containerId, + (d) => isObservedReady(d.observedPhase) && d.containerId, ); if (deploymentsToRestart.length === 0) { @@ -1509,16 +1504,7 @@ export async function removeServiceVolume(volumeId: string) { .from(deployments) .where(eq(deployments.serviceId, volume[0].serviceId)); - const runningStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - ]; - const hasRunning = activeDeployments.some((d) => - runningStatuses.includes(d.status), - ); + const hasRunning = activeDeployments.some(blocksProjectDeletion); if (hasRunning) { throw new Error("Stop the service before removing volumes"); } diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx index 849b10a3..77700a12 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx @@ -47,7 +47,7 @@ export default function ConfigurationPage() { const hasActiveDeploymentForBackup = service.deployments.some( (deployment) => ACTIVE_DELETE_BACKUP_STATUSES.includes( - deployment.status as (typeof ACTIVE_DELETE_BACKUP_STATUSES)[number], + deployment.observedPhase as (typeof ACTIVE_DELETE_BACKUP_STATUSES)[number], ) && !!deployment.containerId, ); const hasVolumes = (service.volumes?.length ?? 0) > 0; diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx index 2357c275..2aab06d2 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx @@ -122,15 +122,13 @@ export default function DeploymentsPage() { }; const hasRunningDeployments = service.deployments.some( - (d) => d.status === "running", + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", ); const hasStoppedOrFailedDeployments = !hasRunningDeployments && service.deployments.some( (d) => - d.status === "stopped" || - d.status === "failed" || - d.status === "rolled_back", + d.runtimeDesiredState === "removed" || d.observedPhase === "failed", ); const canStartAll = hasStoppedOrFailedDeployments && diff --git a/web/components/service/details/deployment-progress.tsx b/web/components/service/details/deployment-progress.tsx index 48bbb925..375c5327 100644 --- a/web/components/service/details/deployment-progress.tsx +++ b/web/components/service/details/deployment-progress.tsx @@ -130,14 +130,16 @@ export function getBarState( if (!latestRolloutJustCompleted) { const inProgressStatuses = ["pending", "pulling", "starting", "healthy"]; const hasInProgressDeployments = service.deployments.some((d) => - inProgressStatuses.includes(d.status), + inProgressStatuses.includes(d.observedPhase), ); if (hasInProgressDeployments) { const maxStageIndex = Math.max( ...service.deployments - .filter((d) => inProgressStatuses.includes(d.status)) - .map((d) => getStageIndex(mapDeploymentStatusToStage(d.status))), + .filter((d) => inProgressStatuses.includes(d.observedPhase)) + .map((d) => + getStageIndex(mapDeploymentStatusToStage(d.observedPhase)), + ), ); return { mode: "deploying", diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index a876dafc..9f6f33b7 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -709,7 +709,10 @@ function buildOverviewData( total: 0, }; summary.total++; - if (deployment.status === "running") { + if ( + deployment.observedPhase === "running" || + deployment.observedPhase === "healthy" + ) { runningDeployments++; summary.running++; } @@ -788,7 +791,7 @@ function getServiceStatus( if (runningDeployments > 0) return { label: "Live", tone: "live" }; for (const deployment of service.deployments || []) { - if (deployment.status === "failed" || deployment.status === "rolled_back") { + if (deployment.observedPhase === "failed") { return { label: "Needs attention", tone: "warning" }; } } diff --git a/web/components/service/service-canvas.tsx b/web/components/service/service-canvas.tsx index 63221d28..1e825499 100644 --- a/web/components/service/service-canvas.tsx +++ b/web/components/service/service-canvas.tsx @@ -307,10 +307,10 @@ function ServiceCard({ p.externalPort, ); const hasInternalDns = service.deployments.some( - (d) => d.status === "running", + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", ); const runningCount = service.deployments.filter( - (d) => d.status === "running", + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", ).length; const hasEndpoints = diff --git a/web/components/service/service-layout-client.tsx b/web/components/service/service-layout-client.tsx index 7af810ba..67fafb1c 100644 --- a/web/components/service/service-layout-client.tsx +++ b/web/components/service/service-layout-client.tsx @@ -66,7 +66,7 @@ export function ServiceLayoutClient({ svc.rollouts?.[0]?.status === "in_progress" || !!svc.migrationStatus || svc.deployments.some((d) => - IN_PROGRESS_DEPLOY_STATUSES.includes(d.status), + IN_PROGRESS_DEPLOY_STATUSES.includes(d.observedPhase), ); setHasActiveActivity(isActive); }, diff --git a/web/components/ui/canvas-wrapper.tsx b/web/components/ui/canvas-wrapper.tsx index 37702625..e863f06f 100644 --- a/web/components/ui/canvas-wrapper.tsx +++ b/web/components/ui/canvas-wrapper.tsx @@ -66,15 +66,26 @@ export function getStatusColor(status: string): StatusColors { } export function getStatusColorFromDeployments( - deployments: { status: string }[], + deployments: { observedPhase: string; runtimeDesiredState?: string }[], ): StatusColors { - const hasRunning = deployments.some((d) => d.status === "running"); + const hasRunning = deployments.some( + (d) => d.observedPhase === "running" || d.observedPhase === "healthy", + ); const hasPending = deployments.some( - (d) => d.status === "pending" || d.status === "pulling", + (d) => + d.observedPhase === "pending" || + d.observedPhase === "pulling" || + d.observedPhase === "starting" || + d.observedPhase === "waking", + ); + const hasFailed = deployments.some((d) => d.observedPhase === "failed"); + const hasStopped = deployments.some( + (d) => + d.observedPhase === "stopped" || + d.observedPhase === "sleeping" || + d.runtimeDesiredState === "removed", ); - const hasFailed = deployments.some((d) => d.status === "failed"); - const hasStopped = deployments.some((d) => d.status === "stopped"); - const hasUnknown = deployments.some((d) => d.status === "unknown"); + const hasUnknown = deployments.some((d) => d.observedPhase === "unknown"); if (hasRunning) return statusColorMap.running; if (hasPending) return statusColorMap.pending; diff --git a/web/db/queries.ts b/web/db/queries.ts index 800895a4..7bf34803 100644 --- a/web/db/queries.ts +++ b/web/db/queries.ts @@ -228,7 +228,7 @@ export async function getServerServices(serverId: string) { const results = await db .selectDistinctOn([services.id], { deploymentId: deployments.id, - deploymentStatus: deployments.status, + deploymentStatus: deployments.observedPhase, serviceId: services.id, serviceName: services.name, serviceImage: services.image, diff --git a/web/db/schema.ts b/web/db/schema.ts index 29f96e8c..23883407 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -532,7 +532,17 @@ export const deployments = pgTable( .references(() => servers.id, { onDelete: "cascade" }), containerId: text("container_id"), ipAddress: text("ip_address"), - status: text("status", { + runtimeDesiredState: text("runtime_desired_state", { + enum: ["running", "stopped", "removed"], + }) + .notNull() + .default("running"), + trafficState: text("traffic_state", { + enum: ["candidate", "active", "draining", "inactive"], + }) + .notNull() + .default("candidate"), + observedPhase: text("observed_phase", { enum: [ "pending", "pulling", @@ -540,18 +550,14 @@ export const deployments = pgTable( "waking", "healthy", "running", - "draining", "sleeping", - "stopping", "stopped", "failed", - "rolled_back", "unknown", ], }) .notNull() .default("pending"), - desired: boolean("desired").notNull().default(true), healthStatus: text("health_status", { enum: ["none", "starting", "healthy", "unhealthy"], }), @@ -579,7 +585,11 @@ export const deployments = pgTable( index("deployments_rollout_id_idx").on(table.rolloutId), index("deployments_service_id_idx").on(table.serviceId), index("deployments_server_id_idx").on(table.serverId), - index("deployments_status_idx").on(table.status), + index("deployments_runtime_desired_state_idx").on( + table.runtimeDesiredState, + ), + index("deployments_traffic_state_idx").on(table.trafficState), + index("deployments_observed_phase_idx").on(table.observedPhase), ], ); diff --git a/web/db/types.ts b/web/db/types.ts index 04239040..e5674209 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -40,7 +40,7 @@ export type MemberInvitation = typeof memberInvitations.$inferSelect; export type MemberRole = User["role"]; export type InvitableMemberRole = MemberInvitation["role"]; -export type DeploymentStatus = NonNullable; +export type DeploymentStatus = NonNullable; export type HealthStatus = Deployment["healthStatus"]; export type RolloutStatus = NonNullable; export type BuildStatus = NonNullable; diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index bf6e483e..b2a36c8f 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -18,7 +18,14 @@ import { getStartingHealthCheckFailureUpdate, getSteadyStateRecreateDecision, } from "@/lib/autoheal-policy"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { + type ObservedPhase, + isObservedActiveContainer, + isObservedReady, + markDeploymentFailedRemoved, + observedStartingPhases, + runtimeExpectedStates, +} from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; @@ -43,8 +50,10 @@ export type ServerlessTransition = | { type: "wake_started"; deploymentId: string } | { type: "wake_failed"; deploymentId: string; error: string }; -export function shouldAttachReportedContainer(status: string) { - return status === "pending" || status === "pulling" || status === "waking"; +export function shouldAttachReportedContainer(observedPhase: ObservedPhase) { + return (observedStartingPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); } function isMigrationTargetStarting(status: string | null | undefined) { @@ -90,7 +99,7 @@ async function applyDeploymentErrors( id: deployments.id, serviceId: deployments.serviceId, rolloutId: deployments.rolloutId, - status: deployments.status, + observedPhase: deployments.observedPhase, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, serverlessEnabled: services.serverlessEnabled, serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, @@ -117,7 +126,7 @@ async function applyDeploymentErrors( continue; } - const isServerlessWakeDeployment = deployment.status === "waking"; + const isServerlessWakeDeployment = deployment.observedPhase === "waking"; const isActiveRolloutDeployment = !isServerlessWakeDeployment && deployment.rolloutId && @@ -137,13 +146,13 @@ async function applyDeploymentErrors( currentFailureCount: deployment.serverlessWakeFailureCount, failedStage: "serverless_wake", }) - : { status: "failed", failedStage: "deploying" }, + : markDeploymentFailedRemoved("deploying"), ) .where( and( eq(deployments.id, deployment.id), inArray( - deployments.status, + deployments.observedPhase, isServerlessWakeDeployment ? ["waking"] : ["pending", "pulling", "starting"], @@ -205,8 +214,9 @@ async function applyServerlessTransitions( serviceId: deployments.serviceId, serverId: deployments.serverId, containerId: deployments.containerId, - status: deployments.status, - desired: deployments.desired, + runtimeDesiredState: deployments.runtimeDesiredState, + trafficState: deployments.trafficState, + observedPhase: deployments.observedPhase, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, serverlessEnabled: services.serverlessEnabled, serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, @@ -239,7 +249,9 @@ async function applyServerlessTransitions( const updated = await db .update(deployments) .set({ - status: "sleeping", + runtimeDesiredState: "stopped", + observedPhase: "sleeping", + containerId: null, healthStatus: null, failedStage: null, }) @@ -248,8 +260,8 @@ async function applyServerlessTransitions( eq(deployments.id, transition.deploymentId), eq(deployments.serverId, serverId), eq(deployments.containerId, transition.containerId), - eq(deployments.desired, true), - inArray(deployments.status, ["healthy", "running"]), + eq(deployments.runtimeDesiredState, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .returning({ id: deployments.id }); @@ -270,7 +282,9 @@ async function applyServerlessTransitions( const updated = await db .update(deployments) .set({ - status: "waking", + runtimeDesiredState: "running", + observedPhase: "waking", + containerId: null, healthStatus: null, failedStage: null, }) @@ -278,8 +292,8 @@ async function applyServerlessTransitions( and( eq(deployments.id, transition.deploymentId), eq(deployments.serverId, serverId), - eq(deployments.desired, true), - eq(deployments.status, "sleeping"), + eq(deployments.runtimeDesiredState, "stopped"), + eq(deployments.observedPhase, "sleeping"), ), ) .returning({ id: deployments.id }); @@ -309,8 +323,8 @@ async function applyServerlessTransitions( and( eq(deployments.id, transition.deploymentId), eq(deployments.serverId, serverId), - eq(deployments.desired, true), - inArray(deployments.status, ["sleeping", "waking"]), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + inArray(deployments.observedPhase, ["sleeping", "waking"]), ), ) .returning({ id: deployments.id }); @@ -358,8 +372,9 @@ function getInvalidServerlessTransitionReason({ | { serverId: string; containerId: string | null; - status: string; - desired: boolean; + runtimeDesiredState: string; + trafficState: string; + observedPhase: string; serverlessEnabled: boolean; serverlessSleepAfterSeconds: number; serverlessWakeTimeoutSeconds: number; @@ -375,11 +390,16 @@ function getInvalidServerlessTransitionReason({ if (!getDeployedServerlessConfig(deployment).enabled) { return "service is not serverless"; } - if (!deployment.desired) return "deployment is not desired"; + if (deployment.runtimeDesiredState === "removed") { + return "deployment is removed"; + } if (transition.type === "sleep") { - if (!["healthy", "running"].includes(deployment.status)) { - return `deployment is not sleepable from ${deployment.status}`; + if (deployment.runtimeDesiredState !== "running") { + return `deployment is not expected running (${deployment.runtimeDesiredState})`; + } + if (!["healthy", "running"].includes(deployment.observedPhase)) { + return `deployment is not sleepable from ${deployment.observedPhase}`; } if (deployment.containerId !== transition.containerId) { return "stale containerId"; @@ -388,16 +408,16 @@ function getInvalidServerlessTransitionReason({ if ( transition.type === "wake_started" && - deployment.status !== "sleeping" + deployment.observedPhase !== "sleeping" ) { - return `deployment is not sleeping (${deployment.status})`; + return `deployment is not sleeping (${deployment.observedPhase})`; } if ( transition.type === "wake_failed" && - !["sleeping", "waking"].includes(deployment.status) + !["sleeping", "waking"].includes(deployment.observedPhase) ) { - return `deployment is not waking or sleeping (${deployment.status})`; + return `deployment is not waking or sleeping (${deployment.observedPhase})`; } return null; @@ -524,45 +544,38 @@ export async function applyStatusReport( .map((c) => c.deploymentId) .filter((id) => id !== ""); - const activeStatuses = [ - "starting", - "healthy", - "running", - "stopping", - ] as const; - const activeDeployments = await db .select({ id: deployments.id, containerId: deployments.containerId, - status: deployments.status, + runtimeDesiredState: deployments.runtimeDesiredState, + observedPhase: deployments.observedPhase, }) .from(deployments) .where( and( eq(deployments.serverId, serverId), isNotNull(deployments.containerId), - inArray(deployments.status, activeStatuses), ), ); for (const dep of activeDeployments) { if (!reportedDeploymentIds.includes(dep.id)) { - if (dep.status === "stopping") { + if (dep.runtimeDesiredState === "removed") { console.log( - `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} was stopping and container gone, deleting`, + `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} was removed and container gone, deleting`, ); await db .delete(deploymentPorts) .where(eq(deploymentPorts.deploymentId, dep.id)); await db.delete(deployments).where(eq(deployments.id, dep.id)); - } else { + } else if (isObservedActiveContainer(dep.observedPhase)) { console.log( `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} NOT reported, marking UNKNOWN`, ); await db .update(deployments) - .set({ status: "unknown", healthStatus: null }) + .set({ observedPhase: "unknown", healthStatus: null }) .where(eq(deployments.id, dep.id)); } } @@ -586,7 +599,6 @@ export async function applyStatusReport( } if (!deployment) { - const stuckStatuses = ["pending", "pulling", "waking"] as const; const [stuckDeployment] = await db .select() .from(deployments) @@ -594,7 +606,7 @@ export async function applyStatusReport( and( eq(deployments.serverId, serverId), isNull(deployments.containerId), - inArray(deployments.status, stuckStatuses), + inArray(deployments.observedPhase, observedStartingPhases), ), ); @@ -616,7 +628,7 @@ export async function applyStatusReport( .update(deployments) .set({ containerId: container.containerId, - status: newStatus, + observedPhase: newStatus, healthStatus: hasHealthCheck ? "starting" : "none", serverlessWakeFailureCount: 0, }) @@ -624,7 +636,7 @@ export async function applyStatusReport( deployment = { ...stuckDeployment, - status: newStatus, + observedPhase: newStatus, containerId: container.containerId, healthStatus: hasHealthCheck ? "starting" : "none", }; @@ -664,7 +676,38 @@ export async function applyStatusReport( updateFields.containerId = container.containerId; } - if (shouldAttachReportedContainer(deployment.status)) { + if (container.status === "stopped") { + updateFields.observedPhase = "stopped"; + updateFields.healthStatus = "none"; + await db + .update(deployments) + .set(updateFields) + .where(eq(deployments.id, deployment.id)); + continue; + } + + if (container.status === "failed") { + updateFields.observedPhase = "failed"; + updateFields.healthStatus = null; + updateFields.failedStage ??= "container_failed"; + await db + .update(deployments) + .set(updateFields) + .where(eq(deployments.id, deployment.id)); + if (deployment.rolloutId) { + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "rollout", + parentId: deployment.rolloutId, + }), + ); + } + continue; + } + + if (shouldAttachReportedContainer(deployment.observedPhase)) { if (container.status !== "running") { continue; } @@ -677,15 +720,15 @@ export async function applyStatusReport( const hasHealthCheck = service?.healthCheckCmd != null; const newStatus = hasHealthCheck ? "starting" : "healthy"; - updateFields.status = newStatus; - if (deployment.status === "waking") { + updateFields.observedPhase = newStatus; + if (deployment.observedPhase === "waking") { updateFields.serverlessWakeFailureCount = 0; } if (hasHealthCheck) { updateFields.healthStatus = "starting"; } console.log( - `[health:attach] deployment ${deployment.id} transitioning from ${deployment.status} to ${newStatus}`, + `[health:attach] deployment ${deployment.id} transitioning from ${deployment.observedPhase} to ${newStatus}`, ); if (deployment.rolloutId) { @@ -725,12 +768,12 @@ export async function applyStatusReport( } } - if (deployment.status === "unknown") { + if (deployment.observedPhase === "unknown") { const newStatus = healthStatus === "healthy" || healthStatus === "none" ? "running" : "starting"; - updateFields.status = newStatus; + updateFields.observedPhase = newStatus; console.log( `[health:restore] deployment ${deployment.id} restored from unknown to ${newStatus}`, ); @@ -738,7 +781,7 @@ export async function applyStatusReport( const canAutoheal = container.status === "running" && - (deployment.status === "running" || deployment.status === "healthy"); + isObservedReady(deployment.observedPhase); const healthRecovered = healthStatus === "healthy" || healthStatus === "none"; if (canAutoheal && healthRecovered) { @@ -766,8 +809,10 @@ export async function applyStatusReport( console.log( `[autoheal] rollout deployment ${deployment.id} exceeded restart limit`, ); - Object.assign(updateFields, markDeploymentUndesired("failed")); - updateFields.failedStage = "autoheal"; + Object.assign( + updateFields, + markDeploymentFailedRemoved("autoheal"), + ); autohealFailed = true; } else { autohealRecreatePayload = prepareAutohealRecreatePayload({ @@ -825,7 +870,7 @@ export async function applyStatusReport( } if ( - deployment.status === "starting" && + deployment.observedPhase === "starting" && container.status === "running" && (healthStatus === "healthy" || healthStatus === "none") ) { @@ -836,7 +881,7 @@ export async function applyStatusReport( await db .update(deployments) .set({ - status: "healthy", + observedPhase: "healthy", autohealRestartCount: 0, autohealRecreateCount: 0, serverlessWakeFailureCount: 0, @@ -875,7 +920,10 @@ export async function applyStatusReport( } } - if (deployment.status === "starting" && healthStatus === "unhealthy") { + if ( + deployment.observedPhase === "starting" && + healthStatus === "unhealthy" + ) { console.log(`[health] deployment ${deployment.id} failed health check`); const rollout = deployment.rolloutId ? await db diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index e6eb40eb..b1ae7719 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -11,9 +11,10 @@ import { } from "@/db/schema"; import { getAllCertificatesForDomains } from "@/lib/acme-manager"; import { - dnsDeploymentStatuses, - expectedDeploymentStatuses, - routableDeploymentStatuses, + activeTrafficStates, + isDeploymentRoutable, + observedReadyPhases, + runtimeExpectedStates, } from "@/lib/deployment-status"; import { getDeployedServerlessConfig, @@ -150,7 +151,9 @@ type ServerlessDeploymentRow = { serviceId: string; serverId: string; ipAddress: string | null; - status: Deployment["status"]; + runtimeDesiredState: Deployment["runtimeDesiredState"]; + trafficState: Deployment["trafficState"]; + observedPhase: Deployment["observedPhase"]; serverIsProxy: boolean; }; @@ -200,8 +203,7 @@ async function buildExpectedContainers( .where( and( eq(deployments.serverId, serverId), - eq(deployments.desired, true), - inArray(deployments.status, expectedDeploymentStatuses), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), ); @@ -325,8 +327,7 @@ export function buildExpectedContainersFromRows({ serverIsProxy && isDeployedServerlessService(service) && sleepableServiceIds.has(service.id) && - (dep.status === "sleeping" || - (dep.status === "draining" && !dep.containerId)) + dep.runtimeDesiredState === "stopped" ? "stopped" : "running", image: normalizeImage(service.image), @@ -379,7 +380,9 @@ async function buildServerlessExpectedState( serviceId: deployments.serviceId, serverId: deployments.serverId, ipAddress: deployments.ipAddress, - status: deployments.status, + runtimeDesiredState: deployments.runtimeDesiredState, + trafficState: deployments.trafficState, + observedPhase: deployments.observedPhase, serverIsProxy: servers.isProxy, }) .from(deployments) @@ -387,8 +390,7 @@ async function buildServerlessExpectedState( .where( and( inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, expectedDeploymentStatuses), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), ), ]); @@ -449,7 +451,7 @@ export function buildServerlessRoutesFromRows({ (deployment) => deployment.serverId === serverId && deployment.serverIsProxy && - deployment.status !== "draining" && + deployment.trafficState === "active" && expectedDeploymentIds.has(deployment.id), ) .map((deployment) => deployment.id) @@ -459,7 +461,7 @@ export function buildServerlessRoutesFromRows({ .filter( (deployment) => deployment.ipAddress && - routableDeploymentStatuses.includes(deployment.status), + isDeploymentRoutable(deployment), ) .map((deployment) => ({ deploymentId: deployment.id, @@ -503,8 +505,9 @@ async function buildDnsRecords(allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, dnsDeploymentStatuses), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + inArray(deployments.trafficState, activeTrafficStates), + inArray(deployments.observedPhase, observedReadyPhases), ), ); @@ -557,8 +560,12 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), - inArray(deployments.status, routableDeploymentStatuses), + inArray( + deployments.runtimeDesiredState, + runtimeExpectedStates, + ), + inArray(deployments.trafficState, activeTrafficStates), + inArray(deployments.observedPhase, observedReadyPhases), ), ) : Promise.resolve([]), @@ -573,9 +580,12 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - eq(deployments.desired, true), eq(servers.isProxy, true), - inArray(deployments.status, expectedDeploymentStatuses), + inArray( + deployments.runtimeDesiredState, + runtimeExpectedStates, + ), + inArray(deployments.trafficState, activeTrafficStates), ), ) : Promise.resolve([]), diff --git a/web/lib/autoheal-policy.ts b/web/lib/autoheal-policy.ts index eecfcae3..d93d59ae 100644 --- a/web/lib/autoheal-policy.ts +++ b/web/lib/autoheal-policy.ts @@ -23,8 +23,15 @@ export function getStartingHealthCheckFailureUpdate({ return { update: { - status: "failed" as const, - desired: !isRolloutDeployment && !recreateLimitReached, + observedPhase: "failed" as const, + runtimeDesiredState: + !isRolloutDeployment && !recreateLimitReached + ? ("running" as const) + : ("removed" as const), + trafficState: + !isRolloutDeployment && !recreateLimitReached + ? ("active" as const) + : ("inactive" as const), failedStage, ...(isRolloutDeployment ? {} @@ -54,8 +61,9 @@ export function getSteadyStateRecreateDecision({ return { limitReached: true, updateFields: { - status: "failed" as const, - desired: false, + observedPhase: "failed" as const, + runtimeDesiredState: "removed" as const, + trafficState: "inactive" as const, failedStage: "autoheal_recreate_limit", unhealthyReportCount: 0, autohealRestartCount: 0, @@ -67,8 +75,9 @@ export function getSteadyStateRecreateDecision({ return { limitReached: false, updateFields: { - status: "failed" as const, - desired: true, + observedPhase: "failed" as const, + runtimeDesiredState: "running" as const, + trafficState: "active" as const, failedStage: "autoheal_recreate", unhealthyReportCount: 0, autohealRestartCount: 0, diff --git a/web/lib/backups/trigger-backup.ts b/web/lib/backups/trigger-backup.ts index 0b0c29ed..abb79dda 100644 --- a/web/lib/backups/trigger-backup.ts +++ b/web/lib/backups/trigger-backup.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { @@ -54,7 +54,7 @@ export async function triggerBackup({ .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/cli-service.ts b/web/lib/cli-service.ts index 3d1222e3..e41ca193 100644 --- a/web/lib/cli-service.ts +++ b/web/lib/cli-service.ts @@ -761,7 +761,7 @@ export async function getManifestStatus(identity: ManifestIdentity) { const serviceDeployments = await db .select({ id: deployments.id, - status: deployments.status, + status: deployments.observedPhase, serverId: deployments.serverId, createdAt: deployments.createdAt, }) diff --git a/web/lib/deployment-status.ts b/web/lib/deployment-status.ts index beba89b5..ed4619ad 100644 --- a/web/lib/deployment-status.ts +++ b/web/lib/deployment-status.ts @@ -1,46 +1,95 @@ import type { deployments } from "@/db/schema"; -export type DeploymentStatus = typeof deployments.$inferSelect.status; +export type RuntimeDesiredState = + typeof deployments.$inferSelect.runtimeDesiredState; +export type TrafficState = typeof deployments.$inferSelect.trafficState; +export type ObservedPhase = typeof deployments.$inferSelect.observedPhase; -export type UndesiredDeploymentStatus = Extract< - DeploymentStatus, - "stopping" | "stopped" | "failed" | "rolled_back" ->; - -type DeploymentStatusCapabilities = { - expected: boolean; - routable: boolean; - dns: boolean; +export type DeploymentState = { + runtimeDesiredState: RuntimeDesiredState; + trafficState: TrafficState; + observedPhase: ObservedPhase; }; -const deploymentStatusCapabilities = { - pending: { expected: true, routable: false, dns: false }, - pulling: { expected: true, routable: false, dns: false }, - starting: { expected: true, routable: false, dns: false }, - waking: { expected: true, routable: false, dns: false }, - healthy: { expected: true, routable: true, dns: true }, - running: { expected: true, routable: true, dns: true }, - draining: { expected: true, routable: false, dns: false }, - sleeping: { expected: true, routable: false, dns: false }, - stopping: { expected: false, routable: false, dns: false }, - stopped: { expected: false, routable: false, dns: false }, - failed: { expected: false, routable: false, dns: false }, - rolled_back: { expected: false, routable: false, dns: false }, - unknown: { expected: true, routable: false, dns: false }, -} satisfies Record; - -export const expectedDeploymentStatuses = statusesWithCapability("expected"); -export const routableDeploymentStatuses = statusesWithCapability("routable"); -export const dnsDeploymentStatuses = statusesWithCapability("dns"); - -export function markDeploymentUndesired(status: UndesiredDeploymentStatus) { - return { status, desired: false }; -} - -function statusesWithCapability( - capability: keyof DeploymentStatusCapabilities, -) { - return Object.entries(deploymentStatusCapabilities) - .filter(([, capabilities]) => capabilities[capability]) - .map(([status]) => status as DeploymentStatus); +export const runtimeExpectedStates = ["running", "stopped"] as const satisfies + readonly RuntimeDesiredState[]; + +export const activeTrafficStates = ["active"] as const satisfies + readonly TrafficState[]; + +export const observedReadyPhases = ["healthy", "running"] as const satisfies + readonly ObservedPhase[]; + +export const observedStartingPhases = [ + "pending", + "pulling", + "starting", + "waking", +] as const satisfies readonly ObservedPhase[]; + +export const observedActiveContainerPhases = [ + "starting", + "healthy", + "running", +] as const satisfies readonly ObservedPhase[]; + +export function isRuntimeExpected( + runtimeDesiredState: RuntimeDesiredState, +): boolean { + return runtimeDesiredState !== "removed"; +} + +export function isObservedReady(observedPhase: ObservedPhase): boolean { + return (observedReadyPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isObservedStarting(observedPhase: ObservedPhase): boolean { + return (observedStartingPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isObservedActiveContainer( + observedPhase: ObservedPhase, +): boolean { + return (observedActiveContainerPhases as readonly ObservedPhase[]).includes( + observedPhase, + ); +} + +export function isTrafficActive(trafficState: TrafficState): boolean { + return trafficState === "active"; +} + +export function isDeploymentExpected( + deployment: Pick, +): boolean { + return isRuntimeExpected(deployment.runtimeDesiredState); +} + +export function isDeploymentRoutable( + deployment: Pick, +): boolean { + return ( + isTrafficActive(deployment.trafficState) && + isObservedReady(deployment.observedPhase) + ); +} + +export function markDeploymentRemoved() { + return { + runtimeDesiredState: "removed" as const, + trafficState: "inactive" as const, + }; +} + +export function markDeploymentFailedRemoved(failedStage: string) { + return { + ...markDeploymentRemoved(), + observedPhase: "failed" as const, + healthStatus: null, + failedStage, + }; } diff --git a/web/lib/inngest/functions/migration-workflow.ts b/web/lib/inngest/functions/migration-workflow.ts index a5dcab97..bdc03276 100644 --- a/web/lib/inngest/functions/migration-workflow.ts +++ b/web/lib/inngest/functions/migration-workflow.ts @@ -9,7 +9,7 @@ import { volumeBackups, } from "@/db/schema"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentRemoved } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -65,7 +65,7 @@ export const migrationWorkflow = inngest.createFunction( await db .update(deployments) - .set(markDeploymentUndesired("stopped")) + .set(markDeploymentRemoved()) .where(eq(deployments.id, sourceDeploymentId)); }); diff --git a/web/lib/inngest/functions/on-deployment-failed.ts b/web/lib/inngest/functions/on-deployment-failed.ts index b863cf82..61969e25 100644 --- a/web/lib/inngest/functions/on-deployment-failed.ts +++ b/web/lib/inngest/functions/on-deployment-failed.ts @@ -17,7 +17,7 @@ export const onDeploymentFailed = inngest.createFunction( id: deployments.id, rolloutId: deployments.rolloutId, serviceId: deployments.serviceId, - status: deployments.status, + observedPhase: deployments.observedPhase, failedStage: deployments.failedStage, }) .from(deployments) @@ -27,7 +27,7 @@ export const onDeploymentFailed = inngest.createFunction( if ( !deployment?.rolloutId || - (deployment.status !== "failed" && deployment.status !== "rolled_back") + deployment.observedPhase !== "failed" ) { return; } diff --git a/web/lib/inngest/functions/restore-trigger-workflow.ts b/web/lib/inngest/functions/restore-trigger-workflow.ts index e217daf1..6409cfa1 100644 --- a/web/lib/inngest/functions/restore-trigger-workflow.ts +++ b/web/lib/inngest/functions/restore-trigger-workflow.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { deployments, volumeBackups } from "@/db/schema"; @@ -51,7 +51,7 @@ export const restoreTriggerWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index a7ef27cf..abe53c72 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -13,11 +13,7 @@ import { serviceVolumes, } from "@/db/schema"; import { getCertificate, issueCertificate } from "@/lib/acme-manager"; -import { - buildCurrentConfig, - getDeployedStateful, - isDeployedServerlessService, -} from "@/lib/service-config"; +import { buildCurrentConfig, getDeployedStateful } from "@/lib/service-config"; import { assignContainerIp } from "@/lib/wireguard"; import { enqueueWork } from "@/lib/work-queue"; @@ -38,7 +34,7 @@ export type DeploymentContext = { }; export function isActiveDeploymentForRollout( - deployment: { status: string }, + deployment: { trafficState: string }, service: { deployedConfig?: string | null; stateful?: boolean | null; @@ -48,13 +44,8 @@ export function isActiveDeploymentForRollout( serverlessMinReadyReplicas?: number | null; }, ) { - if (deployment.status === "running" || deployment.status === "healthy") { - return true; - } - return ( - isDeployedServerlessService(service) && - (deployment.status === "sleeping" || deployment.status === "waking") - ); + void service; + return deployment.trafficState === "active"; } export function normalizeImage(image: string): string { @@ -217,7 +208,7 @@ export async function cleanupTerminalDeployments( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["rolled_back", "stopped", "failed"]), + eq(deployments.runtimeDesiredState, "removed"), ), ); @@ -373,7 +364,9 @@ export async function createDeploymentRecords( serviceId, serverId: server.id, ipAddress, - status: "pending", + runtimeDesiredState: "running", + trafficState: "candidate", + observedPhase: "pending", rolloutId, }); diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index b876661c..f78c3857 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -1,65 +1,35 @@ -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, eq, ne } from "drizzle-orm"; import { db } from "@/db"; -import { deployments, rollouts, services } from "@/db/schema"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { deployments, rollouts } from "@/db/schema"; +import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; import { sendDeploymentFailureAlert } from "@/lib/email"; -import { isDeployedServerlessService } from "@/lib/service-config"; -const ROLLOUT_FAILURE_CLEANUP_STATUSES = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - "sleeping", - "waking", - "failed", -] as const; - -export function shouldRollBackDeploymentStatus(status: string) { - return ROLLOUT_FAILURE_CLEANUP_STATUSES.includes( - status as (typeof ROLLOUT_FAILURE_CLEANUP_STATUSES)[number], - ); +export function shouldRollBackDeploymentState(deployment: { + trafficState: string; + runtimeDesiredState: string; +}) { + void deployment.trafficState; + return deployment.runtimeDesiredState !== "removed"; } -export function shouldRestoreDrainingDeploymentAsSleeping( - deployment: { containerId: string | null }, - service: Parameters[0] | null | undefined, -) { +export function shouldRestoreDrainingDeployment(deployment: { + trafficState: string; + runtimeDesiredState: string; +}) { return ( - !deployment.containerId && !!service && isDeployedServerlessService(service) + deployment.trafficState === "draining" && + deployment.runtimeDesiredState !== "removed" ); } export async function restoreDrainingDeploymentsForRollback(serviceId: string) { - const service = await db - .select() - .from(services) - .where(eq(services.id, serviceId)) - .then((rows) => rows[0]); - - if ( - shouldRestoreDrainingDeploymentAsSleeping({ containerId: null }, service) - ) { - await db - .update(deployments) - .set({ status: "sleeping", healthStatus: null }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), - isNull(deployments.containerId), - ), - ); - } - await db .update(deployments) - .set({ status: "running" }) + .set({ trafficState: "active" }) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), + eq(deployments.trafficState, "draining"), ), ); } @@ -106,11 +76,11 @@ export async function handleRolloutFailure( await db .update(deployments) - .set({ ...markDeploymentUndesired("rolled_back"), failedStage: reason }) + .set(markDeploymentFailedRemoved(reason)) .where( and( eq(deployments.rolloutId, rolloutId), - inArray(deployments.status, ROLLOUT_FAILURE_CLEANUP_STATUSES), + ne(deployments.runtimeDesiredState, "removed"), ), ); diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 3143f9c6..d2f6d5ee 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -2,7 +2,6 @@ import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; import { deployments, rollouts, servers } from "@/db/schema"; -import { isDeployedServerlessService } from "@/lib/service-config"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -415,7 +414,7 @@ export const rolloutWorkflow = inngest.createFunction( .where( and( inArray(deployments.id, deploymentIds), - inArray(deployments.status, ["healthy", "running"]), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ); @@ -444,7 +443,7 @@ export const rolloutWorkflow = inngest.createFunction( const deploymentStates = await db .select({ id: deployments.id, - status: deployments.status, + observedPhase: deployments.observedPhase, serverName: servers.name, }) .from(deployments) @@ -453,7 +452,8 @@ export const rolloutWorkflow = inngest.createFunction( return deploymentStates.filter( (deployment) => - deployment.status !== "healthy" && deployment.status !== "running", + deployment.observedPhase !== "healthy" && + deployment.observedPhase !== "running", ); }, ); @@ -489,42 +489,37 @@ export const rolloutWorkflow = inngest.createFunction( } await step.run("start-dns-sync", async () => { - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - const oldActiveStatuses = isDeployedServerlessService(service) - ? (["running", "healthy", "sleeping", "waking"] as const) - : (["running", "healthy"] as const); - - await db - .update(rollouts) - .set({ currentStage: "dns_sync" }) - .where(eq(rollouts.id, rolloutId)); + await db.transaction(async (tx) => { + await tx + .update(rollouts) + .set({ currentStage: "dns_sync" }) + .where(eq(rollouts.id, rolloutId)); - await db - .update(deployments) - .set({ status: "running" }) - .where( - and( - eq(deployments.rolloutId, rolloutId), - eq(deployments.status, "healthy"), - ), - ); + await tx + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.rolloutId, rolloutId), + eq(deployments.trafficState, "candidate"), + inArray(deployments.observedPhase, ["healthy", "running"]), + ), + ); - await db - .update(deployments) - .set({ status: "draining" }) - .where( - and( - eq(deployments.serviceId, serviceId), - inArray(deployments.status, oldActiveStatuses), - or( - ne(deployments.rolloutId, rolloutId), - isNull(deployments.rolloutId), + await tx + .update(deployments) + .set({ trafficState: "draining" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "active"), + or( + ne(deployments.rolloutId, rolloutId), + isNull(deployments.rolloutId), + ), ), - ), - ); + ); + }); await ingestRolloutLog( rolloutId, @@ -592,11 +587,14 @@ export const rolloutWorkflow = inngest.createFunction( await step.run("stop-old-deployments", async () => { const stoppedDeploymentsWithoutContainers = await db .update(deployments) - .set({ status: "stopped", desired: false }) + .set({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), + eq(deployments.trafficState, "draining"), isNull(deployments.containerId), ), ) @@ -604,11 +602,14 @@ export const rolloutWorkflow = inngest.createFunction( const stoppingDeployments = await db .update(deployments) - .set({ status: "stopping", desired: false }) + .set({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }) .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "draining"), + eq(deployments.trafficState, "draining"), ), ) .returning({ id: deployments.id }); diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index d13911ab..60a15634 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -13,7 +13,7 @@ import { volumeBackups, } from "@/db/schema"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentRemoved } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -81,7 +81,7 @@ export const serviceDeletionWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.status, ["running", "healthy"]), + inArray(deployments.observedPhase, ["running", "healthy"]), ), ) .then((r) => r[0]); @@ -237,16 +237,12 @@ export const serviceDeletionWorkflow = inngest.createFunction( .where(eq(deployments.serviceId, serviceId)); for (const deployment of allDeployments) { - if ( - (deployment.status === "running" || - deployment.status === "healthy") && - deployment.containerId - ) { - await db - .update(deployments) - .set(markDeploymentUndesired("stopping")) - .where(eq(deployments.id, deployment.id)); + await db + .update(deployments) + .set(markDeploymentRemoved()) + .where(eq(deployments.id, deployment.id)); + if (deployment.containerId) { await enqueueWork(deployment.serverId, "stop", { deploymentId: deployment.id, containerId: deployment.containerId, @@ -470,7 +466,7 @@ export const serviceRestoreWorkflow = inngest.createFunction( async () => { return db .select({ - status: deployments.status, + observedPhase: deployments.observedPhase, failedStage: deployments.failedStage, }) .from(deployments) @@ -485,11 +481,11 @@ export const serviceRestoreWorkflow = inngest.createFunction( const healthyDeployment = restoredDeployments.find( (deployment) => - deployment.status === "healthy" || deployment.status === "running", + deployment.observedPhase === "healthy" || + deployment.observedPhase === "running", ); const failedDeployment = restoredDeployments.find( - (deployment) => - deployment.status === "failed" || deployment.status === "rolled_back", + (deployment) => deployment.observedPhase === "failed", ); if (!healthyDeployment || failedDeployment) { diff --git a/web/lib/migrations.ts b/web/lib/migrations.ts index 3d041c08..ecfe684f 100644 --- a/web/lib/migrations.ts +++ b/web/lib/migrations.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { deployments, services, serviceVolumes } from "@/db/schema"; @@ -53,7 +53,7 @@ export async function startMigrationInternal( .where( and( eq(deployments.serviceId, serviceId), - eq(deployments.status, "running"), + inArray(deployments.observedPhase, ["healthy", "running"]), ), ) .then((r) => r[0]); diff --git a/web/lib/project-deletion.ts b/web/lib/project-deletion.ts index 2aa154e8..fb133926 100644 --- a/web/lib/project-deletion.ts +++ b/web/lib/project-deletion.ts @@ -2,9 +2,9 @@ import type { deployments } from "@/db/schema"; type ProjectDeletionDeployment = Pick< typeof deployments.$inferSelect, - "desired" + "runtimeDesiredState" >; export function blocksProjectDeletion(deployment: ProjectDeletionDeployment) { - return deployment.desired; + return deployment.runtimeDesiredState !== "removed"; } diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index c7177135..3bd882d2 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -26,14 +26,6 @@ async function triggerRecoveryForOfflineServers( ): Promise { if (offlineServerIds.length === 0) return; - const activeStatuses = [ - "pending", - "pulling", - "starting", - "healthy", - "running", - ] as const; - const affectedDeployments = await db .select({ deploymentId: deployments.id, @@ -49,7 +41,8 @@ async function triggerRecoveryForOfflineServers( .where( and( inArray(deployments.serverId, offlineServerIds), - inArray(deployments.status, activeStatuses), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + inArray(deployments.trafficState, ["candidate", "active"]), isNull(services.deletedAt), ), ); diff --git a/web/lib/serverless-wake-failures.ts b/web/lib/serverless-wake-failures.ts index 646066d0..a5c47ec4 100644 --- a/web/lib/serverless-wake-failures.ts +++ b/web/lib/serverless-wake-failures.ts @@ -1,4 +1,4 @@ -import { markDeploymentUndesired } from "@/lib/deployment-status"; +import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; export const SERVERLESS_WAKE_FAILURE_LIMIT = 3; @@ -21,15 +21,14 @@ export function getServerlessWakeFailureUpdate({ if (serverlessEnabled && nextFailureCount < SERVERLESS_WAKE_FAILURE_LIMIT) { return { ...baseUpdate, - status: "sleeping" as const, - desired: true, + runtimeDesiredState: "stopped" as const, + observedPhase: "sleeping" as const, failedStage: null, }; } return { - ...markDeploymentUndesired("failed"), + ...markDeploymentFailedRemoved(failedStage), ...baseUpdate, - failedStage, }; } diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts index c592442b..807334a5 100644 --- a/web/lib/work-queue.ts +++ b/web/lib/work-queue.ts @@ -347,8 +347,8 @@ async function runForceCleanupCompletionSideEffects( .update(deployments) .set({ containerId: null, - status: "pending", - desired: true, + runtimeDesiredState: "running", + observedPhase: "pending", healthStatus: null, unhealthyReportCount: 0, autohealRestartCount: 0, @@ -357,7 +357,7 @@ async function runForceCleanupCompletionSideEffects( .where( and( eq(deployments.id, payload.deploymentId), - eq(deployments.status, "failed"), + eq(deployments.observedPhase, "failed"), eq(deployments.failedStage, "autoheal_recreate"), ), ); diff --git a/web/tests/autoheal-policy.test.ts b/web/tests/autoheal-policy.test.ts index bb54169f..f3f0f7ca 100644 --- a/web/tests/autoheal-policy.test.ts +++ b/web/tests/autoheal-policy.test.ts @@ -13,8 +13,9 @@ describe("autoheal policy", () => { expect(decision.recreateLimitReached).toBe(false); expect(decision.update).toMatchObject({ - status: "failed", - desired: true, + observedPhase: "failed", + runtimeDesiredState: "running", + trafficState: "active", failedStage: "autoheal_recreate", autohealRecreateCount: 3, }); @@ -28,8 +29,9 @@ describe("autoheal policy", () => { expect(decision.recreateLimitReached).toBe(true); expect(decision.update).toMatchObject({ - status: "failed", - desired: false, + observedPhase: "failed", + runtimeDesiredState: "removed", + trafficState: "inactive", failedStage: "autoheal_recreate_limit", autohealRecreateCount: 3, }); @@ -42,8 +44,9 @@ describe("autoheal policy", () => { }); expect(decision.update).toEqual({ - status: "failed", - desired: false, + observedPhase: "failed", + runtimeDesiredState: "removed", + trafficState: "inactive", failedStage: "health_check", }); }); @@ -60,8 +63,9 @@ describe("autoheal policy", () => { expect(decision.limitReached).toBe(false); expect(decision.updateFields).toMatchObject({ - status: "failed", - desired: true, + observedPhase: "failed", + runtimeDesiredState: "running", + trafficState: "active", failedStage: "autoheal_recreate", autohealRecreateCount: 2, }); diff --git a/web/tests/deployment-status.test.ts b/web/tests/deployment-status.test.ts index 69f58a4d..06d958c7 100644 --- a/web/tests/deployment-status.test.ts +++ b/web/tests/deployment-status.test.ts @@ -1,20 +1,48 @@ import { describe, expect, it } from "vitest"; import { - dnsDeploymentStatuses, - expectedDeploymentStatuses, - routableDeploymentStatuses, + isDeploymentRoutable, + isRuntimeExpected, + isObservedReady, + markDeploymentRemoved, } from "@/lib/deployment-status"; -describe("deployment status capabilities", () => { - it("keeps sleeping deployments expected but unroutable", () => { - expect(expectedDeploymentStatuses).toContain("sleeping"); - expect(routableDeploymentStatuses).not.toContain("sleeping"); - expect(dnsDeploymentStatuses).not.toContain("sleeping"); +describe("deployment state helpers", () => { + it("keeps runtime intent separate from observed phase", () => { + expect(isRuntimeExpected("running")).toBe(true); + expect(isRuntimeExpected("stopped")).toBe(true); + expect(isRuntimeExpected("removed")).toBe(false); + + expect(isObservedReady("healthy")).toBe(true); + expect(isObservedReady("running")).toBe(true); + expect(isObservedReady("sleeping")).toBe(false); + expect(isObservedReady("waking")).toBe(false); + }); + + it("requires active traffic intent and a ready observation for routing", () => { + expect( + isDeploymentRoutable({ + trafficState: "active", + observedPhase: "healthy", + }), + ).toBe(true); + expect( + isDeploymentRoutable({ + trafficState: "candidate", + observedPhase: "healthy", + }), + ).toBe(false); + expect( + isDeploymentRoutable({ + trafficState: "active", + observedPhase: "sleeping", + }), + ).toBe(false); }); - it("keeps waking deployments expected but unroutable", () => { - expect(expectedDeploymentStatuses).toContain("waking"); - expect(routableDeploymentStatuses).not.toContain("waking"); - expect(dnsDeploymentStatuses).not.toContain("waking"); + it("represents removal as explicit runtime and traffic intent", () => { + expect(markDeploymentRemoved()).toEqual({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }); }); }); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index ff004bba..e3922cc0 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -42,13 +42,17 @@ describe("expected-state pure builders", () => { id: "dep_bbbbbbbb", serviceId: "svc_1", ipAddress: "10.0.0.2", - status: "sleeping", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", }, { id: "dep_aaaaaaaa", serviceId: "svc_1", ipAddress: "10.0.0.1", - status: "running", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", }, ] as any, services: [ @@ -157,7 +161,9 @@ describe("expected-state pure builders", () => { id: "dep_sleeping", serviceId: "svc_private", ipAddress: "10.0.0.10", - status: "sleeping", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", }, ] as any, services: [ @@ -187,7 +193,9 @@ describe("expected-state pure builders", () => { id: "dep_sleeping", serviceId: "svc_public", ipAddress: "10.0.0.10", - status: "sleeping", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", }, ] as any, services: [ @@ -217,7 +225,9 @@ describe("expected-state pure builders", () => { id: "dep_sleeping", serviceId: "svc_public", ipAddress: "10.0.0.10", - status: "sleeping", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", }, ] as any, services: [ @@ -248,7 +258,9 @@ describe("expected-state pure builders", () => { id: "dep_draining", serviceId: "svc_public", ipAddress: "10.0.0.10", - status: "draining", + runtimeDesiredState: "stopped", + trafficState: "draining", + observedPhase: "sleeping", containerId: null, }, ] as any, @@ -434,22 +446,26 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_proxy", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - status: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_proxy", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, { id: "dep_worker", - serviceId: "svc_1", - serverId: "worker_1", - ipAddress: "10.0.0.20", - status: "healthy", - serverIsProxy: false, - }, + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, ] as any, containers: [ { @@ -505,14 +521,16 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_stateful", - serviceId: "svc_stateful", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - status: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_stateful", + serviceId: "svc_stateful", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, ] as any, containers: [ { @@ -560,22 +578,26 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_old", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - status: "draining", - serverIsProxy: true, - }, + { + id: "dep_old", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "draining", + observedPhase: "sleeping", + serverIsProxy: true, + }, { id: "dep_new", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.11", - status: "running", - serverIsProxy: true, - }, + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.11", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", + serverIsProxy: true, + }, ] as any, containers: [ { deploymentId: "dep_old", desiredState: "stopped" }, @@ -605,14 +627,16 @@ describe("expected-state pure builders", () => { ] as any, ports: [], deployments: [ - { - id: "dep_sleeping", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - status: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_sleeping", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, ] as any, containers: [ { deploymentId: "dep_sleeping", desiredState: "stopped" }, @@ -687,14 +711,16 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_worker", - serviceId: "svc_1", - serverId: "worker_1", - ipAddress: "10.0.0.20", - status: "healthy", - serverIsProxy: false, - }, + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, ] as any, containers: [], }); diff --git a/web/tests/project-deletion.test.ts b/web/tests/project-deletion.test.ts index be247088..519ff95c 100644 --- a/web/tests/project-deletion.test.ts +++ b/web/tests/project-deletion.test.ts @@ -2,8 +2,11 @@ import { describe, expect, it } from "vitest"; import { blocksProjectDeletion } from "@/lib/project-deletion"; describe("project deletion guard", () => { - it("blocks deletion for desired deployments regardless of lifecycle status", () => { - expect(blocksProjectDeletion({ desired: true })).toBe(true); - expect(blocksProjectDeletion({ desired: false })).toBe(false); + it("blocks deletion for deployments that still have runtime intent", () => { + expect(blocksProjectDeletion({ runtimeDesiredState: "running" })).toBe(true); + expect(blocksProjectDeletion({ runtimeDesiredState: "stopped" })).toBe(true); + expect(blocksProjectDeletion({ runtimeDesiredState: "removed" })).toBe( + false, + ); }); }); diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts index 5557060b..65f96954 100644 --- a/web/tests/rollout-helpers.test.ts +++ b/web/tests/rollout-helpers.test.ts @@ -16,68 +16,24 @@ vi.mock("@/lib/work-queue", () => ({ import { isActiveDeploymentForRollout } from "@/lib/inngest/functions/rollout-helpers"; describe("rollout helpers", () => { - it("treats sleeping and waking serverless deployments as active rollout versions", () => { - const deployedServerlessConfig = JSON.stringify({ - source: { type: "image", image: "nginx" }, - stateful: false, - replicas: [], - healthCheck: null, - ports: [], - serverless: { - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - minReadyReplicas: 1, - }, - }); - + it("treats active traffic deployments as the live rollout version", () => { expect( isActiveDeploymentForRollout( - { status: "sleeping" }, + { trafficState: "active" }, { serverlessEnabled: true }, ), ).toBe(true); expect( isActiveDeploymentForRollout( - { status: "waking" }, + { trafficState: "candidate" }, { serverlessEnabled: true }, ), - ).toBe(true); - expect( - isActiveDeploymentForRollout( - { status: "sleeping" }, - { - serverlessEnabled: false, - deployedConfig: deployedServerlessConfig, - }, - ), - ).toBe(true); - expect( - isActiveDeploymentForRollout( - { status: "sleeping" }, - { - serverlessEnabled: false, - deployedConfig: JSON.stringify({ - source: { type: "image", image: "nginx" }, - stateful: false, - replicas: [], - healthCheck: null, - ports: [], - serverless: { - enabled: false, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - minReadyReplicas: 1, - }, - }), - }, - ), ).toBe(false); expect( isActiveDeploymentForRollout( - { status: "healthy" }, + { trafficState: "draining" }, { serverlessEnabled: false }, ), - ).toBe(true); + ).toBe(false); }); }); diff --git a/web/tests/rollout-utils.test.ts b/web/tests/rollout-utils.test.ts index cc3132d4..7dc9bb7e 100644 --- a/web/tests/rollout-utils.test.ts +++ b/web/tests/rollout-utils.test.ts @@ -6,61 +6,50 @@ vi.mock("@/lib/email", () => ({ })); import { - shouldRestoreDrainingDeploymentAsSleeping, - shouldRollBackDeploymentStatus, + shouldRestoreDrainingDeployment, + shouldRollBackDeploymentState, } from "@/lib/inngest/functions/rollout-utils"; -const deployedServerlessConfig = JSON.stringify({ - source: { type: "image", image: "nginx" }, - stateful: false, - replicas: [], - healthCheck: null, - ports: [], - serverless: { - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - minReadyReplicas: 1, - }, -}); - describe("rollout failure helpers", () => { - it("cleans up sleeping and waking rollout deployments", () => { - expect(shouldRollBackDeploymentStatus("sleeping")).toBe(true); - expect(shouldRollBackDeploymentStatus("waking")).toBe(true); - expect(shouldRollBackDeploymentStatus("running")).toBe(true); - expect(shouldRollBackDeploymentStatus("draining")).toBe(false); - expect(shouldRollBackDeploymentStatus("stopped")).toBe(false); + it("cleans up live rollout deployments even after DNS promotion", () => { + expect( + shouldRollBackDeploymentState({ + trafficState: "candidate", + runtimeDesiredState: "running", + }), + ).toBe(true); + expect( + shouldRollBackDeploymentState({ + trafficState: "active", + runtimeDesiredState: "running", + }), + ).toBe(true); + expect( + shouldRollBackDeploymentState({ + trafficState: "candidate", + runtimeDesiredState: "removed", + }), + ).toBe(false); }); - it("restores no-container deployed-serverless drains as sleeping", () => { - const service = { - serverlessEnabled: false, - stateful: true, - deployedConfig: deployedServerlessConfig, - }; - + it("restores draining deployments by traffic intent only", () => { expect( - shouldRestoreDrainingDeploymentAsSleeping( - { containerId: null }, - service, - ), + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "running", + }), ).toBe(true); expect( - shouldRestoreDrainingDeploymentAsSleeping( - { containerId: "ctr_old" }, - service, - ), - ).toBe(false); + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "stopped", + }), + ).toBe(true); expect( - shouldRestoreDrainingDeploymentAsSleeping( - { containerId: null }, - { - serverlessEnabled: false, - stateful: false, - deployedConfig: null, - }, - ), + shouldRestoreDrainingDeployment({ + trafficState: "draining", + runtimeDesiredState: "removed", + }), ).toBe(false); }); }); diff --git a/web/tests/serverless-wake-failures.test.ts b/web/tests/serverless-wake-failures.test.ts index 1c5e6957..71a96e39 100644 --- a/web/tests/serverless-wake-failures.test.ts +++ b/web/tests/serverless-wake-failures.test.ts @@ -10,8 +10,8 @@ describe("serverless wake failure policy", () => { failedStage: "serverless_wake", }), ).toMatchObject({ - status: "sleeping", - desired: true, + runtimeDesiredState: "stopped", + observedPhase: "sleeping", failedStage: null, serverlessWakeFailureCount: 2, }); @@ -25,8 +25,9 @@ describe("serverless wake failure policy", () => { failedStage: "serverless_wake", }), ).toMatchObject({ - status: "failed", - desired: false, + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "failed", failedStage: "serverless_wake", serverlessWakeFailureCount: 3, }); @@ -40,8 +41,9 @@ describe("serverless wake failure policy", () => { failedStage: "serverless_wake_timeout", }), ).toMatchObject({ - status: "failed", - desired: false, + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "failed", failedStage: "serverless_wake_timeout", serverlessWakeFailureCount: 1, }); From 7959b9d4d8bee19c62d2e75b607335600ccfbbe5 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:59:26 +1000 Subject: [PATCH 27/29] Fix explicit stopped deployment projection --- docs/architecture.mdx | 2 +- web/lib/agent-status.ts | 28 ++++- web/lib/agent/expected-state.ts | 88 ++++------------ web/tests/agent-status.test.ts | 22 +++- web/tests/expected-state.test.ts | 173 ++++++++++++++++++------------- 5 files changed, 165 insertions(+), 148 deletions(-) diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 52ea851e..fa9729cb 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -222,7 +222,7 @@ deployments, but proxy agents own the local lifecycle decision: - The proxy gateway wakes local sleeping deployments from expected-state metadata and reports `wake_started` through the next status report. - The proxy agent reports local sleep with a `sleep` status transition after it - removes the local container. + stops the local container. ```mermaid sequenceDiagram diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index b2a36c8f..d2a4a969 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -56,6 +56,23 @@ export function shouldAttachReportedContainer(observedPhase: ObservedPhase) { ); } +export function getStoppedContainerReportUpdate(deployment: { + runtimeDesiredState: string; +}) { + if (deployment.runtimeDesiredState === "stopped") { + return { + containerId: null, + observedPhase: "sleeping" as const, + healthStatus: null, + }; + } + + return { + observedPhase: "stopped" as const, + healthStatus: "none" as const, + }; +} + function isMigrationTargetStarting(status: string | null | undefined) { return status === "deploying_target" || status === "starting"; } @@ -385,7 +402,8 @@ function getInvalidServerlessTransitionReason({ | undefined; }) { if (!deployment) return "deployment not found"; - if (deployment.serverId !== serverId) return "deployment belongs to another server"; + if (deployment.serverId !== serverId) + return "deployment belongs to another server"; if (!deployment.serverIsProxy) return "server is not a proxy"; if (!getDeployedServerlessConfig(deployment).enabled) { return "service is not serverless"; @@ -423,7 +441,10 @@ function getInvalidServerlessTransitionReason({ return null; } -async function emitDeploymentStatusChanged(deploymentId: string, serviceId: string) { +async function emitDeploymentStatusChanged( + deploymentId: string, + serviceId: string, +) { await inngest.send( inngestEvents.resourceStatusChanged.create({ type: "deployment", @@ -677,8 +698,7 @@ export async function applyStatusReport( } if (container.status === "stopped") { - updateFields.observedPhase = "stopped"; - updateFields.healthStatus = "none"; + Object.assign(updateFields, getStoppedContainerReportUpdate(deployment)); await db .update(deployments) .set(updateFields) diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index b1ae7719..28d33b4f 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -169,7 +169,7 @@ export async function buildAgentExpectedState( server: Server, ): Promise { const allServices = await getActiveServices(); - const containers = await buildExpectedContainers(server.id, server.isProxy); + const containers = await buildExpectedContainers(server.id); const dnsRecords = await buildDnsRecords(allServices); const traefikConfig = await buildTraefikConfig(server, allServices); const serverless = await buildServerlessExpectedState( @@ -195,7 +195,6 @@ async function getActiveServices() { async function buildExpectedContainers( serverId: string, - serverIsProxy: boolean, ): Promise { const serverDeployments = await db .select() @@ -210,8 +209,8 @@ async function buildExpectedContainers( const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); if (serviceIds.length === 0) return []; - const [activeServices, depPorts, serviceSecrets, volumes, servicePortRows] = - await Promise.all([ + const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all( + [ db .select() .from(services) @@ -235,11 +234,7 @@ async function buildExpectedContainers( }) .from(serviceVolumes) .where(inArray(serviceVolumes.serviceId, serviceIds)), - fetchServicePorts(serviceIds), - ]); - const serverlessRoutableServiceIds = getServerlessRoutableServiceIds( - activeServices, - servicePortRows, + ], ); return buildExpectedContainersFromRows({ @@ -248,8 +243,6 @@ async function buildExpectedContainers( deploymentPorts: depPorts, secrets: serviceSecrets, volumes, - serverIsProxy, - serverlessRoutableServiceIds, }); } @@ -282,27 +275,16 @@ export function buildExpectedContainersFromRows({ deploymentPorts: deploymentPortRows, secrets: secretRows, volumes: volumeRows, - serverIsProxy = true, - serverlessRoutableServiceIds, }: { deployments: Deployment[]; services: Service[]; deploymentPorts: DeploymentPortRow[]; secrets: SecretRow[]; volumes: VolumeRow[]; - serverIsProxy?: boolean; - serverlessRoutableServiceIds?: Set; }): ExpectedContainer[] { const servicesById = new Map( serviceRows.map((service) => [service.id, service]), ); - const sleepableServiceIds = - serverlessRoutableServiceIds ?? - new Set( - serviceRows - .filter((service) => isDeployedServerlessService(service)) - .map((service) => service.id), - ); const portsByDeploymentId = groupBy( deploymentPortRows, (port) => port.deploymentId, @@ -324,12 +306,7 @@ export function buildExpectedContainersFromRows({ serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, desiredState: - serverIsProxy && - isDeployedServerlessService(service) && - sleepableServiceIds.has(service.id) && - dep.runtimeDesiredState === "stopped" - ? "stopped" - : "running", + dep.runtimeDesiredState === "stopped" ? "stopped" : "running", image: normalizeImage(service.image), ipAddress: dep.ipAddress, ports: (portsByDeploymentId.get(dep.id) ?? []) @@ -353,7 +330,7 @@ export function buildExpectedContainersFromRows({ resourceMemoryLimitMb: service.resourceMemoryLimitMb, }, ]; - }); + }); } async function buildServerlessExpectedState( @@ -361,7 +338,10 @@ async function buildServerlessExpectedState( allServices: Service[], containers: ExpectedContainer[], ): Promise<{ routes: ServerlessRoute[] }> { - if (!server.isProxy || !hasAgentCapability(server, SERVERLESS_GATEWAY_CAPABILITY)) { + if ( + !server.isProxy || + !hasAgentCapability(server, SERVERLESS_GATEWAY_CAPABILITY) + ) { return { routes: [] }; } @@ -460,8 +440,7 @@ export function buildServerlessRoutesFromRows({ const upstreams = serviceDeployments .filter( (deployment) => - deployment.ipAddress && - isDeploymentRoutable(deployment), + deployment.ipAddress && isDeploymentRoutable(deployment), ) .map((deployment) => ({ deploymentId: deployment.id, @@ -541,8 +520,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { SERVERLESS_GATEWAY_CAPABILITY, ); const [ports, routableDeployments, proxyHostedServerlessDeployments] = - await Promise.all( - [ + await Promise.all([ serviceIds.length > 0 ? db .select() @@ -560,10 +538,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - inArray( - deployments.runtimeDesiredState, - runtimeExpectedStates, - ), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), inArray(deployments.trafficState, activeTrafficStates), inArray(deployments.observedPhase, observedReadyPhases), ), @@ -581,16 +556,12 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { and( inArray(deployments.serviceId, serviceIds), eq(servers.isProxy, true), - inArray( - deployments.runtimeDesiredState, - runtimeExpectedStates, - ), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), inArray(deployments.trafficState, activeTrafficStates), ), ) : Promise.resolve([]), - ], - ); + ]); const proxyHostedServerlessServiceIds = new Set( proxyHostedServerlessDeployments.map((deployment) => deployment.serviceId), ); @@ -609,9 +580,10 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .map((service) => service.id), ); const serverlessServiceIds = new Set( - [...serverlessProxyRoutedServiceIds].filter((serviceId) => - supportsServerlessGateway && - localProxyHostedServerlessServiceIds.has(serviceId), + [...serverlessProxyRoutedServiceIds].filter( + (serviceId) => + supportsServerlessGateway && + localProxyHostedServerlessServiceIds.has(serviceId), ), ); const serverlessRouteSuppressedServiceIds = new Set( @@ -852,28 +824,6 @@ function getRuntimeServicePortsForRoutes( }); } -function getServerlessRoutableServiceIds( - serviceRows: Service[], - portRows: RouteServicePort[], -) { - return new Set( - serviceRows - .filter((service) => hasDeployedPublicHttpPort(service, portRows)) - .map((service) => service.id), - ); -} - -function hasDeployedPublicHttpPort( - service: Service, - portRows: RouteServicePort[], -) { - if (!isDeployedServerlessService(service)) return false; - const livePorts = portRows.filter((port) => port.serviceId === service.id); - return getRuntimeServicePortsForRoutes(service, livePorts).some( - (port) => port.isPublic && port.protocol === "http" && !!port.domain, - ); -} - function compareServicePorts(a: RouteServicePort, b: RouteServicePort) { return ( a.serviceId.localeCompare(b.serviceId) || diff --git a/web/tests/agent-status.test.ts b/web/tests/agent-status.test.ts index 16da17e6..b3df6253 100644 --- a/web/tests/agent-status.test.ts +++ b/web/tests/agent-status.test.ts @@ -17,7 +17,10 @@ vi.mock("@/lib/work-queue", () => ({ enqueueWork: vi.fn(), })); -import { shouldAttachReportedContainer } from "@/lib/agent-status"; +import { + getStoppedContainerReportUpdate, + shouldAttachReportedContainer, +} from "@/lib/agent-status"; describe("agent status serverless attachment", () => { it("does not attach reported containers to sleeping deployments", () => { @@ -27,4 +30,21 @@ describe("agent status serverless attachment", () => { expect(shouldAttachReportedContainer("sleeping")).toBe(false); expect(shouldAttachReportedContainer("failed")).toBe(false); }); + + it("preserves sleeping observation for intended-stopped container reports", () => { + expect( + getStoppedContainerReportUpdate({ runtimeDesiredState: "stopped" }), + ).toEqual({ + containerId: null, + observedPhase: "sleeping", + healthStatus: null, + }); + + expect( + getStoppedContainerReportUpdate({ runtimeDesiredState: "running" }), + ).toEqual({ + observedPhase: "stopped", + healthStatus: "none", + }); + }); }); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index e3922cc0..33f5ca89 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -177,7 +177,6 @@ describe("expected-state pure builders", () => { deploymentPorts: [], secrets: [], volumes: [], - serverlessRoutableServiceIds: new Set(), }); expect(containers[0]).toMatchObject({ @@ -186,6 +185,37 @@ describe("expected-state pure builders", () => { }); }); + it("projects stopped runtime intent without checking route eligibility", () => { + const containers = buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_stopped", + serviceId: "svc_private", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + }, + ] as any, + services: [ + { + id: "svc_private", + name: "private-api", + image: "nginx", + serverlessEnabled: false, + }, + ] as any, + deploymentPorts: [], + secrets: [], + volumes: [], + }); + + expect(containers[0]).toMatchObject({ + deploymentId: "dep_stopped", + desiredState: "stopped", + }); + }); + it("marks public serverless deployments stopped while sleeping", () => { const containers = buildExpectedContainersFromRows({ deployments: [ @@ -209,7 +239,6 @@ describe("expected-state pure builders", () => { deploymentPorts: [], secrets: [], volumes: [], - serverlessRoutableServiceIds: new Set(["svc_public"]), }); expect(containers[0]).toMatchObject({ @@ -242,7 +271,6 @@ describe("expected-state pure builders", () => { deploymentPorts: [], secrets: [], volumes: [], - serverlessRoutableServiceIds: new Set(["svc_public"]), }); expect(containers[0]).toMatchObject({ @@ -275,7 +303,6 @@ describe("expected-state pure builders", () => { deploymentPorts: [], secrets: [], volumes: [], - serverlessRoutableServiceIds: new Set(["svc_public"]), }); expect(containers[0]).toMatchObject({ @@ -446,26 +473,26 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_proxy", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - runtimeDesiredState: "stopped", - trafficState: "active", - observedPhase: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_proxy", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, { id: "dep_worker", - serviceId: "svc_1", - serverId: "worker_1", - ipAddress: "10.0.0.20", - runtimeDesiredState: "running", - trafficState: "active", - observedPhase: "healthy", - serverIsProxy: false, - }, + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, ] as any, containers: [ { @@ -521,16 +548,16 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_stateful", - serviceId: "svc_stateful", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - runtimeDesiredState: "stopped", - trafficState: "active", - observedPhase: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_stateful", + serviceId: "svc_stateful", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, ] as any, containers: [ { @@ -578,26 +605,26 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_old", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - runtimeDesiredState: "stopped", - trafficState: "draining", - observedPhase: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_old", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "draining", + observedPhase: "sleeping", + serverIsProxy: true, + }, { id: "dep_new", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.11", - runtimeDesiredState: "running", - trafficState: "active", - observedPhase: "running", - serverIsProxy: true, - }, + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.11", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "running", + serverIsProxy: true, + }, ] as any, containers: [ { deploymentId: "dep_old", desiredState: "stopped" }, @@ -606,9 +633,9 @@ describe("expected-state pure builders", () => { }); expect(routes[0]?.localDeploymentIds).toEqual(["dep_new"]); - expect(routes[0]?.upstreams.map((upstream) => upstream.deploymentId)).toEqual([ - "dep_new", - ]); + expect( + routes[0]?.upstreams.map((upstream) => upstream.deploymentId), + ).toEqual(["dep_new"]); }); it("keeps wake metadata from deployed serverless config while live settings are disabled", () => { @@ -627,16 +654,16 @@ describe("expected-state pure builders", () => { ] as any, ports: [], deployments: [ - { - id: "dep_sleeping", - serviceId: "svc_1", - serverId: "proxy_1", - ipAddress: "10.0.0.10", - runtimeDesiredState: "stopped", - trafficState: "active", - observedPhase: "sleeping", - serverIsProxy: true, - }, + { + id: "dep_sleeping", + serviceId: "svc_1", + serverId: "proxy_1", + ipAddress: "10.0.0.10", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "sleeping", + serverIsProxy: true, + }, ] as any, containers: [ { deploymentId: "dep_sleeping", desiredState: "stopped" }, @@ -711,16 +738,16 @@ describe("expected-state pure builders", () => { }, ] as any, deployments: [ - { - id: "dep_worker", - serviceId: "svc_1", - serverId: "worker_1", - ipAddress: "10.0.0.20", - runtimeDesiredState: "running", - trafficState: "active", - observedPhase: "healthy", - serverIsProxy: false, - }, + { + id: "dep_worker", + serviceId: "svc_1", + serverId: "worker_1", + ipAddress: "10.0.0.20", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, ] as any, containers: [], }); From aa42abb3cfb3db59dcebdaddabc506346108492b Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:39:46 +1000 Subject: [PATCH 28/29] Tighten running-only route upstreams --- web/actions/projects.ts | 3 ++- web/lib/agent/expected-state.ts | 8 +++++--- web/lib/project-deletion.ts | 1 + web/tests/expected-state.test.ts | 10 ++++++++++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/web/actions/projects.ts b/web/actions/projects.ts index ed695da2..3e3865c8 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1080,7 +1080,6 @@ export async function updateServiceServerlessSettings( serverlessMinReadyReplicas: validated.minReadyReplicas, }) .where(eq(services.id, serviceId)); - }); return { success: true }; @@ -1259,6 +1258,8 @@ export async function stopService(serviceId: string) { ); for (const dep of desiredDeployments) { + // User stop is teardown; runtimeDesiredState "stopped" is reserved for + // serverless sleep where the deployment must remain wakeable. await db .update(deployments) .set(markDeploymentRemoved()) diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 28d33b4f..c4838999 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -440,7 +440,9 @@ export function buildServerlessRoutesFromRows({ const upstreams = serviceDeployments .filter( (deployment) => - deployment.ipAddress && isDeploymentRoutable(deployment), + deployment.ipAddress && + deployment.runtimeDesiredState === "running" && + isDeploymentRoutable(deployment), ) .map((deployment) => ({ deploymentId: deployment.id, @@ -484,7 +486,7 @@ async function buildDnsRecords(allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + eq(deployments.runtimeDesiredState, "running"), inArray(deployments.trafficState, activeTrafficStates), inArray(deployments.observedPhase, observedReadyPhases), ), @@ -538,7 +540,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { .where( and( inArray(deployments.serviceId, serviceIds), - inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + eq(deployments.runtimeDesiredState, "running"), inArray(deployments.trafficState, activeTrafficStates), inArray(deployments.observedPhase, observedReadyPhases), ), diff --git a/web/lib/project-deletion.ts b/web/lib/project-deletion.ts index fb133926..5b11f732 100644 --- a/web/lib/project-deletion.ts +++ b/web/lib/project-deletion.ts @@ -6,5 +6,6 @@ type ProjectDeletionDeployment = Pick< >; export function blocksProjectDeletion(deployment: ProjectDeletionDeployment) { + // Sleeping serverless deployments still have runtime intent and can wake. return deployment.runtimeDesiredState !== "removed"; } diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 33f5ca89..52d93b25 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -493,6 +493,16 @@ describe("expected-state pure builders", () => { observedPhase: "healthy", serverIsProxy: false, }, + { + id: "dep_stopped_stale_ready", + serviceId: "svc_1", + serverId: "worker_2", + ipAddress: "10.0.0.30", + runtimeDesiredState: "stopped", + trafficState: "active", + observedPhase: "healthy", + serverIsProxy: false, + }, ] as any, containers: [ { From 5fb584fa1a8097c0dec77519e72aa9b2bc5b7856 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:46:25 +1000 Subject: [PATCH 29/29] Show redeploy for sleeping deployments --- .../[env]/services/[serviceId]/page.tsx | 52 +++++++-------- web/lib/service-deployment-actions.ts | 38 +++++++++++ web/tests/service-deployment-actions.test.ts | 63 +++++++++++++++++++ 3 files changed, 124 insertions(+), 29 deletions(-) create mode 100644 web/lib/service-deployment-actions.ts create mode 100644 web/tests/service-deployment-actions.test.ts diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx index 2aab06d2..278395c1 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/page.tsx @@ -46,6 +46,7 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { getServiceDeploymentActionState } from "@/lib/service-deployment-actions"; type ConfirmAction = "redeploy" | "stop" | "delete" | null; @@ -121,18 +122,7 @@ export default function DeploymentsPage() { }, }; - const hasRunningDeployments = service.deployments.some( - (d) => d.observedPhase === "running" || d.observedPhase === "healthy", - ); - const hasStoppedOrFailedDeployments = - !hasRunningDeployments && - service.deployments.some( - (d) => - d.runtimeDesiredState === "removed" || d.observedPhase === "failed", - ); - const canStartAll = - hasStoppedOrFailedDeployments && - (service.configuredReplicas || []).length > 0; + const deploymentActions = getServiceDeploymentActionState(service); return (
@@ -161,8 +151,8 @@ export default function DeploymentsPage() { projectSlug={projectSlug} envName={envName} actions={ - service.deployments.length > 0 ? ( - hasRunningDeployments ? ( + deploymentActions.hasDeployments ? ( + deploymentActions.hasExpectedDeployments ? (