From 0835a4aac97b84cfdcd3c5afa24e791cfa860414 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 13 Jul 2026 18:21:44 +1000 Subject: [PATCH 01/10] Show service traffic totals for metric range --- .../details/service-details-overview.tsx | 24 ++++--- web/lib/victoria-metrics.ts | 46 +++++++++++++ .../victoria-metrics-service-metrics.test.ts | 68 ++++++++++++++++++- 3 files changed, 124 insertions(+), 14 deletions(-) diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index 6eeb344..ca0f51e 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -40,6 +40,8 @@ export type ServiceMetricsResponse = { windowEnd: string; stepSeconds: number; totalRequests: number; + totalIngressBytes: number | null; + totalEgressBytes: number | null; statusCodes: string[]; buckets: Array<{ timestamp: string; @@ -1013,10 +1015,12 @@ function buildServiceMetricSummaryItems( hasMetricData: boolean, rangeLabel = "24h", ): ServiceMetricSummaryItem[] { + const summaryRangeLabel = stats?.range ?? rangeLabel; + if (mode === "requests") { return [ { - label: `requests in ${rangeLabel}`, + label: `requests in ${summaryRangeLabel}`, value: hasMetricData && stats ? formatCompactNumber(stats.totalRequests) @@ -1061,18 +1065,16 @@ function buildServiceMetricSummaryItems( if (mode === "traffic") { return [ { - label: "ingress", - value: formatNullableMetric( - getLatestValue(rows, "ingressBytesPerSecond"), - (value) => `${formatBytes(value)}/s`, - ), + label: `ingress in ${summaryRangeLabel}`, + value: hasMetricData + ? formatNullableMetric(stats?.totalIngressBytes ?? null, formatBytes) + : "-", }, { - label: "egress", - value: formatNullableMetric( - getLatestValue(rows, "egressBytesPerSecond"), - (value) => `${formatBytes(value)}/s`, - ), + label: `egress in ${summaryRangeLabel}`, + value: hasMetricData + ? formatNullableMetric(stats?.totalEgressBytes ?? null, formatBytes) + : "-", }, ]; } diff --git a/web/lib/victoria-metrics.ts b/web/lib/victoria-metrics.ts index 2bc4b7d..a164c83 100644 --- a/web/lib/victoria-metrics.ts +++ b/web/lib/victoria-metrics.ts @@ -79,6 +79,8 @@ export type ServiceMetrics = { windowEnd: string; stepSeconds: number; totalRequests: number; + totalIngressBytes: number | null; + totalEgressBytes: number | null; statusCodes: string[]; buckets: ServiceMetricsBucket[]; }; @@ -310,6 +312,8 @@ export function createEmptyServiceMetrics( windowEnd: window.end.toISOString(), stepSeconds: window.stepSeconds, totalRequests: 0, + totalIngressBytes: null, + totalEgressBytes: null, statusCodes: [], buckets: [], }; @@ -328,6 +332,7 @@ export async function queryServiceMetrics(options: { const serviceMatcher = buildTraefikServiceMatcher(options.serviceId); const serviceId = escapePromQL(options.serviceId); const rangeWindow = formatPromDuration(window.stepSeconds); + const totalWindow = formatPromDuration(window.durationMs / 1000); const traefikFilter = `service=~"${serviceMatcher}"`; const queryStart = addMilliseconds( window.start, @@ -345,6 +350,8 @@ export async function queryServiceMetrics(options: { cpuResults, memoryPercentResults, memoryBytesResults, + totalIngressBytes, + totalEgressBytes, ] = await Promise.all([ queryRangePromQL(endpoint, { query: `sum by (code) (increase(traefik_service_requests_total{${traefikFilter}}[${rangeWindow}]))`, @@ -406,6 +413,16 @@ export async function queryServiceMetrics(options: { end: window.end, stepSeconds: window.stepSeconds, }).catch(() => []), + queryInstantPromQL( + endpoint, + `sum(increase(traefik_service_requests_bytes_total{${traefikFilter}}[${totalWindow}]))`, + window.end, + ).catch(() => null), + queryInstantPromQL( + endpoint, + `sum(increase(traefik_service_responses_bytes_total{${traefikFilter}}[${totalWindow}]))`, + window.end, + ).catch(() => null), ]); const buckets = createServiceMetricBuckets(window); @@ -468,11 +485,40 @@ export async function queryServiceMetrics(options: { (total, bucket) => total + bucket.totalRequests, 0, ), + totalIngressBytes, + totalEgressBytes, statusCodes: sortStatusCodes([...statusCodes]), buckets, }; } +async function queryInstantPromQL( + endpoint: EndpointConfig, + query: string, + time: Date, +): Promise { + const url = new URL(`${endpoint.url}/api/v1/query`); + url.searchParams.set("query", query); + url.searchParams.set("time", String(Math.floor(time.getTime() / 1000))); + + const response = await fetch(url.toString(), buildFetchOptions(endpoint)); + if (!response.ok) { + throw new Error( + `Failed to query metrics: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as VictoriaInstantResponse; + if (data.status !== "success") { + throw new Error(data.error || "Failed to query metrics"); + } + + const rawValue = data.data?.result?.[0]?.value?.[1]; + if (rawValue === undefined) return null; + const value = Number.parseFloat(rawValue); + return Number.isFinite(value) ? value : null; +} + async function queryInstantMetric( endpoint: EndpointConfig, metricName: string, diff --git a/web/tests/victoria-metrics-service-metrics.test.ts b/web/tests/victoria-metrics-service-metrics.test.ts index 40010cc..042837c 100644 --- a/web/tests/victoria-metrics-service-metrics.test.ts +++ b/web/tests/victoria-metrics-service-metrics.test.ts @@ -41,6 +41,8 @@ describe("VictoriaMetrics service metrics", () => { windowEnd: "2026-07-02T01:00:00.000Z", stepSeconds: 60, totalRequests: 0, + totalIngressBytes: null, + totalEgressBytes: null, statusCodes: [], buckets: [], }); @@ -56,9 +58,21 @@ describe("VictoriaMetrics service metrics", () => { const url = new URL(String(input)); const query = url.searchParams.get("query") || ""; queries.push(query); - starts.push(url.searchParams.get("start") || ""); + if (url.pathname === "/api/v1/query_range") { + starts.push(url.searchParams.get("start") || ""); + } + + expect(["/api/v1/query", "/api/v1/query_range"]).toContain(url.pathname); - expect(url.pathname).toBe("/api/v1/query_range"); + if (url.pathname === "/api/v1/query") { + expect(url.searchParams.get("time")).toBe(String(END_TS)); + if (query.includes("traefik_service_requests_bytes_total")) { + return instantJsonResponse("460"); + } + if (query.includes("traefik_service_responses_bytes_total")) { + return instantJsonResponse("683051"); + } + } if (query.includes("traefik_service_requests_total")) { return jsonResponse([ @@ -91,6 +105,9 @@ describe("VictoriaMetrics service metrics", () => { if (query.includes("histogram_quantile(0.95")) { return jsonResponse([{ metric: {}, values: [[END_TS, "0.123"]] }]); } + if (query.includes("traefik_service_requests_bytes_total")) { + return jsonResponse([{ metric: {}, values: [[END_TS, "512"]] }]); + } if (query.includes("traefik_service_responses_bytes_total")) { return jsonResponse([{ metric: {}, values: [[END_TS, "2048"]] }]); } @@ -108,7 +125,7 @@ describe("VictoriaMetrics service metrics", () => { now: new Date("2026-07-02T12:34:00Z"), }); - expect(fetchMock).toHaveBeenCalledTimes(10); + expect(fetchMock).toHaveBeenCalledTimes(12); expect(queries.some((query) => query.includes("LogSQL"))).toBe(false); expect(queries).toContain( `sum by (code) (increase(traefik_service_requests_total{service=~"^${SERVICE_ID}(@file)?$"}[5m]))`, @@ -119,10 +136,18 @@ describe("VictoriaMetrics service metrics", () => { expect(queries).toContain( `sum(avg_over_time(techulus_service_memory_usage_percent{service_id="${SERVICE_ID}"}[5m]))`, ); + expect(queries).toContain( + `sum(increase(traefik_service_requests_bytes_total{service=~"^${SERVICE_ID}(@file)?$"}[1d]))`, + ); + expect(queries).toContain( + `sum(increase(traefik_service_responses_bytes_total{service=~"^${SERVICE_ID}(@file)?$"}[1d]))`, + ); expect( starts.every((start) => start === String(END_TS - 24 * 60 * 60 + 5 * 60)), ).toBe(true); expect(stats.totalRequests).toBe(21); + expect(stats.totalIngressBytes).toBe(460); + expect(stats.totalEgressBytes).toBe(683051); expect(stats.statusCodes).toEqual(["2xx", "3xx", "4xx", "5xx"]); expect(stats.buckets).toHaveLength(288); expect(stats.windowEnd).toBe("2026-07-02T12:30:00.000Z"); @@ -133,10 +158,38 @@ describe("VictoriaMetrics service metrics", () => { totalRequests: 21, statuses: { "2xx": 5, "3xx": 1, "4xx": 4, "5xx": 11 }, p95ResponseTimeMs: 123, + ingressBytesPerSecond: 512, egressBytesPerSecond: 2048, cpuUsagePercent: 42, }); }); + + it("keeps service metrics available when traffic total queries fail", async () => { + vi.stubEnv("VICTORIA_METRICS_URL", "http://victoria.test"); + vi.stubEnv("VICTORIA_METRICS_PRIVATE_URL", ""); + + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + if (url.pathname === "/api/v1/query") { + return new Response("Unavailable", { status: 503 }); + } + return jsonResponse([]); + }), + ); + + const stats = await queryServiceMetrics({ + serviceId: SERVICE_ID, + range: "1h", + now: new Date("2026-07-02T12:34:00Z"), + }); + + expect(stats.metricsEnabled).toBe(true); + expect(stats.totalIngressBytes).toBeNull(); + expect(stats.totalEgressBytes).toBeNull(); + expect(stats.buckets).toHaveLength(60); + }); }); function jsonResponse(result: unknown[]) { @@ -147,3 +200,12 @@ function jsonResponse(result: unknown[]) { }), ); } + +function instantJsonResponse(value: string) { + return new Response( + JSON.stringify({ + status: "success", + data: { result: [{ metric: {}, value: [END_TS, value] }] }, + }), + ); +} From 5ff4c5616bf1c0f6424fdb1ea44ce49df53665fd Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 13 Jul 2026 18:24:00 +1000 Subject: [PATCH 02/10] Keep metric axes inside chart containers --- web/components/service/details/service-details-overview.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index ca0f51e..9ef2e26 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -316,7 +316,7 @@ export function ServiceMetricsPanel({ margin={{ top: 8, right: 4, - left: getYAxisMargin(chartMode), + left: fixedMode ? 0 : getYAxisMargin(chartMode), bottom: 0, }} > From 3bfcee3ef7ad6730c4a4d819cc3bdc2fd9b640cd Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 13 Jul 2026 18:31:30 +1000 Subject: [PATCH 03/10] Keep metric chart labels inside containers --- web/components/service/details/service-details-overview.tsx | 2 +- web/tests/victoria-metrics-service-metrics.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index 9ef2e26..b9cd6f4 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -315,7 +315,7 @@ export function ServiceMetricsPanel({ data={chartRows} margin={{ top: 8, - right: 4, + right: fixedMode ? 48 : 4, left: fixedMode ? 0 : getYAxisMargin(chartMode), bottom: 0, }} diff --git a/web/tests/victoria-metrics-service-metrics.test.ts b/web/tests/victoria-metrics-service-metrics.test.ts index 042837c..ef489e0 100644 --- a/web/tests/victoria-metrics-service-metrics.test.ts +++ b/web/tests/victoria-metrics-service-metrics.test.ts @@ -54,6 +54,7 @@ describe("VictoriaMetrics service metrics", () => { const queries: string[] = []; const starts: string[] = []; + const instantTimes: string[] = []; const fetchMock = vi.fn(async (input: string | URL | Request) => { const url = new URL(String(input)); const query = url.searchParams.get("query") || ""; @@ -65,7 +66,7 @@ describe("VictoriaMetrics service metrics", () => { expect(["/api/v1/query", "/api/v1/query_range"]).toContain(url.pathname); if (url.pathname === "/api/v1/query") { - expect(url.searchParams.get("time")).toBe(String(END_TS)); + instantTimes.push(url.searchParams.get("time") || ""); if (query.includes("traefik_service_requests_bytes_total")) { return instantJsonResponse("460"); } @@ -126,6 +127,7 @@ describe("VictoriaMetrics service metrics", () => { }); expect(fetchMock).toHaveBeenCalledTimes(12); + expect(instantTimes).toEqual([String(END_TS), String(END_TS)]); expect(queries.some((query) => query.includes("LogSQL"))).toBe(false); expect(queries).toContain( `sum by (code) (increase(traefik_service_requests_total{service=~"^${SERVICE_ID}(@file)?$"}[5m]))`, From 78214067cbb20aa388b606e9d325f2566da3de34 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Mon, 13 Jul 2026 18:45:26 +1000 Subject: [PATCH 04/10] Widen metrics period selector --- .../service/details/service-metrics-page.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/web/components/service/details/service-metrics-page.tsx b/web/components/service/details/service-metrics-page.tsx index 98b3373..538f4be 100644 --- a/web/components/service/details/service-metrics-page.tsx +++ b/web/components/service/details/service-metrics-page.tsx @@ -53,10 +53,17 @@ export function ServiceMetricsPage() {
- }> + + } + > {LABELS[range]} - + setRange(value as typeof range)} From 174367ba02e9307466257e47ad5fbbb8e7f2181d Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:23:35 +1000 Subject: [PATCH 05/10] Implement immutable service revisions --- agent/internal/agent/agent.go | 5 + agent/internal/agent/drift.go | 131 ++++- agent/internal/agent/reconcile_failures.go | 160 ++++++ .../internal/agent/reconcile_failures_test.go | 88 +++ agent/internal/agent/reporting.go | 19 +- .../internal/agent/revision_reconcile_test.go | 226 ++++++++ agent/internal/agent/run.go | 47 +- agent/internal/agent/run_test.go | 36 ++ agent/internal/agent/serverless.go | 17 +- agent/internal/container/runtime.go | 7 +- agent/internal/container/runtime_test.go | 30 ++ agent/internal/container/types.go | 4 + agent/internal/http/client.go | 95 +++- agent/internal/http/client_test.go | 54 ++ agent/internal/reconcile/reconcile.go | 2 + deployment/README.md | 48 ++ deployment/compose.postgres.yml | 4 +- deployment/compose.production.yml | 4 +- web/Dockerfile | 4 +- web/actions/projects.ts | 161 ++++-- .../[serviceId]/rollouts/[rolloutId]/page.tsx | 18 +- .../dashboard/servers/[id]/page.tsx | 1 + web/app/api/projects/[id]/services/route.ts | 64 ++- web/app/api/services/[id]/rollouts/route.ts | 22 +- .../api/v1/agent/builds/[id]/status/route.ts | 5 +- web/app/api/v1/agent/expected-state/route.ts | 50 +- .../server/server-health-details.tsx | 42 +- web/components/server/server-services.tsx | 12 +- .../service/details/rollout-details.tsx | 8 + .../service/details/rollout-history.tsx | 6 + .../service/service-layout-client.tsx | 8 +- web/db/queries.ts | 19 +- web/db/schema.ts | 80 ++- web/db/types.ts | 11 +- web/lib/agent-capabilities.ts | 13 + web/lib/agent-status.ts | 91 ++-- web/lib/agent/expected-state.ts | 507 +++++++++++------- web/lib/cli-service.ts | 2 +- web/lib/deploy-service.ts | 18 +- web/lib/deployment-status.ts | 41 +- web/lib/inngest/functions/build-workflow.ts | 37 +- .../inngest/functions/migration-workflow.ts | 120 +++-- web/lib/inngest/functions/rollout-helpers.ts | 416 +++++++------- web/lib/inngest/functions/rollout-utils.ts | 141 +++-- web/lib/inngest/functions/rollout-workflow.ts | 200 ++++--- .../functions/service-deletion-workflow.ts | 55 +- web/lib/scheduler.ts | 2 +- web/lib/service-config.ts | 124 ++--- web/lib/service-revision-cutover.ts | 168 ++++++ web/lib/service-revision-spec.ts | 185 +++++++ web/lib/service-revisions.ts | 186 +++++++ web/lib/wireguard.ts | 34 +- web/package.json | 1 + web/scripts/cutover-service-revisions.ts | 434 +++++++++++++++ web/tests/agent-capabilities.test.ts | 28 + web/tests/deployment-status.test.ts | 20 +- web/tests/expected-state-route.test.ts | 93 ++++ web/tests/expected-state.test.ts | 468 ++++++++++++---- web/tests/rollout-helpers.test.ts | 38 +- web/tests/service-config.test.ts | 75 ++- web/tests/service-revision-cutover.test.ts | 138 +++++ web/tests/service-revision-spec.test.ts | 107 ++++ 62 files changed, 4145 insertions(+), 1085 deletions(-) create mode 100644 agent/internal/agent/reconcile_failures.go create mode 100644 agent/internal/agent/reconcile_failures_test.go create mode 100644 agent/internal/agent/revision_reconcile_test.go create mode 100644 agent/internal/agent/run_test.go create mode 100644 agent/internal/http/client_test.go create mode 100644 web/lib/agent-capabilities.ts create mode 100644 web/lib/service-revision-cutover.ts create mode 100644 web/lib/service-revision-spec.ts create mode 100644 web/lib/service-revisions.ts create mode 100644 web/scripts/cutover-service-revisions.ts create mode 100644 web/tests/agent-capabilities.test.ts create mode 100644 web/tests/expected-state-route.test.ts create mode 100644 web/tests/service-revision-cutover.test.ts create mode 100644 web/tests/service-revision-spec.test.ts diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index fa85f72..bfece7f 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -64,6 +64,10 @@ type Agent struct { pendingDeploymentErrors []agenthttp.DeploymentError deployLockMutex sync.Mutex deploymentDeployLocks map[string]*sync.Mutex + reconcileFailureMutex sync.Mutex + reconcileFailures map[string]reconcileActionFailure + legacyCutoverHealthWait string + legacyCutoverHealthWaitSince time.Time serverlessMutex sync.Mutex pendingServerlessTransitions []agenthttp.ServerlessTransition pendingServerlessSleep map[string]serverlessTransitionGuard @@ -120,6 +124,7 @@ func NewAgent( IsProxy: isProxy, DisableDNS: disableDNS, deploymentDeployLocks: map[string]*sync.Mutex{}, + reconcileFailures: map[string]reconcileActionFailure{}, pendingServerlessSleep: map[string]serverlessTransitionGuard{}, pendingServerlessWake: map[string]serverlessTransitionGuard{}, } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 18216d8..63025bf 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -26,12 +26,15 @@ const ( actionStopExpectedContainer reconcileActionKind = "stop_expected_container" actionStartContainer reconcileActionKind = "start_container" actionRedeployContainer reconcileActionKind = "redeploy_container" + actionWaitLegacyCutover reconcileActionKind = "wait_legacy_cutover" actionUpdateDNS reconcileActionKind = "update_dns" actionUpdateTraefik reconcileActionKind = "update_traefik" actionUpdateCertificates reconcileActionKind = "update_certificates" actionWriteChallengeRoute reconcileActionKind = "write_challenge_route" actionUpdateWireGuard reconcileActionKind = "update_wireguard" actionStartWireGuard reconcileActionKind = "start_wireguard" + legacyCutoverStabilizationDelay = 30 * time.Second + legacyCutoverMaxWait = 5 * time.Minute ) type reconcileAction struct { @@ -144,25 +147,124 @@ func (a *Agent) handleProcessing() { a.updateDnsInSync(a.expectedState, actual) actions := a.planReconcile(a.expectedState, actual) + actions, waitingForCutoverHealth := a.gateLegacyCutoverRecreations( + actions, + actual, + ) if len(actions) == 0 { + if waitingForCutoverHealth { + return + } + a.clearReconcileFailures() log.Printf("[processing] state converged, transitioning to IDLE") a.transitionToIdle() return } - action := actions[0] + action, eligible, nextRetryAt := a.nextEligibleReconcileAction(actions, time.Now()) + if !eligible { + log.Printf("[processing] all %d pending actions are backed off; next retry at %s", len(actions), nextRetryAt.Format(time.RFC3339)) + return + } if err := a.applyReconcileAction(action); err != nil { - log.Printf("[processing] reconciliation failed: %v, transitioning to IDLE", err) + failure := a.recordReconcileFailure(action, err, time.Now()) + log.Printf("[processing] reconciliation action failed (attempt %d); continuing with remaining actions, retry at %s: %v", failure.Attempts, failure.NextRetryAt.Format(time.RFC3339), err) a.RecordDeploymentError(action.DeploymentID, err) a.RequestStatusReport("reconcile failed") - a.transitionToIdle() return } + a.clearReconcileFailure(action) + if a.legacyCutoverHealthWait == action.DeploymentID && + (action.Kind == actionDeployMissingContainer || + action.Kind == actionStartContainer || + action.Kind == actionRedeployContainer) { + a.legacyCutoverHealthWaitSince = time.Now() + } + if isLegacyCutoverRecreation(action) { + a.legacyCutoverHealthWait = action.DeploymentID + a.legacyCutoverHealthWaitSince = time.Now() + log.Printf("[processing] waiting for deployment %s to stabilize before recreating another legacy container", action.DeploymentID) + } a.RequestStatusReport("reconcile completed") } +func isLegacyCutoverRecreation(action reconcileAction) bool { + return action.Kind == actionRedeployContainer && + action.Actual != nil && + action.Actual.SpecHash == "" && + action.Expected != nil && + action.Expected.ContainerSpecHash != "" +} + +func (a *Agent) gateLegacyCutoverRecreations( + actions []reconcileAction, + actual *ActualState, +) ([]reconcileAction, bool) { + if a.legacyCutoverHealthWait == "" { + return actions, false + } + var waitingExpected *agenthttp.ExpectedContainer + for _, expectedContainer := range a.expectedState.Containers { + if expectedContainer.DeploymentID == a.legacyCutoverHealthWait { + copy := expectedContainer + waitingExpected = © + break + } + } + if waitingExpected == nil { + a.legacyCutoverHealthWait = "" + a.legacyCutoverHealthWaitSince = time.Time{} + return actions, false + } + + var waitingContainer *container.Container + for i := range actual.Containers { + if actual.Containers[i].DeploymentID == a.legacyCutoverHealthWait { + waitingContainer = &actual.Containers[i] + break + } + } + ready := false + if waitingContainer != nil && waitingContainer.State == "running" { + if waitingExpected.HealthCheck != nil { + ready = container.GetHealthStatus(waitingContainer.ID) == "healthy" + } else { + ready = time.Since(a.legacyCutoverHealthWaitSince) >= legacyCutoverStabilizationDelay + } + } + if ready { + log.Printf("[processing] deployment %s is stable; legacy recreation may continue", a.legacyCutoverHealthWait) + a.legacyCutoverHealthWait = "" + a.legacyCutoverHealthWaitSince = time.Time{} + return actions, false + } + + if time.Since(a.legacyCutoverHealthWaitSince) >= legacyCutoverMaxWait { + deploymentID := a.legacyCutoverHealthWait + log.Printf("[processing] deployment %s did not stabilize; releasing legacy recreation gate", deploymentID) + a.legacyCutoverHealthWait = "" + a.legacyCutoverHealthWaitSince = time.Time{} + return []reconcileAction{{ + Kind: actionWaitLegacyCutover, + DeploymentID: deploymentID, + Description: fmt.Sprintf("WAIT for legacy deployment %s to stabilize", deploymentID), + Expected: waitingExpected, + Actual: waitingContainer, + }}, false + } + + filtered := make([]reconcileAction, 0, len(actions)) + for _, action := range actions { + if isLegacyCutoverRecreation(action) { + continue + } + filtered = append(filtered, action) + } + return filtered, true +} + func (a *Agent) updateDnsInSync(expected *agenthttp.ExpectedState, actual *ActualState) { if a.DisableDNS { a.dnsInSync = true @@ -287,7 +389,19 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS continue } - if normalizeImage(exp.Image) != normalizeImage(act.Image) { + if exp.ContainerSpecHash == "" || act.SpecHash != exp.ContainerSpecHash { + reason := "container specification changed" + if act.SpecHash == "" { + reason = "legacy container is missing revision labels" + } + actions = append(actions, reconcileAction{ + Kind: actionRedeployContainer, + Description: fmt.Sprintf("REDEPLOY %s (%s)", exp.Name, reason), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) + } else if normalizeImage(exp.Image) != normalizeImage(act.Image) { actions = append(actions, reconcileAction{ Kind: actionRedeployContainer, Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), @@ -483,9 +597,6 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { } if err := container.Start(action.Actual.ID); err != nil { log.Printf("[reconcile] start failed, will redeploy: %v", err) - if err := container.Stop(action.Actual.ID); err != nil { - log.Printf("[reconcile] warning: failed to stop old container: %v", err) - } if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } @@ -496,9 +607,6 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if action.Actual == nil || action.Expected == nil { return fmt.Errorf("missing container state for %s", action.Kind) } - if err := container.Stop(action.Actual.ID); err != nil { - log.Printf("[reconcile] warning: failed to stop old container: %v", err) - } if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } @@ -556,6 +664,9 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { } return nil + case actionWaitLegacyCutover: + return fmt.Errorf("deployment %s did not stabilize within %s", action.DeploymentID, legacyCutoverMaxWait) + default: return fmt.Errorf("unknown reconcile action: %s", action.Kind) } diff --git a/agent/internal/agent/reconcile_failures.go b/agent/internal/agent/reconcile_failures.go new file mode 100644 index 0000000..6ab86a1 --- /dev/null +++ b/agent/internal/agent/reconcile_failures.go @@ -0,0 +1,160 @@ +package agent + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "time" + + agenthttp "techulus/cloud-agent/internal/http" +) + +const ( + reconcileFailureBaseBackoff = 5 * time.Second + reconcileFailureMaxBackoff = 5 * time.Minute +) + +type reconcileActionFailure struct { + Key string + Kind reconcileActionKind + DeploymentID string + Description string + LastError string + Attempts int + NextRetryAt time.Time +} + +func (a *Agent) reconcileActionKey(action reconcileAction) string { + target := action.DeploymentID + if target == "" && action.Actual != nil { + target = action.Actual.ID + } + + var desired any + switch action.Kind { + case actionDeployMissingContainer, actionStopExpectedContainer, actionStartContainer, actionRedeployContainer: + desired = action.Expected + case actionUpdateDNS: + desired = a.expectedState.Dns + case actionUpdateTraefik: + desired = a.expectedState.Traefik + case actionUpdateCertificates: + desired = a.expectedState.Traefik.Certificates + case actionWriteChallengeRoute: + desired = a.expectedState.Traefik.ChallengeRoute + case actionUpdateWireGuard, actionStartWireGuard: + desired = a.expectedState.Wireguard + default: + desired = action.Description + } + + payload, _ := json.Marshal(desired) + digest := sha256.Sum256(payload) + return fmt.Sprintf("%s:%s:%s", action.Kind, target, hex.EncodeToString(digest[:8])) +} + +func reconcileFailureBackoff(attempts int, key string) time.Duration { + backoff := reconcileFailureBaseBackoff + for i := 1; i < attempts && backoff < reconcileFailureMaxBackoff; i++ { + backoff *= 2 + } + if backoff >= reconcileFailureMaxBackoff { + return reconcileFailureMaxBackoff + } + digest := sha256.Sum256([]byte(key)) + jitterWindow := backoff / 5 + jitter := time.Duration( + binary.BigEndian.Uint64(digest[:8]) % uint64(jitterWindow+1), + ) + if backoff+jitter > reconcileFailureMaxBackoff { + return reconcileFailureMaxBackoff + } + return backoff + jitter +} + +func (a *Agent) recordReconcileFailure(action reconcileAction, err error, now time.Time) reconcileActionFailure { + key := a.reconcileActionKey(action) + a.reconcileFailureMutex.Lock() + defer a.reconcileFailureMutex.Unlock() + + failure := a.reconcileFailures[key] + failure.Key = key + failure.Kind = action.Kind + failure.DeploymentID = action.DeploymentID + failure.Description = action.Description + failure.LastError = err.Error() + failure.Attempts++ + failure.NextRetryAt = now.Add(reconcileFailureBackoff(failure.Attempts, key)) + a.reconcileFailures[key] = failure + return failure +} + +func (a *Agent) clearReconcileFailure(action reconcileAction) { + key := a.reconcileActionKey(action) + a.reconcileFailureMutex.Lock() + defer a.reconcileFailureMutex.Unlock() + delete(a.reconcileFailures, key) +} + +func (a *Agent) clearReconcileFailures() { + a.reconcileFailureMutex.Lock() + defer a.reconcileFailureMutex.Unlock() + clear(a.reconcileFailures) +} + +func (a *Agent) nextEligibleReconcileAction(actions []reconcileAction, now time.Time) (reconcileAction, bool, time.Time) { + activeKeys := make(map[string]struct{}, len(actions)) + keys := make([]string, len(actions)) + for i, action := range actions { + keys[i] = a.reconcileActionKey(action) + activeKeys[keys[i]] = struct{}{} + } + + a.reconcileFailureMutex.Lock() + defer a.reconcileFailureMutex.Unlock() + for key := range a.reconcileFailures { + if _, active := activeKeys[key]; !active { + delete(a.reconcileFailures, key) + } + } + + var earliestRetry time.Time + for i, action := range actions { + failure, failed := a.reconcileFailures[keys[i]] + if !failed || !now.Before(failure.NextRetryAt) { + return action, true, earliestRetry + } + if earliestRetry.IsZero() || failure.NextRetryAt.Before(earliestRetry) { + earliestRetry = failure.NextRetryAt + } + } + + return reconcileAction{}, false, earliestRetry +} + +func (a *Agent) SnapshotReconciliationFailures() []agenthttp.ReconciliationFailure { + a.reconcileFailureMutex.Lock() + defer a.reconcileFailureMutex.Unlock() + + failures := make([]agenthttp.ReconciliationFailure, 0, len(a.reconcileFailures)) + for _, failure := range a.reconcileFailures { + failures = append(failures, agenthttp.ReconciliationFailure{ + Action: string(failure.Kind), + DeploymentID: failure.DeploymentID, + Description: failure.Description, + LastError: failure.LastError, + Attempts: failure.Attempts, + NextRetryAt: failure.NextRetryAt, + }) + } + sort.Slice(failures, func(i, j int) bool { + if failures[i].Action != failures[j].Action { + return failures[i].Action < failures[j].Action + } + return failures[i].DeploymentID < failures[j].DeploymentID + }) + return failures +} diff --git a/agent/internal/agent/reconcile_failures_test.go b/agent/internal/agent/reconcile_failures_test.go new file mode 100644 index 0000000..c6fe9db --- /dev/null +++ b/agent/internal/agent/reconcile_failures_test.go @@ -0,0 +1,88 @@ +package agent + +import ( + "errors" + "testing" + "time" + + agenthttp "techulus/cloud-agent/internal/http" +) + +func TestFailedReconcileActionDoesNotBlockFollowingActions(t *testing.T) { + now := time.Now() + agent := &Agent{ + expectedState: &agenthttp.ExpectedState{}, + reconcileFailures: map[string]reconcileActionFailure{}, + } + badContainer := agenthttp.ExpectedContainer{ + DeploymentID: "bad-deployment", + ContainerSpecHash: "bad-spec", + } + badAction := reconcileAction{ + Kind: actionDeployMissingContainer, + Description: "deploy image that cannot be pulled", + DeploymentID: badContainer.DeploymentID, + Expected: &badContainer, + } + clusterAction := reconcileAction{ + Kind: actionUpdateDNS, + Description: "update DNS", + } + + agent.recordReconcileFailure(badAction, errors.New("image pull failed"), now) + selected, eligible, _ := agent.nextEligibleReconcileAction( + []reconcileAction{badAction, clusterAction}, + now, + ) + + if !eligible { + t.Fatal("expected the independent action to remain eligible") + } + if selected.Kind != actionUpdateDNS { + t.Fatalf("expected DNS action, got %s", selected.Kind) + } + + selected, eligible, _ = agent.nextEligibleReconcileAction( + []reconcileAction{badAction, clusterAction}, + now.Add(2*reconcileFailureBaseBackoff), + ) + if !eligible || selected.Kind != actionDeployMissingContainer { + t.Fatalf("expected failed action to become eligible after backoff, got %s", selected.Kind) + } +} + +func TestChangedContainerSpecClearsActionBackoff(t *testing.T) { + now := time.Now() + agent := &Agent{ + expectedState: &agenthttp.ExpectedState{}, + reconcileFailures: map[string]reconcileActionFailure{}, + } + original := agenthttp.ExpectedContainer{ + DeploymentID: "deployment", + ContainerSpecHash: "original-spec", + } + originalAction := reconcileAction{ + Kind: actionRedeployContainer, + Description: "redeploy original", + DeploymentID: original.DeploymentID, + Expected: &original, + } + agent.recordReconcileFailure(originalAction, errors.New("image pull failed"), now) + + updated := original + updated.ContainerSpecHash = "updated-spec" + updatedAction := originalAction + updatedAction.Description = "redeploy updated" + updatedAction.Expected = &updated + + selected, eligible, _ := agent.nextEligibleReconcileAction( + []reconcileAction{updatedAction}, + now, + ) + if !eligible || selected.Expected.ContainerSpecHash != "updated-spec" { + t.Fatal("expected a changed container specification to bypass stale backoff") + } + if failures := agent.SnapshotReconciliationFailures(); len(failures) != 0 { + t.Fatalf("expected stale failure to be pruned, got %d", len(failures)) + } +} diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index 70f75cc..b9236fe 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -18,7 +18,10 @@ import ( var Version = "dev" -const serverlessGatewayCapability = "serverless_gateway" +const ( + serverlessGatewayCapability = "serverless_gateway" + serviceRevisionCapability = "service_revision_v1" +) var ( agentStartTime = time.Now() @@ -33,9 +36,10 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport Containers: []agenthttp.ContainerStatus{}, DnsInSync: a.dnsInSync, AgentHealth: &agenthttp.AgentHealth{ - Version: Version, - UptimeSecs: int64(time.Since(agentStartTime).Seconds()), - Capabilities: a.agentCapabilities(), + Version: Version, + UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Capabilities: a.agentCapabilities(), + ReconciliationFailures: a.SnapshotReconciliationFailures(), }, } @@ -116,10 +120,11 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport } func (a *Agent) agentCapabilities() []string { - if !a.IsProxy || !a.serverlessGatewayRunning.Load() { - return nil + capabilities := []string{serviceRevisionCapability} + if a.IsProxy && a.serverlessGatewayRunning.Load() { + capabilities = append(capabilities, serverlessGatewayCapability) } - return []string{serverlessGatewayCapability} + return capabilities } func (a *Agent) RecordDeploymentError(deploymentID string, err error) { diff --git a/agent/internal/agent/revision_reconcile_test.go b/agent/internal/agent/revision_reconcile_test.go new file mode 100644 index 0000000..a6cb0c8 --- /dev/null +++ b/agent/internal/agent/revision_reconcile_test.go @@ -0,0 +1,226 @@ +package agent + +import ( + "testing" + "time" + + "techulus/cloud-agent/internal/container" + agenthttp "techulus/cloud-agent/internal/http" +) + +func TestContainerRevisionHashControlsRecreation(t *testing.T) { + tests := []struct { + name string + actualHash string + wantAction bool + }{ + {name: "matching hash", actualHash: "spec-v1", wantAction: false}, + {name: "changed hash", actualHash: "spec-v0", wantAction: true}, + {name: "legacy missing hash", actualHash: "", wantAction: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + agent := &Agent{DisableDNS: true} + expected := &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + { + DeploymentID: "deployment", + ServiceID: "service", + Name: "service-deployment", + Image: "docker.io/library/nginx", + DesiredState: "running", + ContainerSpecHash: "spec-v1", + }, + }, + } + actual := &ActualState{ + Containers: []container.Container{ + { + ID: "container", + DeploymentID: "deployment", + Image: "docker.io/library/nginx", + State: "running", + SpecHash: tt.actualHash, + }, + }, + } + + found := false + for _, action := range agent.planReconcile(expected, actual) { + if action.DeploymentID == "deployment" && action.Kind == actionRedeployContainer { + found = true + } + } + if found != tt.wantAction { + t.Fatalf("redeploy action = %v, want %v", found, tt.wantAction) + } + }) + } +} + +func TestRunningContainerOnlySkipsDeployWhenRevisionSpecMatches(t *testing.T) { + expected := agenthttp.ExpectedContainer{ + RevisionID: "revision-v2", + ContainerSpecHash: "spec-v2", + } + + tests := []struct { + name string + actual container.Container + want bool + }{ + { + name: "matching revision and spec", + actual: container.Container{ + RevisionID: "revision-v2", + SpecHash: "spec-v2", + }, + want: true, + }, + { + name: "legacy container without revision labels", + actual: container.Container{ + Image: "docker.io/library/nginx", + }, + }, + { + name: "changed spec", + actual: container.Container{ + RevisionID: "revision-v2", + SpecHash: "spec-v1", + }, + }, + { + name: "changed revision", + actual: container.Container{ + RevisionID: "revision-v1", + SpecHash: "spec-v2", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := runningContainerMatchesExpectedRevision(tt.actual, expected); got != tt.want { + t.Fatalf("revision/spec match = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLegacyCutoverHealthWaitOnlyDefersOtherLegacyRecreations(t *testing.T) { + healthCheck := &agenthttp.HealthCheck{Cmd: "check"} + agent := &Agent{ + expectedState: &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{ + {DeploymentID: "first", HealthCheck: healthCheck}, + }, + }, + legacyCutoverHealthWait: "first", + legacyCutoverHealthWaitSince: time.Now(), + } + legacyActual := container.Container{DeploymentID: "second"} + legacyExpected := agenthttp.ExpectedContainer{ + DeploymentID: "second", + ContainerSpecHash: "spec-v1", + } + actions := []reconcileAction{ + { + Kind: actionRedeployContainer, + DeploymentID: "second", + Actual: &legacyActual, + Expected: &legacyExpected, + }, + {Kind: actionUpdateDNS}, + } + + filtered, waiting := agent.gateLegacyCutoverRecreations(actions, &ActualState{}) + if !waiting { + t.Fatal("expected cutover health gate to remain active") + } + if len(filtered) != 1 || filtered[0].Kind != actionUpdateDNS { + t.Fatalf("expected cluster reconciliation to continue, got %+v", filtered) + } +} + +func TestLegacyCutoverStabilizationTimeoutBecomesVisibleFailure(t *testing.T) { + agent := &Agent{ + expectedState: &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{{DeploymentID: "first"}}, + }, + legacyCutoverHealthWait: "first", + legacyCutoverHealthWaitSince: time.Now().Add(-legacyCutoverMaxWait), + } + + filtered, waiting := agent.gateLegacyCutoverRecreations( + []reconcileAction{{ + Kind: actionRedeployContainer, + DeploymentID: "second", + Actual: &container.Container{DeploymentID: "second"}, + Expected: &agenthttp.ExpectedContainer{ + DeploymentID: "second", + ContainerSpecHash: "spec-v1", + }, + }}, + &ActualState{}, + ) + + if waiting || len(filtered) != 1 || filtered[0].Kind != actionWaitLegacyCutover { + t.Fatalf("expected a visible stabilization failure action, got %+v", filtered) + } + if agent.legacyCutoverHealthWait != "" { + t.Fatal("expected timed-out legacy recreation gate to be released") + } + + nextActions, waiting := agent.gateLegacyCutoverRecreations( + []reconcileAction{{ + Kind: actionRedeployContainer, + DeploymentID: "second", + Actual: &container.Container{DeploymentID: "second"}, + Expected: &agenthttp.ExpectedContainer{ + DeploymentID: "second", + ContainerSpecHash: "spec-v1", + }, + }}, + &ActualState{}, + ) + if waiting || len(nextActions) != 1 || nextActions[0].Kind != actionRedeployContainer { + t.Fatalf("expected remaining legacy recreation to proceed, got %+v", nextActions) + } +} + +func TestLegacyCutoverWithoutHealthCheckWaitsForStabilization(t *testing.T) { + agent := &Agent{ + expectedState: &agenthttp.ExpectedState{ + Containers: []agenthttp.ExpectedContainer{{DeploymentID: "first"}}, + }, + legacyCutoverHealthWait: "first", + legacyCutoverHealthWaitSince: time.Now(), + } + actual := &ActualState{Containers: []container.Container{{ + ID: "container", + DeploymentID: "first", + State: "running", + }}} + actions := []reconcileAction{{ + Kind: actionRedeployContainer, + DeploymentID: "second", + Actual: &container.Container{DeploymentID: "second"}, + Expected: &agenthttp.ExpectedContainer{ + DeploymentID: "second", + ContainerSpecHash: "spec-v1", + }, + }} + + filtered, waiting := agent.gateLegacyCutoverRecreations(actions, actual) + if !waiting || len(filtered) != 0 { + t.Fatal("expected another legacy recreation to remain gated") + } + + agent.legacyCutoverHealthWaitSince = time.Now().Add(-legacyCutoverStabilizationDelay) + filtered, waiting = agent.gateLegacyCutoverRecreations(actions, actual) + if waiting || len(filtered) != len(actions) { + t.Fatal("expected recreation to continue after stabilization") + } +} diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index fbf35ca..e826db9 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -12,6 +12,8 @@ import ( "techulus/cloud-agent/internal/serverless" ) +const startupStatusReportMaxAttempts = 12 + const ( traefikMetricsURL = "http://127.0.0.1:9100/metrics" traefikMetricsInterval = 15 * time.Second @@ -63,6 +65,14 @@ func (a *Agent) Run(ctx context.Context) { cleanupTickerC = cleanupTicker.C } + if !retryStartupStatus(ctx, startupStatusReportMaxAttempts, 5*time.Second, func() bool { + return a.reportStatus("startup") + }) { + if ctx.Err() != nil { + return + } + log.Printf("[status] startup report unavailable after %d attempts; continuing with validated cached state", startupStatusReportMaxAttempts) + } go a.StatusReportLoop(ctx) a.Tick() @@ -94,6 +104,36 @@ func (a *Agent) Run(ctx context.Context) { } } +func retryStartupStatus( + ctx context.Context, + maxAttempts int, + retryDelay time.Duration, + report func() bool, +) bool { + for attempt := 1; attempt <= maxAttempts; attempt++ { + if report() { + return true + } + if attempt == maxAttempts { + return false + } + log.Printf("[status] startup report failed; retrying before first expected-state fetch") + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return false + case <-timer.C: + } + } + return false +} + func (a *Agent) TraefikMetricsLoop(ctx context.Context) { ticker := time.NewTicker(traefikMetricsInterval) defer ticker.Stop() @@ -147,8 +187,6 @@ func (a *Agent) ForwardTraefikMetrics(ctx context.Context) error { } func (a *Agent) StatusReportLoop(ctx context.Context) { - a.reportStatus("startup") - timer := time.NewTimer(nextStatusReportDelay()) defer timer.Stop() @@ -181,7 +219,7 @@ func (a *Agent) RequestStatusReport(reason string) { } } -func (a *Agent) reportStatus(reason string) { +func (a *Agent) reportStatus(reason string) bool { startedAt := time.Now() report := a.BuildStatusReport(true) reportedDeploymentErrorCount := len(report.DeploymentErrors) @@ -190,7 +228,7 @@ func (a *Agent) reportStatus(reason string) { response, err := a.Client.ReportStatus(report, completed, active, serverlessTransitions) if err != nil { log.Printf("[status] failed to report (%s) latency=%s: %v", reason, time.Since(startedAt).Round(time.Millisecond), err) - return + return false } a.ClearReportedDeploymentErrors(reportedDeploymentErrorCount) a.AcknowledgeServerlessTransitions(response.ServerlessTransitionResults, len(serverlessTransitions)) @@ -198,6 +236,7 @@ func (a *Agent) reportStatus(reason string) { a.LogRejectedActiveWorkItems(response.RejectedActiveWorkItems) a.AcceptLeasedWorkItems(response.WorkItems) log.Printf("[status] reported (%s) latency=%s", reason, time.Since(startedAt).Round(time.Millisecond)) + return true } func nextStatusReportDelay() time.Duration { diff --git a/agent/internal/agent/run_test.go b/agent/internal/agent/run_test.go new file mode 100644 index 0000000..0448faf --- /dev/null +++ b/agent/internal/agent/run_test.go @@ -0,0 +1,36 @@ +package agent + +import ( + "context" + "testing" +) + +func TestRetryStartupStatusIsBounded(t *testing.T) { + attempts := 0 + reported := retryStartupStatus(context.Background(), 3, 0, func() bool { + attempts++ + return false + }) + + if reported { + t.Fatal("expected startup reporting to fall through to cached state") + } + if attempts != 3 { + t.Fatalf("attempts = %d, want 3", attempts) + } +} + +func TestRetryStartupStatusStopsAfterSuccess(t *testing.T) { + attempts := 0 + reported := retryStartupStatus(context.Background(), 3, 0, func() bool { + attempts++ + return attempts == 2 + }) + + if !reported { + t.Fatal("expected startup reporting to succeed") + } + if attempts != 2 { + t.Fatalf("attempts = %d, want 2", attempts) + } +} diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index 7bbbc70..cafeb79 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -41,12 +41,10 @@ func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) if actual.DeploymentID != expected.DeploymentID { continue } - if normalizeImage(actual.Image) != normalizeImage(expected.Image) { + if !runningContainerMatchesExpectedRevision(actual, expected) { log.Printf( - "[serverless] recreate deployment %s because image changed (%s -> %s)", + "[serverless] recreate deployment %s because revision/spec changed", Truncate(expected.DeploymentID, 8), - actual.Image, - expected.Image, ) return a.Reconciler.Deploy(expected) } @@ -81,7 +79,7 @@ func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) er if actual.DeploymentID != expected.DeploymentID || actual.State != "running" { continue } - if normalizeImage(actual.Image) == normalizeImage(expected.Image) { + if runningContainerMatchesExpectedRevision(actual, expected) { return nil } } @@ -90,6 +88,15 @@ func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) er }) } +func runningContainerMatchesExpectedRevision( + actual container.Container, + expected agenthttp.ExpectedContainer, +) bool { + return actual.SpecHash != "" && + actual.SpecHash == expected.ContainerSpecHash && + actual.RevisionID == expected.RevisionID +} + func (a *Agent) withDeploymentDeployLock(deploymentID string, fn func() error) error { a.deployLockMutex.Lock() if a.deploymentDeployLocks == nil { diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index 8cb55e6..c86e96c 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -71,8 +71,6 @@ func Deploy(config *DeployConfig) (*DeployResult, error) { image := config.Image - exec.Command("podman", "rm", "-f", config.Name).Run() - logFunc("stdout", fmt.Sprintf("Pulling image: %s", image)) pullCmd := exec.Command("podman", "pull", "--tls-verify=false", image) @@ -92,6 +90,7 @@ func Deploy(config *DeployConfig) (*DeployResult, error) { } args := buildPodmanRunArgs(config, image) + exec.Command("podman", "rm", "-f", config.Name).Run() logFunc("stdout", fmt.Sprintf("Starting container: %s", config.Name)) @@ -156,6 +155,8 @@ func buildPodmanRunArgs(config *DeployConfig, image string) []string { "--label", fmt.Sprintf("techulus.service.id=%s", config.ServiceID), "--label", fmt.Sprintf("techulus.service.name=%s", config.ServiceName), "--label", fmt.Sprintf("techulus.deployment.id=%s", config.DeploymentID), + "--label", fmt.Sprintf("techulus.service.revision=%s", config.RevisionID), + "--label", fmt.Sprintf("techulus.container.spec-hash=%s", config.ContainerSpecHash), ) if config.IPAddress != "" { args = append(args, "--network", NetworkName, "--ip", config.IPAddress) @@ -515,6 +516,8 @@ func List() ([]Container, error) { Labels: pc.Labels, DeploymentID: pc.Labels["techulus.deployment.id"], ServiceID: pc.Labels["techulus.service.id"], + RevisionID: pc.Labels["techulus.service.revision"], + SpecHash: pc.Labels["techulus.container.spec-hash"], } } diff --git a/agent/internal/container/runtime_test.go b/agent/internal/container/runtime_test.go index 1fcf4b1..f517d1a 100644 --- a/agent/internal/container/runtime_test.go +++ b/agent/internal/container/runtime_test.go @@ -1,7 +1,10 @@ package container import ( + "os" + "path/filepath" "slices" + "strings" "testing" ) @@ -73,3 +76,30 @@ func TestBuildPodmanRunArgsDoesNotPublishStaticIPPortsByDefault(t *testing.T) { t.Fatalf("args unexpectedly publish ports: %+v", args) } } + +func TestDeployPullFailureLeavesExistingContainerUntouched(t *testing.T) { + tempDir := t.TempDir() + logPath := filepath.Join(tempDir, "podman.log") + podmanPath := filepath.Join(tempDir, "podman") + script := "#!/bin/sh\nprintf '%s\\n' \"$1\" >> \"$PODMAN_TEST_LOG\"\nif [ \"$1\" = pull ]; then exit 1; fi\n" + if err := os.WriteFile(podmanPath, []byte(script), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", tempDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("PODMAN_TEST_LOG", logPath) + + _, err := Deploy(&DeployConfig{ + Name: "existing-container", + Image: "registry.invalid/missing:latest", + }) + if err == nil { + t.Fatal("expected image pull to fail") + } + commands, readErr := os.ReadFile(logPath) + if readErr != nil { + t.Fatal(readErr) + } + if got := strings.TrimSpace(string(commands)); got != "pull" { + t.Fatalf("podman commands = %q, want only pull before failure", got) + } +} diff --git a/agent/internal/container/types.go b/agent/internal/container/types.go index f057158..a81395c 100644 --- a/agent/internal/container/types.go +++ b/agent/internal/container/types.go @@ -31,6 +31,8 @@ type DeployConfig struct { ServiceID string ServiceName string DeploymentID string + RevisionID string + ContainerSpecHash string WireGuardIP string IPAddress string PortMappings []PortMapping @@ -57,6 +59,8 @@ type Container struct { Labels map[string]string `json:"Labels"` DeploymentID string ServiceID string + RevisionID string + SpecHash string } type containerInspect struct { diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 22ee9dd..363233b 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -3,6 +3,7 @@ package http import ( "bytes" "encoding/json" + "errors" "fmt" "io" "log" @@ -10,6 +11,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "time" "techulus/cloud-agent/internal/crypto" @@ -17,12 +19,22 @@ import ( ) type Client struct { - baseURL string - serverID string - keyPair *crypto.KeyPair - client *http.Client - longClient *http.Client - dataDir string + baseURL string + serverID string + keyPair *crypto.KeyPair + client *http.Client + longClient *http.Client + dataDir string + upgradeLogMutex sync.Mutex + lastUpgradeRequiredLog time.Time +} + +type UpgradeRequiredError struct { + Message string +} + +func (e *UpgradeRequiredError) Error() string { + return fmt.Sprintf("agent upgrade required: %s", e.Message) } func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string) *Client { @@ -70,6 +82,8 @@ type VolumeMount struct { type ExpectedContainer struct { DeploymentID string `json:"deploymentId"` + RevisionID string `json:"revisionId"` + ContainerSpecHash string `json:"containerSpecHash"` ServiceID string `json:"serviceId"` ServiceName string `json:"serviceName"` Name string `json:"name"` @@ -153,9 +167,10 @@ type WireGuardPeer struct { } type ExpectedState struct { - ServerName string `json:"serverName"` - Containers []ExpectedContainer `json:"containers"` - Dns struct { + SchemaVersion int `json:"schemaVersion"` + ServerName string `json:"serverName"` + Containers []ExpectedContainer `json:"containers"` + Dns struct { Records []DnsRecord `json:"records"` } `json:"dns"` Serverless struct { @@ -200,6 +215,9 @@ func (c *Client) loadCachedExpectedState() (*ExpectedState, error) { if err := json.Unmarshal(data, &state); err != nil { return nil, err } + if err := validateExpectedState(&state); err != nil { + return nil, fmt.Errorf("cached expected state is invalid: %w", err) + } return &state, nil } @@ -219,6 +237,9 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusUpgradeRequired { + return nil, &UpgradeRequiredError{Message: string(body)} + } return nil, fmt.Errorf("expected state request failed with status %d: %s", resp.StatusCode, string(body)) } @@ -226,6 +247,9 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { if err := json.NewDecoder(resp.Body).Decode(&state); err != nil { return nil, fmt.Errorf("failed to decode expected state: %w", err) } + if err := validateExpectedState(&state); err != nil { + return nil, err + } if err := c.cacheExpectedState(&state); err != nil { log.Printf("[cache] failed to cache expected state: %v", err) @@ -234,13 +258,38 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { return &state, nil } +func validateExpectedState(state *ExpectedState) error { + if state.SchemaVersion != 1 { + return fmt.Errorf("unsupported expected state schema version %d", state.SchemaVersion) + } + for _, expectedContainer := range state.Containers { + if expectedContainer.DeploymentID == "" { + return fmt.Errorf("expected container is missing deploymentId") + } + if expectedContainer.RevisionID == "" { + return fmt.Errorf("expected container %s is missing revisionId", expectedContainer.DeploymentID) + } + if expectedContainer.ContainerSpecHash == "" { + return fmt.Errorf("expected container %s is missing containerSpecHash", expectedContainer.DeploymentID) + } + } + return nil +} + func (c *Client) GetExpectedStateWithFallback() (*ExpectedState, bool, error) { state, err := c.getExpectedState() if err == nil { return state, false, nil } - log.Printf("[state] CP unreachable, attempting to use cached state: %v", err) + var upgradeRequired *UpgradeRequiredError + if errors.As(err, &upgradeRequired) { + if c.shouldLogUpgradeRequired() { + log.Printf("[state] UPGRADE REQUIRED: control plane rejected this agent; serving cached state until upgraded: %v", err) + } + } else { + log.Printf("[state] CP unreachable, attempting to use cached state: %v", err) + } cachedState, cacheErr := c.loadCachedExpectedState() if cacheErr != nil { return nil, false, fmt.Errorf("CP unreachable and no cached state available: %w (cache error: %v)", err, cacheErr) @@ -249,6 +298,16 @@ func (c *Client) GetExpectedStateWithFallback() (*ExpectedState, bool, error) { return cachedState, true, nil } +func (c *Client) shouldLogUpgradeRequired() bool { + c.upgradeLogMutex.Lock() + defer c.upgradeLogMutex.Unlock() + if time.Since(c.lastUpgradeRequiredLog) < time.Minute { + return false + } + c.lastUpgradeRequiredLog = time.Now() + return true +} + type ContainerStatus struct { DeploymentID string `json:"deploymentId"` ContainerID string `json:"containerId"` @@ -268,9 +327,19 @@ type Resources struct { } type AgentHealth struct { - Version string `json:"version"` - UptimeSecs int64 `json:"uptimeSecs"` - Capabilities []string `json:"capabilities,omitempty"` + Version string `json:"version"` + UptimeSecs int64 `json:"uptimeSecs"` + Capabilities []string `json:"capabilities,omitempty"` + ReconciliationFailures []ReconciliationFailure `json:"reconciliationFailures,omitempty"` +} + +type ReconciliationFailure struct { + Action string `json:"action"` + DeploymentID string `json:"deploymentId,omitempty"` + Description string `json:"description"` + LastError string `json:"lastError"` + Attempts int `json:"attempts"` + NextRetryAt time.Time `json:"nextRetryAt"` } type StatusReport struct { diff --git a/agent/internal/http/client_test.go b/agent/internal/http/client_test.go new file mode 100644 index 0000000..c3451a7 --- /dev/null +++ b/agent/internal/http/client_test.go @@ -0,0 +1,54 @@ +package http + +import "testing" + +func TestValidateExpectedStateRejectsPartialRevisionContract(t *testing.T) { + tests := []struct { + name string + state ExpectedState + }{ + {name: "old schema", state: ExpectedState{}}, + { + name: "missing revision", + state: ExpectedState{ + SchemaVersion: 1, + Containers: []ExpectedContainer{ + {DeploymentID: "deployment", ContainerSpecHash: "hash"}, + }, + }, + }, + { + name: "missing container hash", + state: ExpectedState{ + SchemaVersion: 1, + Containers: []ExpectedContainer{ + {DeploymentID: "deployment", RevisionID: "revision"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := validateExpectedState(&tt.state); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestValidateExpectedStateAcceptsCompleteRevisionContract(t *testing.T) { + state := ExpectedState{ + SchemaVersion: 1, + Containers: []ExpectedContainer{ + { + DeploymentID: "deployment", + RevisionID: "revision", + ContainerSpecHash: "hash", + }, + }, + } + if err := validateExpectedState(&state); err != nil { + t.Fatalf("validateExpectedState() error = %v", err) + } +} diff --git a/agent/internal/reconcile/reconcile.go b/agent/internal/reconcile/reconcile.go index 65635af..4449764 100644 --- a/agent/internal/reconcile/reconcile.go +++ b/agent/internal/reconcile/reconcile.go @@ -69,6 +69,8 @@ func (r *Reconciler) Deploy(exp agenthttp.ExpectedContainer) error { ServiceID: exp.ServiceID, ServiceName: exp.ServiceName, DeploymentID: exp.DeploymentID, + RevisionID: exp.RevisionID, + ContainerSpecHash: exp.ContainerSpecHash, IPAddress: exp.IPAddress, PortMappings: portMappings, PublishLocalPorts: exp.PublishLocalPorts, diff --git a/deployment/README.md b/deployment/README.md index 1d60552..6463a03 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -90,6 +90,54 @@ start, so scaling `WEB_REPLICAS` does not run migrations from every replica. Schema is synced automatically by the one-shot `migrate` service via `drizzle-kit push --force`. This keeps deployment non-interactive, including schema changes Drizzle classifies as data-loss operations such as dropping columns. If schema sync fails, `web` startup is blocked; inspect the failure with `docker compose -f compose.production.yml logs migrate`. +### Immutable service revision cutover + +The service-revision release is a maintenance-window cutover. Do not start the +new agents across the fleet at the same time. + +1. Pause builds, schedules, migrations, restores, and rollout workers. Verify + that no rollout is `queued` or `in_progress`. Verify every service has no + pending secret additions, edits, or removals; deploy or revert those changes + before the maintenance window. +2. Set `EXPECTED_STATE_MAINTENANCE_MODE=true`. Stop every old agent so each + server keeps running its last applied container and cluster state. +3. Stop the old `web` and `inngest` services. Take and verify a PostgreSQL + backup. +4. Run the new `migrate` image. It executes + `scripts/cutover-service-revisions.ts` before `drizzle-kit push`. The script + aborts and rolls back if active rollouts exist or any runtime row cannot be + attached to a baseline revision. +5. Start the new control plane while maintenance mode remains enabled. Install + the new agent binary on every server, but leave the agents stopped. +6. Set `EXPECTED_STATE_MAINTENANCE_MODE=false` and restart the control plane. + Confirm server health identifies every stopped or old node as requiring the + `service_revision_v1` capability. +7. Start one server agent. Its synchronous startup report must register the + capability before it requests expected state. +8. Let that server recreate its pre-cutover containers one at a time. Verify + container health, volume mounts, DNS, HTTP and L4 routes, certificates, + WireGuard, and serverless behavior before starting the next server. +9. Continue through the fleet. Handle stateful and single-replica services in + an explicit maintenance order, then resume workflow producers. + +Each recreation pulls and verifies the image before removing the old container. +A failed pull leaves the old container running and backs off only that action; +other container and cluster reconciliation continues. + +The agent waits for each recreated container before starting the next legacy +recreation. Services with a health check must become healthy; services without +one must remain running through a 30-second stabilization period. + +Pull-before-remove minimizes downtime but cannot eliminate it. The replacement +uses the same static IP, so every container has a stop-to-start gap. Existing +connections to that replica may reset. Multi-replica services remain available +only through healthy replicas on other containers. Single-replica and stateful +services have bounded customer-visible downtime. Verify a healthy remaining +replica before allowing the next replica recreation. + +Keep `services.deployed_config` unchanged for the burn-in release. The +application does not read or write it; it remains only as recovery evidence. + **Future plan:** Once the schema stabilizes, switch to `drizzle-kit generate` + `drizzle-orm migrate()` with pre-generated SQL migration files. This will eliminate the esbuild/drizzle-kit dependency from the production image. ## Commands diff --git a/deployment/compose.postgres.yml b/deployment/compose.postgres.yml index 2ca6c88..0a5226a 100644 --- a/deployment/compose.postgres.yml +++ b/deployment/compose.postgres.yml @@ -90,10 +90,11 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} + - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: postgres: condition: service_healthy - command: ["npx", "drizzle-kit", "push", "--force"] + command: ["sh", "-c", "npx tsx scripts/cutover-service-revisions.ts && npx drizzle-kit push --force"] restart: on-failure web: @@ -118,6 +119,7 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} + - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: migrate: condition: service_completed_successfully diff --git a/deployment/compose.production.yml b/deployment/compose.production.yml index 9060030..1efc382 100644 --- a/deployment/compose.production.yml +++ b/deployment/compose.production.yml @@ -72,7 +72,8 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - command: ["npx", "drizzle-kit", "push", "--force"] + - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} + command: ["sh", "-c", "npx tsx scripts/cutover-service-revisions.ts && npx drizzle-kit push --force"] restart: on-failure web: @@ -97,6 +98,7 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} + - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: migrate: condition: service_completed_successfully diff --git a/web/Dockerfile b/web/Dockerfile index 6206f45..4e7fb73 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -17,7 +17,7 @@ RUN npx next build FROM node:24-slim AS drizzle WORKDIR /drizzle -RUN npm install drizzle-kit drizzle-orm +RUN npm install drizzle-kit drizzle-orm pg tsx FROM node:24-slim AS runner WORKDIR /app @@ -27,6 +27,8 @@ COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static COPY --from=builder /app/public ./public COPY --from=builder /app/db ./db +COPY --from=builder /app/lib/service-revision-spec.ts ./lib/service-revision-spec.ts +COPY --from=builder /app/lib/service-revision-cutover.ts ./lib/service-revision-cutover.ts COPY --from=builder /app/scripts ./scripts COPY --from=builder /app/drizzle.config.ts ./drizzle.config.ts COPY --from=drizzle /drizzle/node_modules ./node_modules diff --git a/web/actions/projects.ts b/web/actions/projects.ts index a3b9ed2..7d419a6 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -13,7 +13,6 @@ import { getService, } from "@/db/queries"; import { - deploymentPorts, deployments, environments, githubRepos, @@ -23,6 +22,7 @@ import { servers, servicePorts, serviceReplicas, + serviceRevisions, services, serviceVolumes, volumeBackups, @@ -32,12 +32,13 @@ import { requireDeveloperRole, verifyDeleteConfirmation } from "@/lib/auth"; import { deployServiceInternal } from "@/lib/deploy-service"; import { isObservedReady, + markDeploymentFailedRemoved, markDeploymentRemoved, runtimeExpectedStates, + selectNewestRevisionId, } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; -import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; import { blocksProjectDeletion } from "@/lib/project-deletion"; import { @@ -545,10 +546,6 @@ async function hardDeleteService(serviceId: string) { containerId: dep.containerId, }); } - - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, dep.id)); } await db.delete(deployments).where(eq(deployments.serviceId, serviceId)); @@ -889,7 +886,7 @@ export async function updateServiceGithubRepo( export async function deployService(serviceId: string) { await requireDeveloperRole(); - return deployServiceInternal(serviceId); + return deployServiceInternal(serviceId, { trigger: "ui" }); } export async function deleteDeployments(serviceId: string) { @@ -1161,9 +1158,6 @@ export async function updateServiceConfig( if (config.ports) { if (config.ports.remove && config.ports.remove.length > 0) { for (const portId of config.ports.remove) { - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.servicePortId, portId)); await db.delete(servicePorts).where(eq(servicePorts.id, portId)); } } @@ -1354,15 +1348,86 @@ export async function restartService(serviceId: string) { export async function abortRollout(serviceId: string) { await requireDeveloperRole(); - const activeRollouts = await db - .select({ id: rollouts.id, status: rollouts.status }) - .from(rollouts) - .where( - and( - eq(rollouts.serviceId, serviceId), - inArray(rollouts.status, ["queued", "in_progress"]), - ), - ); + const { activeRollouts, rolloutDeployments } = await db.transaction( + async (tx) => { + const activeRollouts = await tx + .select({ id: rollouts.id, status: rollouts.status }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .for("update"); + + if (activeRollouts.length === 0) { + return { activeRollouts, rolloutDeployments: [] }; + } + + const activeRolloutIds = activeRollouts.map((rollout) => rollout.id); + const rolloutDeployments = await tx + .select() + .from(deployments) + .where(inArray(deployments.rolloutId, activeRolloutIds)); + + await tx + .update(rollouts) + .set({ + status: "failed", + currentStage: "aborted", + completedAt: new Date(), + }) + .where(inArray(rollouts.id, activeRolloutIds)); + + await tx + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + ), + ); + + await tx + .update(deployments) + .set(markDeploymentFailedRemoved("aborted")) + .where(inArray(deployments.rolloutId, activeRolloutIds)); + + const activeRevisionRows = await tx + .select({ + serviceRevisionId: deployments.serviceRevisionId, + revisionNumber: serviceRevisions.revisionNumber, + }) + .from(deployments) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "active"), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + ), + ); + const activeRevisionCount = new Set( + activeRevisionRows.map((row) => row.serviceRevisionId), + ).size; + if (activeRevisionCount > 1) { + console.error( + `[rollout:abort] found ${activeRevisionCount} active revisions for ${serviceId}; selecting the newest`, + ); + } + await tx + .update(services) + .set({ activeRevisionId: selectNewestRevisionId(activeRevisionRows) }) + .where(eq(services.id, serviceId)); + + return { activeRollouts, rolloutDeployments }; + }, + ); if (activeRollouts.length === 0) { return { success: false, error: "No in-progress rollout found" }; @@ -1370,33 +1435,21 @@ export async function abortRollout(serviceId: string) { const activeRolloutIds = activeRollouts.map((rollout) => rollout.id); - await db - .update(rollouts) - .set({ - status: "failed", - currentStage: "aborted", - completedAt: new Date(), - }) - .where(inArray(rollouts.id, activeRolloutIds)); - for (const rolloutId of activeRolloutIds) { - await inngest.send( - inngestEvents.rolloutCancelled.create({ - rolloutId, - }), - ); + await inngest + .send( + inngestEvents.rolloutCancelled.create({ + rolloutId, + }), + ) + .catch((error) => { + console.error( + `[rollout:${rolloutId}] failed to send cancellation event:`, + error, + ); + }); } - await restoreDrainingDeploymentsForRollback(serviceId); - - const rolloutDeployments = - activeRolloutIds.length > 0 - ? await db - .select() - .from(deployments) - .where(inArray(deployments.rolloutId, activeRolloutIds)) - : []; - const serverContainers = new Map(); for (const dep of rolloutDeployments) { @@ -1411,19 +1464,21 @@ export async function abortRollout(serviceId: string) { await enqueueWork(serverId, "force_cleanup", { serviceId, containerIds, + }).catch((error) => { + console.error( + `[rollout] failed to enqueue cleanup on server ${serverId}:`, + error, + ); }); } - for (const dep of rolloutDeployments) { - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, dep.id)); - } - - if (activeRolloutIds.length > 0) { + const rolloutDeploymentIds = rolloutDeployments.map( + (deployment) => deployment.id, + ); + if (rolloutDeploymentIds.length > 0) { await db .delete(deployments) - .where(inArray(deployments.rolloutId, activeRolloutIds)); + .where(inArray(deployments.id, rolloutDeploymentIds)); } const pendingWork = @@ -1440,11 +1495,11 @@ export async function abortRollout(serviceId: string) { ) : []; - const rolloutDeploymentIds = new Set(rolloutDeployments.map((d) => d.id)); + const rolloutDeploymentIdSet = new Set(rolloutDeploymentIds); const workToDelete = pendingWork.filter((w) => { try { const parsed = JSON.parse(w.payload); - return rolloutDeploymentIds.has(parsed.deploymentId); + return rolloutDeploymentIdSet.has(parsed.deploymentId); } catch { return false; } diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx index 6ecc746..fc544bc 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx @@ -3,7 +3,7 @@ import { notFound } from "next/navigation"; import { SetBreadcrumbs } from "@/components/core/breadcrumb-data"; import { RolloutDetails } from "@/components/service/details/rollout-details"; import { db } from "@/db"; -import { projects, rollouts, services } from "@/db/schema"; +import { projects, rollouts, serviceRevisions, services } from "@/db/schema"; async function getRollout( projectSlug: string, @@ -27,8 +27,22 @@ async function getRollout( if (!service) return null; const rollout = await db - .select() + .select({ + id: rollouts.id, + serviceId: rollouts.serviceId, + serviceRevisionId: rollouts.serviceRevisionId, + status: rollouts.status, + currentStage: rollouts.currentStage, + createdAt: rollouts.createdAt, + completedAt: rollouts.completedAt, + revisionNumber: serviceRevisions.revisionNumber, + contentHash: serviceRevisions.contentHash, + }) .from(rollouts) + .innerJoin( + serviceRevisions, + eq(rollouts.serviceRevisionId, serviceRevisions.id), + ) .where(and(eq(rollouts.id, rolloutId), eq(rollouts.serviceId, serviceId))) .then((r) => r[0]); diff --git a/web/app/(dashboard)/dashboard/servers/[id]/page.tsx b/web/app/(dashboard)/dashboard/servers/[id]/page.tsx index ca9ad17..bed4841 100644 --- a/web/app/(dashboard)/dashboard/servers/[id]/page.tsx +++ b/web/app/(dashboard)/dashboard/servers/[id]/page.tsx @@ -182,6 +182,7 @@ export default async function ServerDetailPage({ networkHealth: server.networkHealth, containerHealth: server.containerHealth, agentHealth: server.agentHealth, + agentCompatibilityStatus: server.agentCompatibilityStatus, }} /> diff --git a/web/app/api/projects/[id]/services/route.ts b/web/app/api/projects/[id]/services/route.ts index 7f69ca9..9897533 100644 --- a/web/app/api/projects/[id]/services/route.ts +++ b/web/app/api/projects/[id]/services/route.ts @@ -1,6 +1,7 @@ export const dynamic = "force-dynamic"; -import { and, desc, eq, isNull } from "drizzle-orm"; +import { createHash } from "node:crypto"; +import { and, desc, eq, inArray, isNull } from "drizzle-orm"; import { headers } from "next/headers"; import { db } from "@/db"; import { @@ -12,12 +13,18 @@ import { servers, servicePorts, serviceReplicas, + serviceRevisions, services, serviceVolumes, volumeBackups, } from "@/db/schema"; import { auth } from "@/lib/auth"; import { getTimestamp } from "@/lib/date"; +import { revisionSpecToDeployedConfig } from "@/lib/service-config"; + +function secretFingerprint(encryptedValue: string) { + return createHash("sha256").update(encryptedValue).digest("hex"); +} export async function GET( request: Request, @@ -60,6 +67,7 @@ export async function GET( volumes, lockedServer, latestBuild, + activeRevision, ] = await Promise.all([ db .select() @@ -83,7 +91,11 @@ export async function GET( .innerJoin(servers, eq(serviceReplicas.serverId, servers.id)) .where(eq(serviceReplicas.serviceId, service.id)), db - .select({ key: secrets.key, updatedAt: secrets.updatedAt }) + .select({ + key: secrets.key, + updatedAt: secrets.updatedAt, + encryptedValue: secrets.encryptedValue, + }) .from(secrets) .where(eq(secrets.serviceId, service.id)), db @@ -112,8 +124,43 @@ export async function GET( .limit(1) .then((r) => r[0] || null) : Promise.resolve(null), + service.activeRevisionId + ? db + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where(eq(serviceRevisions.id, service.activeRevisionId)) + .then((rows) => rows[0] ?? null) + : Promise.resolve(null), ]); + const revisionServerIds = + activeRevision?.specification.placements.map( + (placement) => placement.serverId, + ) ?? []; + const revisionServers = + revisionServerIds.length > 0 + ? await db + .select({ id: servers.id, name: servers.name }) + .from(servers) + .where(inArray(servers.id, revisionServerIds)) + : []; + const serverNames = Object.fromEntries( + revisionServers.map((server) => [server.id, server.name]), + ); + const secretFingerprints = Object.fromEntries( + activeRevision?.specification.secrets.map((secret) => [ + secret.key, + secretFingerprint(secret.encryptedValue), + ]) ?? [], + ); + const activeConfig = activeRevision + ? revisionSpecToDeployedConfig( + activeRevision.specification, + serverNames, + secretFingerprints, + ) + : null; + const deploymentsWithDetails = await Promise.all( serviceDeployments.map(async (deployment) => { const [depPorts, server] = await Promise.all([ @@ -121,13 +168,9 @@ export async function GET( .select({ id: deploymentPorts.id, hostPort: deploymentPorts.hostPort, - containerPort: servicePorts.port, + containerPort: deploymentPorts.containerPort, }) .from(deploymentPorts) - .innerJoin( - servicePorts, - eq(deploymentPorts.servicePortId, servicePorts.id), - ) .where(eq(deploymentPorts.deploymentId, deployment.id)), db .select({ name: servers.name, wireguardIp: servers.wireguardIp }) @@ -200,7 +243,12 @@ export async function GET( ports, configuredReplicas: replicas, deployments: deploymentsWithDetails, - secrets: serviceSecrets, + secrets: serviceSecrets.map((secret) => ({ + key: secret.key, + updatedAt: secret.updatedAt, + fingerprint: secretFingerprint(secret.encryptedValue), + })), + activeConfig, rollouts: serviceRollouts, volumes, lockedServer, diff --git a/web/app/api/services/[id]/rollouts/route.ts b/web/app/api/services/[id]/rollouts/route.ts index b5053a3..3e1c1a4 100644 --- a/web/app/api/services/[id]/rollouts/route.ts +++ b/web/app/api/services/[id]/rollouts/route.ts @@ -1,19 +1,33 @@ export const dynamic = "force-dynamic"; -import { NextRequest, NextResponse } from "next/server"; import { desc, eq } from "drizzle-orm"; +import { type NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; -import { rollouts } from "@/db/schema"; +import { rollouts, serviceRevisions } from "@/db/schema"; export async function GET( - request: NextRequest, + _request: NextRequest, { params }: { params: Promise<{ id: string }> }, ) { const { id: serviceId } = await params; const rolloutsList = await db - .select() + .select({ + id: rollouts.id, + serviceId: rollouts.serviceId, + serviceRevisionId: rollouts.serviceRevisionId, + status: rollouts.status, + currentStage: rollouts.currentStage, + createdAt: rollouts.createdAt, + completedAt: rollouts.completedAt, + revisionNumber: serviceRevisions.revisionNumber, + contentHash: serviceRevisions.contentHash, + }) .from(rollouts) + .innerJoin( + serviceRevisions, + eq(rollouts.serviceRevisionId, serviceRevisions.id), + ) .where(eq(rollouts.serviceId, serviceId)) .orderBy(desc(rollouts.createdAt)) .limit(50); diff --git a/web/app/api/v1/agent/builds/[id]/status/route.ts b/web/app/api/v1/agent/builds/[id]/status/route.ts index 0e53d83..003abe2 100644 --- a/web/app/api/v1/agent/builds/[id]/status/route.ts +++ b/web/app/api/v1/agent/builds/[id]/status/route.ts @@ -352,7 +352,10 @@ export async function POST( ); try { - await deployServiceInternal(build.serviceId); + await deployServiceInternal(build.serviceId, { + trigger: "agent_build", + buildId, + }); } catch (error) { console.error("[build:complete] deployment failed:", error); await db diff --git a/web/app/api/v1/agent/expected-state/route.ts b/web/app/api/v1/agent/expected-state/route.ts index 57593db..cbdc55d 100644 --- a/web/app/api/v1/agent/expected-state/route.ts +++ b/web/app/api/v1/agent/expected-state/route.ts @@ -1,9 +1,10 @@ import { type NextRequest, NextResponse } from "next/server"; -import { - buildAgentExpectedState, - getServer, -} from "@/lib/agent/expected-state"; +import { buildAgentExpectedState, getServer } from "@/lib/agent/expected-state"; import { verifyAgentRequest } from "@/lib/agent-auth"; +import { + getAgentCompatibilityStatus, + SERVICE_REVISION_CAPABILITY, +} from "@/lib/agent-capabilities"; export async function GET(request: NextRequest) { const auth = await verifyAgentRequest(request); @@ -15,6 +16,45 @@ export async function GET(request: NextRequest) { if (!server) { return NextResponse.json({ error: "Server not found" }, { status: 404 }); } + if (process.env.EXPECTED_STATE_MAINTENANCE_MODE === "true") { + return NextResponse.json( + { + error: "Expected state is paused for maintenance", + code: "EXPECTED_STATE_MAINTENANCE", + }, + { + status: 503, + headers: { "Retry-After": "30" }, + }, + ); + } + if (getAgentCompatibilityStatus(server.agentHealth) === "upgrade_required") { + return NextResponse.json( + { + error: "Agent upgrade required", + code: "AGENT_UPGRADE_REQUIRED", + requiredCapabilities: [SERVICE_REVISION_CAPABILITY], + }, + { status: 426 }, + ); + } - return NextResponse.json(await buildAgentExpectedState(server)); + try { + return NextResponse.json(await buildAgentExpectedState(server)); + } catch (error) { + console.error( + `[expected-state] failed to build state for server ${server.id}:`, + error, + ); + return NextResponse.json( + { + error: "Expected state is temporarily unavailable", + code: "EXPECTED_STATE_BUILD_FAILED", + }, + { + status: 503, + headers: { "Retry-After": "15" }, + }, + ); + } } diff --git a/web/components/server/server-health-details.tsx b/web/components/server/server-health-details.tsx index de6973e..fbd774a 100644 --- a/web/components/server/server-health-details.tsx +++ b/web/components/server/server-health-details.tsx @@ -2,6 +2,7 @@ import { Activity, + AlertTriangle, Container, Cpu, HardDrive, @@ -11,6 +12,7 @@ import { import useSWR from "swr"; import { HealthIndicator } from "@/components/cluster/health-indicator"; import { ResourceBar } from "@/components/cluster/resource-bar"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import type { HealthStats, Server } from "@/db/types"; @@ -21,6 +23,7 @@ type ServerHealthData = { networkHealth: Server["networkHealth"]; containerHealth: Server["containerHealth"]; agentHealth: Server["agentHealth"]; + agentCompatibilityStatus: "compatible" | "upgrade_required" | null; }; type ClusterHealthResponse = { @@ -56,6 +59,10 @@ export function ServerHealthDetails({ const containerHealth = serverData?.containerHealth ?? initialData.containerHealth; const agentHealth = serverData?.agentHealth ?? initialData.agentHealth; + const agentCompatibilityStatus = + serverData?.agentCompatibilityStatus ?? + initialData.agentCompatibilityStatus; + const reconciliationFailures = agentHealth?.reconciliationFailures ?? []; if (!healthStats && !networkHealth && !containerHealth && !agentHealth) { return null; @@ -67,6 +74,30 @@ export function ServerHealthDetails({ System Health + {agentCompatibilityStatus === "upgrade_required" && ( + + + Agent upgrade required + + This agent is incompatible with the current deployment protocol. + It will keep its cached state but cannot receive updates until it + is upgraded. + + + )} + {reconciliationFailures.length > 0 && ( + + + Reconciliation delayed + + {reconciliationFailures.length} action + {reconciliationFailures.length === 1 ? " is" : "s are"} being + retried without blocking other server updates. Latest:{" "} + {reconciliationFailures[0]?.description} ( + {reconciliationFailures[0]?.lastError}). + + + )} {healthStats && (
} /> } />
diff --git a/web/components/server/server-services.tsx b/web/components/server/server-services.tsx index 1b1240f..44441cf 100644 --- a/web/components/server/server-services.tsx +++ b/web/components/server/server-services.tsx @@ -1,5 +1,5 @@ -import Link from "next/link"; import { ArrowUpRight, Layers } from "lucide-react"; +import Link from "next/link"; import { Card, CardContent, @@ -47,8 +47,14 @@ export async function ServerServices({ serverId }: { serverId: string }) { href={`/dashboard/projects/${service.projectSlug}/${service.environmentName}/services/${service.serviceId}`} className="flex items-center justify-between gap-3 px-3 py-2.5 text-sm transition-colors hover:bg-muted" > - - {service.serviceName} + + + {service.serviceName} + + + revision {service.revisionNumber} ·{" "} + {service.revisionContentHash.slice(0, 12)} + diff --git a/web/components/service/details/rollout-details.tsx b/web/components/service/details/rollout-details.tsx index d6adce6..3d286df 100644 --- a/web/components/service/details/rollout-details.tsx +++ b/web/components/service/details/rollout-details.tsx @@ -17,6 +17,8 @@ import { formatElapsedDurationBetween, formatRelativeTime } from "@/lib/date"; type RolloutWithDates = Omit & { createdAt: string | Date; completedAt: string | Date | null; + revisionNumber: number; + contentHash: string; }; const STATUS_CONFIG: Record< @@ -135,6 +137,12 @@ export function RolloutDetails({
+
+ Immutable revision + #{rollout.revisionNumber} + · + {rollout.contentHash} +

Rollout Logs

+ + Revision {rollout.revisionNumber} ·{" "} + {rollout.contentHash.slice(0, 12)} + {formatRelativeTime(rollout.createdAt)} Duration:{" "} diff --git a/web/components/service/service-layout-client.tsx b/web/components/service/service-layout-client.tsx index 7d4d0fe..15db849 100644 --- a/web/components/service/service-layout-client.tsx +++ b/web/components/service/service-layout-client.tsx @@ -7,11 +7,7 @@ import useSWR from "swr"; import type { ServiceWithDetails as Service } from "@/db/types"; import { fetcher } from "@/lib/fetcher"; import type { ConfigChange } from "@/lib/service-config"; -import { - buildCurrentConfig, - diffConfigs, - parseDeployedConfig, -} from "@/lib/service-config"; +import { buildCurrentConfig, diffConfigs } from "@/lib/service-config"; import { cn } from "@/lib/utils"; const ACTIVE_BUILD_STATUSES = [ @@ -77,7 +73,7 @@ export function ServiceLayoutClient({ const pendingChanges = useMemo(() => { if (!service) return []; - const deployed = parseDeployedConfig(service.deployedConfig); + const deployed = service.activeConfig ?? null; const replicas = (service.configuredReplicas || []).map((r) => ({ serverId: r.serverId, serverName: r.serverName, diff --git a/web/db/queries.ts b/web/db/queries.ts index 7f70e81..ebe04ae 100644 --- a/web/db/queries.ts +++ b/web/db/queries.ts @@ -5,10 +5,12 @@ import { environments, projects, servers, + serviceRevisions, services, settings, } from "@/db/schema"; import type { HealthStats } from "@/db/types"; +import { getAgentCompatibilityStatus } from "@/lib/agent-capabilities"; import type { ControlPlaneUpdateState, ControlPlaneUpgradeState, @@ -125,7 +127,15 @@ export async function getServerDetails(id: string) { .where(eq(servers.id, id)); const server = serverResults[0]; - return server ? { ...server, healthStats: null } : null; + return server + ? { + ...server, + healthStats: null, + agentCompatibilityStatus: getAgentCompatibilityStatus( + server.agentHealth, + ), + } + : null; } export async function getClusterHealth() { @@ -155,6 +165,7 @@ export async function getClusterHealth() { const serversWithHealth = allServers.map((server) => ({ ...server, healthStats: metricSnapshotToHealthStats(metricsByServer.get(server.id)), + agentCompatibilityStatus: getAgentCompatibilityStatus(server.agentHealth), })); const serversWithCurrentMetrics = serversWithHealth.filter( (server) => server.status === "online" && server.healthStats, @@ -233,6 +244,8 @@ export async function getServerServices(serverId: string) { .selectDistinctOn([services.id], { deploymentId: deployments.id, deploymentStatus: deployments.observedPhase, + revisionNumber: serviceRevisions.revisionNumber, + revisionContentHash: serviceRevisions.contentHash, serviceId: services.id, serviceName: services.name, serviceImage: services.image, @@ -243,6 +256,10 @@ export async function getServerServices(serverId: string) { }) .from(deployments) .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) .innerJoin(projects, eq(services.projectId, projects.id)) .innerJoin(environments, eq(services.environmentId, environments.id)) .where(eq(deployments.serverId, serverId)); diff --git a/web/db/schema.ts b/web/db/schema.ts index a3bc22c..7586b83 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -1,7 +1,9 @@ import { relations, sql } from "drizzle-orm"; import { + type AnyPgColumn, bigint, boolean, + foreignKey, index, integer, jsonb, @@ -9,8 +11,10 @@ import { real, text, timestamp, + unique, uniqueIndex, } from "drizzle-orm/pg-core"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; export const user = pgTable("user", { id: text("id").primaryKey(), @@ -303,6 +307,14 @@ export type AgentHealth = { version: string; uptimeSecs: number; capabilities?: string[]; + reconciliationFailures?: Array<{ + action: string; + deploymentId?: string; + description: string; + lastError: string; + attempts: number; + nextRetryAt: string; + }>; }; export type AgentUpgradeStatus = @@ -417,6 +429,10 @@ export const services = pgTable("services", { serverlessWakeTimeoutSeconds: integer("serverless_wake_timeout_seconds") .notNull() .default(300), + activeRevisionId: text("active_revision_id").references( + (): AnyPgColumn => serviceRevisions.id, + { onDelete: "set null" }, + ), deployedConfig: text("deployed_config"), deploymentSchedule: text("deployment_schedule"), lastScheduledDeploymentRunAt: timestamp("last_scheduled_deployment_run_at", { @@ -547,6 +563,44 @@ export const secrets = pgTable("secrets", { .notNull(), }); +export const serviceRevisions = pgTable( + "service_revisions", + { + id: text("id").primaryKey(), + serviceId: text("service_id") + .notNull() + .references(() => services.id, { onDelete: "cascade" }), + revisionNumber: integer("revision_number").notNull(), + schemaVersion: integer("schema_version").notNull(), + specification: jsonb("specification") + .$type() + .notNull(), + contentHash: text("content_hash").notNull(), + sourceMetadata: + jsonb("source_metadata").$type< + Record + >(), + createdAt: timestamp("created_at", { withTimezone: true }) + .defaultNow() + .notNull(), + }, + (table) => [ + uniqueIndex("service_revisions_service_revision_number_idx").on( + table.serviceId, + table.revisionNumber, + ), + uniqueIndex("service_revisions_service_content_hash_idx").on( + table.serviceId, + table.contentHash, + ), + unique("service_revisions_id_service_id_unique").on( + table.id, + table.serviceId, + ), + index("service_revisions_service_id_idx").on(table.serviceId), + ], +); + export const deployments = pgTable( "deployments", { @@ -554,6 +608,7 @@ export const deployments = pgTable( serviceId: text("service_id") .notNull() .references(() => services.id, { onDelete: "cascade" }), + serviceRevisionId: text("service_revision_id").notNull(), serverId: text("server_id") .notNull() .references(() => servers.id, { onDelete: "cascade" }), @@ -611,12 +666,22 @@ export const deployments = pgTable( index("deployments_container_id_idx").on(table.containerId), index("deployments_rollout_id_idx").on(table.rolloutId), index("deployments_service_id_idx").on(table.serviceId), + index("deployments_service_revision_id_idx").on(table.serviceRevisionId), index("deployments_server_id_idx").on(table.serverId), + uniqueIndex("deployments_server_ip_address_idx").on( + table.serverId, + table.ipAddress, + ), index("deployments_runtime_desired_state_idx").on( table.runtimeDesiredState, ), index("deployments_traffic_state_idx").on(table.trafficState), index("deployments_observed_phase_idx").on(table.observedPhase), + foreignKey({ + name: "deployments_service_revision_service_fk", + columns: [table.serviceRevisionId, table.serviceId], + foreignColumns: [serviceRevisions.id, serviceRevisions.serviceId], + }).onDelete("no action"), ], ); @@ -627,6 +692,7 @@ export const rollouts = pgTable( serviceId: text("service_id") .notNull() .references(() => services.id, { onDelete: "cascade" }), + serviceRevisionId: text("service_revision_id").notNull(), status: text("status", { enum: ["queued", "in_progress", "completed", "failed", "rolled_back"], }) @@ -638,7 +704,15 @@ export const rollouts = pgTable( .notNull(), completedAt: timestamp("completed_at", { withTimezone: true }), }, - (table) => [index("rollouts_service_id_idx").on(table.serviceId)], + (table) => [ + index("rollouts_service_id_idx").on(table.serviceId), + index("rollouts_service_revision_id_idx").on(table.serviceRevisionId), + foreignKey({ + name: "rollouts_service_revision_service_fk", + columns: [table.serviceRevisionId, table.serviceId], + foreignColumns: [serviceRevisions.id, serviceRevisions.serviceId], + }).onDelete("no action"), + ], ); export const deploymentPorts = pgTable("deployment_ports", { @@ -646,9 +720,7 @@ export const deploymentPorts = pgTable("deployment_ports", { deploymentId: text("deployment_id") .notNull() .references(() => deployments.id, { onDelete: "cascade" }), - servicePortId: text("service_port_id") - .notNull() - .references(() => servicePorts.id, { onDelete: "cascade" }), + containerPort: integer("container_port").notNull(), hostPort: integer("host_port").notNull(), createdAt: timestamp("created_at", { withTimezone: true }) .defaultNow() diff --git a/web/db/types.ts b/web/db/types.ts index 32c4d44..973ef96 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -1,3 +1,4 @@ +import type { DeployedConfig } from "@/lib/service-config"; import type { builds, deploymentPorts, @@ -12,6 +13,7 @@ import type { servers, servicePorts, serviceReplicas, + serviceRevisions, services, serviceVolumes, user, @@ -26,6 +28,7 @@ export type Service = typeof services.$inferSelect; export type ServicePort = typeof servicePorts.$inferSelect; export type ServiceVolume = typeof serviceVolumes.$inferSelect; export type ServiceReplica = typeof serviceReplicas.$inferSelect; +export type ServiceRevision = typeof serviceRevisions.$inferSelect; export type Secret = typeof secrets.$inferSelect; export type Deployment = typeof deployments.$inferSelect; export type DeploymentPort = typeof deploymentPorts.$inferSelect; @@ -67,7 +70,13 @@ export type ServiceWithDetails = Service & { } >; volumes?: ServiceVolume[]; - secrets?: Array & { updatedAt: Date | string }>; + secrets?: Array< + Pick & { + updatedAt: Date | string; + fingerprint: string; + } + >; + activeConfig?: DeployedConfig | null; rollouts?: Rollout[]; lockedServer?: Pick | null; latestBuild?: Pick | null; diff --git a/web/lib/agent-capabilities.ts b/web/lib/agent-capabilities.ts new file mode 100644 index 0000000..c015ccc --- /dev/null +++ b/web/lib/agent-capabilities.ts @@ -0,0 +1,13 @@ +import type { AgentHealth } from "@/db/schema"; + +export const SERVICE_REVISION_CAPABILITY = "service_revision_v1"; + +export type AgentCompatibilityStatus = "compatible" | "upgrade_required"; + +export function getAgentCompatibilityStatus( + agentHealth: AgentHealth | null | undefined, +): AgentCompatibilityStatus { + return agentHealth?.capabilities?.includes(SERVICE_REVISION_CAPABILITY) + ? "compatible" + : "upgrade_required"; +} diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index dc4a362..000f5ae 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -3,11 +3,11 @@ import { db } from "@/db"; import { type AgentHealth, type ContainerHealth, - deploymentPorts, deployments, type NetworkHealth, rollouts, servers, + serviceRevisions, services, workQueue, } from "@/db/schema"; @@ -29,7 +29,7 @@ import { import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; -import { getDeployedServerlessConfig } from "@/lib/service-config"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { enqueueWork } from "@/lib/work-queue"; @@ -153,17 +153,17 @@ async function applyDeploymentErrors( rolloutId: deployments.rolloutId, observedPhase: deployments.observedPhase, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, - serverlessEnabled: services.serverlessEnabled, - serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, - serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, - stateful: services.stateful, - deployedConfig: services.deployedConfig, + revisionSpec: serviceRevisions.specification, rolloutStatus: rollouts.status, serverName: servers.name, }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) .leftJoin(rollouts, eq(deployments.rolloutId, rollouts.id)) .where( and( @@ -192,8 +192,7 @@ async function applyDeploymentErrors( .set( isServerlessWakeDeployment ? getServerlessWakeFailureUpdate({ - serverlessEnabled: - getDeployedServerlessConfig(deployment).enabled, + serverlessEnabled: deployment.revisionSpec.serverless.enabled, currentFailureCount: deployment.serverlessWakeFailureCount, failedStage: "serverless_wake", }) @@ -284,16 +283,16 @@ async function applyServerlessTransitions( trafficState: deployments.trafficState, observedPhase: deployments.observedPhase, serverlessWakeFailureCount: deployments.serverlessWakeFailureCount, - serverlessEnabled: services.serverlessEnabled, - serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, - serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, - stateful: services.stateful, - deployedConfig: services.deployedConfig, + revisionSpec: serviceRevisions.specification, serverIsProxy: servers.isProxy, serverName: servers.name, }) .from(deployments) .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) .innerJoin(servers, eq(deployments.serverId, servers.id)) .where(eq(deployments.id, transition.deploymentId)) .then((rows) => rows[0]); @@ -418,7 +417,7 @@ async function applyServerlessTransitions( .update(deployments) .set( getServerlessWakeFailureUpdate({ - serverlessEnabled: getDeployedServerlessConfig(deployment).enabled, + serverlessEnabled: deployment.revisionSpec.serverless.enabled, currentFailureCount: deployment.serverlessWakeFailureCount, failedStage: "serverless_wake", }), @@ -565,10 +564,7 @@ function getInvalidServerlessTransitionReason({ runtimeDesiredState: string; trafficState: string; observedPhase: string; - serverlessEnabled: boolean; - serverlessSleepAfterSeconds: number; - serverlessWakeTimeoutSeconds: number; - deployedConfig: string | null; + revisionSpec: ServiceRevisionSpec; serverIsProxy: boolean; } | undefined; @@ -577,7 +573,7 @@ function getInvalidServerlessTransitionReason({ if (deployment.serverId !== serverId) return "deployment belongs to another server"; if (!deployment.serverIsProxy) return "server is not a proxy"; - if (!getDeployedServerlessConfig(deployment).enabled) { + if (!deployment.revisionSpec.serverless.enabled) { return "service is not serverless"; } if (deployment.runtimeDesiredState === "removed") { @@ -764,9 +760,6 @@ export async function applyStatusReport( console.log( `[status:${serverId.slice(0, 8)}] deployment ${dep.id.slice(0, 8)} was removed and container gone, deleting`, ); - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, dep.id)); await db.delete(deployments).where(eq(deployments.id, dep.id)); } else if ( isObservedActiveContainer(dep.observedPhase) && @@ -817,13 +810,20 @@ export async function applyStatusReport( `[health:recover] found stuck deployment ${stuckDeployment.id}, attaching container ${container.containerId}`, ); - const service = await db - .select() - .from(services) - .where(eq(services.id, stuckDeployment.serviceId)) - .then((r) => r[0]); - - const hasHealthCheck = service?.healthCheckCmd != null; + const [service, revision] = await Promise.all([ + db + .select() + .from(services) + .where(eq(services.id, stuckDeployment.serviceId)) + .then((r) => r[0]), + db + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where(eq(serviceRevisions.id, stuckDeployment.serviceRevisionId)) + .then((r) => r[0]), + ]); + + const hasHealthCheck = revision?.specification.healthCheck != null; const newStatus = hasHealthCheck ? "starting" : "healthy"; await db @@ -913,17 +913,17 @@ export async function applyStatusReport( deployment.runtimeDesiredState === "running" && ["sleeping", "stopped"].includes(deployment.observedPhase) ) { - const service = await db - .select() - .from(services) - .where(eq(services.id, deployment.serviceId)) + const revision = await db + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where(eq(serviceRevisions.id, deployment.serviceRevisionId)) .then((r) => r[0]); - if (service && getDeployedServerlessConfig(service).enabled) { + if (revision?.specification.serverless.enabled) { Object.assign( updateFields, getStaleStoppedServerlessReportUpdate({ - hasHealthCheck: service.healthCheckCmd != null, + hasHealthCheck: revision.specification.healthCheck != null, healthStatus, }), ); @@ -938,13 +938,20 @@ export async function applyStatusReport( continue; } - const service = await db - .select() - .from(services) - .where(eq(services.id, deployment.serviceId)) - .then((r) => r[0]); + const [revision, service] = await Promise.all([ + db + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where(eq(serviceRevisions.id, deployment.serviceRevisionId)) + .then((r) => r[0]), + db + .select({ migrationStatus: services.migrationStatus }) + .from(services) + .where(eq(services.id, deployment.serviceId)) + .then((r) => r[0]), + ]); - const hasHealthCheck = service?.healthCheckCmd != null; + const hasHealthCheck = revision?.specification.healthCheck != null; const newStatus = hasHealthCheck ? "starting" : "healthy"; updateFields.observedPhase = newStatus; if (deployment.observedPhase === "waking") { diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index cb13d98..adc95c7 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -1,13 +1,12 @@ +import { createHash } from "node:crypto"; import { and, eq, inArray, isNull } from "drizzle-orm"; import { db } from "@/db"; import { deploymentPorts, deployments, - secrets, servers, - servicePorts, + serviceRevisions, services, - serviceVolumes, } from "@/db/schema"; import { getAllCertificatesForDomains } from "@/lib/acme-manager"; import { @@ -17,33 +16,51 @@ import { runtimeExpectedStates, } from "@/lib/deployment-status"; import { - getDeployedServerlessConfig, - getDeployedServicePorts, - isDeployedServerlessService, -} from "@/lib/service-config"; -import { slugify } from "@/lib/utils"; + SERVICE_REVISION_SCHEMA_VERSION, + type ServiceRevisionSecret, + type ServiceRevisionSpec, +} from "@/lib/service-revision-spec"; import { getWireGuardPeers } from "@/lib/wireguard"; type Server = typeof servers.$inferSelect; type Service = typeof services.$inferSelect; type Deployment = typeof deployments.$inferSelect; -type ServicePort = typeof servicePorts.$inferSelect; +type ServiceRevision = typeof serviceRevisions.$inferSelect; const SERVERLESS_GATEWAY_PORT = 18080; -type RouteServicePort = Pick< - ServicePort, - | "id" - | "serviceId" - | "port" - | "isPublic" - | "domain" - | "protocol" - | "externalPort" - | "tlsPassthrough" ->; +type RouteServicePort = { + id: string; + serviceId: string; + port: number; + isPublic: boolean; + domain: string | null; + protocol: "http" | "tcp" | "udp"; + externalPort: number | null; + tlsPassthrough: boolean; +}; + +export type RuntimeServiceRevision = { + id: string; + name: string; + revisionId: string; + specification: ServiceRevisionSpec; +}; + +export type RuntimeServiceRevisionRow = { + deploymentId: string; + serviceId: string; + serviceName: string; + serviceActiveRevisionId: string | null; + revisionId: string; + revisionServiceId: string; + revisionSchemaVersion: number; + specification: ServiceRevisionSpec; +}; export type ExpectedContainer = { deploymentId: string; + revisionId: string; + containerSpecHash: string; serviceId: string; serviceName: string; name: string; @@ -107,6 +124,7 @@ export type ServerlessRoute = { }; export type AgentExpectedState = { + schemaVersion: 1; serverName: string; containers: ExpectedContainer[]; dns: { records: Array<{ name: string; ips: string[] }> }; @@ -127,18 +145,6 @@ type DeploymentPortRow = { containerPort: number; }; -type SecretRow = { - serviceId: string; - key: string; - encryptedValue: string; -}; - -type VolumeRow = { - serviceId: string; - name: string; - containerPath: string; -}; - type RoutableDeploymentRow = { serviceId: string; ipAddress: string | null; @@ -172,18 +178,22 @@ export async function getServer(serverId: string) { export async function buildAgentExpectedState( server: Server, ): Promise { - const allServices = await getActiveServices(); - const containers = await buildExpectedContainers(server.id); - const dnsRecords = await buildDnsRecords(allServices); - const traefikConfig = await buildTraefikConfig(server, allServices); + const runtimeServices = await getRuntimeServiceRevisions(); + const [containers, dnsRecords, traefikConfig, wireguardPeers] = + await Promise.all([ + buildExpectedContainers(server.id), + buildDnsRecords(runtimeServices), + buildTraefikConfig(server, runtimeServices), + getWireGuardPeers(server.id, server.privateIp), + ]); const serverless = await buildServerlessExpectedState( server, - allServices, + runtimeServices, containers, ); - const wireguardPeers = await getWireGuardPeers(server.id, server.privateIp); return { + schemaVersion: 1, serverName: server.name, containers, dns: { records: dnsRecords }, @@ -193,8 +203,104 @@ export async function buildAgentExpectedState( }; } -async function getActiveServices() { - return db.select().from(services).where(isNull(services.deletedAt)); +async function getRuntimeServiceRevisions(): Promise { + const rows = await db + .select({ + deploymentId: deployments.id, + serviceId: services.id, + serviceName: services.name, + serviceActiveRevisionId: services.activeRevisionId, + revisionId: serviceRevisions.id, + revisionServiceId: serviceRevisions.serviceId, + revisionSchemaVersion: serviceRevisions.schemaVersion, + specification: serviceRevisions.specification, + }) + .from(deployments) + .innerJoin(services, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + isNull(services.deletedAt), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), + inArray(deployments.trafficState, activeTrafficStates), + ), + ); + + const { services: runtimeServices, errors } = + selectRuntimeServiceRevisions(rows); + for (const error of errors) { + console.error(`[expected-state] ${error}`); + } + return runtimeServices; +} + +export function selectRuntimeServiceRevisions( + rows: RuntimeServiceRevisionRow[], +): { services: RuntimeServiceRevision[]; errors: string[] } { + const rowsByService = groupBy(rows, (row) => row.serviceId); + const runtimeServices: RuntimeServiceRevision[] = []; + const errors: string[] = []; + + for (const [serviceId, serviceRows] of [...rowsByService.entries()].sort( + ([a], [b]) => a.localeCompare(b), + )) { + const invalidOwnership = serviceRows.find( + (row) => row.revisionServiceId !== serviceId, + ); + if (invalidOwnership) { + errors.push( + `service ${serviceId} omitted: deployment ${invalidOwnership.deploymentId} revision belongs to another service`, + ); + continue; + } + + const unsupported = serviceRows.find( + (row) => + row.revisionSchemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || + row.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION, + ); + if (unsupported) { + errors.push( + `service ${serviceId} omitted: deployment ${unsupported.deploymentId} uses an unsupported service revision`, + ); + continue; + } + + const rowsByRevision = new Map( + serviceRows.map((row) => [row.revisionId, row]), + ); + let selected: RuntimeServiceRevisionRow | undefined = [ + ...rowsByRevision.values(), + ][0]; + if (rowsByRevision.size > 1) { + const activeRevisionId = serviceRows[0]?.serviceActiveRevisionId; + selected = activeRevisionId + ? rowsByRevision.get(activeRevisionId) + : undefined; + if (!selected) { + errors.push( + `service ${serviceId} omitted: multiple active revisions have no authoritative active revision`, + ); + continue; + } + errors.push( + `service ${serviceId} has multiple active revisions; using authoritative revision ${selected.revisionId}`, + ); + } + + if (!selected) continue; + runtimeServices.push({ + id: serviceId, + name: selected.serviceName, + revisionId: selected.revisionId, + specification: selected.specification, + }); + } + + return { services: runtimeServices, errors }; } async function buildExpectedContainers( @@ -211,42 +317,28 @@ async function buildExpectedContainers( ); const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); + const revisionIds = unique( + serverDeployments.map((dep) => dep.serviceRevisionId), + ); if (serviceIds.length === 0) return []; - const [activeServices, depPorts, serviceSecrets, volumes] = await Promise.all( - [ - db - .select() - .from(services) - .where( - and(inArray(services.id, serviceIds), isNull(services.deletedAt)), - ), - fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), - db - .select({ - serviceId: secrets.serviceId, - key: secrets.key, - encryptedValue: secrets.encryptedValue, - }) - .from(secrets) - .where(inArray(secrets.serviceId, serviceIds)), - db - .select({ - serviceId: serviceVolumes.serviceId, - name: serviceVolumes.name, - containerPath: serviceVolumes.containerPath, - }) - .from(serviceVolumes) - .where(inArray(serviceVolumes.serviceId, serviceIds)), - ], - ); + const [activeServices, revisions, depPorts] = await Promise.all([ + db + .select() + .from(services) + .where(and(inArray(services.id, serviceIds), isNull(services.deletedAt))), + db + .select() + .from(serviceRevisions) + .where(inArray(serviceRevisions.id, revisionIds)), + fetchDeploymentPorts(serverDeployments.map((dep) => dep.id)), + ]); return buildExpectedContainersFromRows({ deployments: serverDeployments, services: activeServices, + revisions, deploymentPorts: depPorts, - secrets: serviceSecrets, - volumes, }); } @@ -257,25 +349,27 @@ async function fetchDeploymentPorts(deploymentIds: string[]) { .select({ deploymentId: deploymentPorts.deploymentId, hostPort: deploymentPorts.hostPort, - containerPort: servicePorts.port, + containerPort: deploymentPorts.containerPort, }) .from(deploymentPorts) - .innerJoin(servicePorts, eq(deploymentPorts.servicePortId, servicePorts.id)) - .where(inArray(deploymentPorts.deploymentId, deploymentIds)); + .where(inArray(deploymentPorts.deploymentId, deploymentIds)) + .orderBy( + deploymentPorts.deploymentId, + deploymentPorts.containerPort, + deploymentPorts.hostPort, + ); } export function buildExpectedContainersFromRows({ deployments: deploymentRows, services: serviceRows, + revisions: revisionRows, deploymentPorts: deploymentPortRows, - secrets: secretRows, - volumes: volumeRows, }: { deployments: Deployment[]; services: Service[]; + revisions: ServiceRevision[]; deploymentPorts: DeploymentPortRow[]; - secrets: SecretRow[]; - volumes: VolumeRow[]; }): ExpectedContainer[] { const servicesById = new Map( serviceRows.map((service) => [service.id, service]), @@ -284,46 +378,100 @@ export function buildExpectedContainersFromRows({ deploymentPortRows, (port) => port.deploymentId, ); - const secretsByServiceId = groupBy(secretRows, (secret) => secret.serviceId); - const volumesByServiceId = groupBy(volumeRows, (volume) => volume.serviceId); + const revisionsById = new Map( + revisionRows.map((revision) => [revision.id, revision]), + ); return deploymentRows .slice() .sort((a, b) => a.id.localeCompare(b.id)) .flatMap((dep) => { const service = servicesById.get(dep.serviceId); - if (!service) return []; + const revision = revisionsById.get(dep.serviceRevisionId); + if (!service) { + console.error( + `[expected-state] deployment ${dep.id} omitted because its service is deleted`, + ); + return []; + } + if (!revision) { + throw new Error(`Deployment ${dep.id} has no service revision`); + } + if (revision.serviceId !== dep.serviceId) { + throw new Error( + `Deployment ${dep.id} revision belongs to another service`, + ); + } + if ( + revision.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || + revision.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION + ) { + throw new Error( + `Deployment ${dep.id} uses an unsupported service revision`, + ); + } + const specification = revision.specification; + const ports = (portsByDeploymentId.get(dep.id) ?? []) + .slice() + .sort( + (a, b) => + a.containerPort - b.containerPort || a.hostPort - b.hostPort, + ) + .map((port) => ({ + containerPort: port.containerPort, + hostPort: port.hostPort, + })); + const expectedContainerPorts = specification.ports + .map((port) => port.containerPort) + .sort((a, b) => a - b); + const allocatedContainerPorts = ports.map((port) => port.containerPort); + if ( + JSON.stringify(expectedContainerPorts) !== + JSON.stringify(allocatedContainerPorts) + ) { + throw new Error(`Deployment ${dep.id} has incomplete port allocation`); + } + const env = buildEnv(specification.secrets); + const volumes = specification.volumes + .slice() + .sort((a, b) => a.containerPath.localeCompare(b.containerPath)) + .map((volume) => ({ + name: volume.name, + containerPath: volume.containerPath, + })); + const creationSpec = { + image: normalizeImage(specification.image), + ipAddress: dep.ipAddress, + ports, + publishLocalPorts: specification.serverless.enabled, + env, + startCommand: specification.startCommand, + healthCheck: specification.healthCheck, + volumes, + resourceCpuLimit: specification.resourceLimits.cpuCores, + resourceMemoryLimitMb: specification.resourceLimits.memoryMb, + }; return [ { deploymentId: dep.id, + revisionId: revision.id, + containerSpecHash: hashContainerCreationSpec(creationSpec), serviceId: dep.serviceId, serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, desiredState: dep.runtimeDesiredState === "stopped" ? "stopped" : "running", - image: normalizeImage(service.image), + image: creationSpec.image, ipAddress: dep.ipAddress, - ports: (portsByDeploymentId.get(dep.id) ?? []) - .slice() - .sort((a, b) => a.containerPort - b.containerPort) - .map((p) => ({ - containerPort: p.containerPort, - hostPort: p.hostPort, - })), - publishLocalPorts: isDeployedServerlessService(service), - env: buildEnv(secretsByServiceId.get(dep.serviceId) ?? []), - startCommand: service.startCommand || null, - healthCheck: buildHealthCheck(service), - volumes: (volumesByServiceId.get(dep.serviceId) ?? []) - .slice() - .sort((a, b) => a.containerPath.localeCompare(b.containerPath)) - .map((v) => ({ - name: v.name, - containerPath: v.containerPath, - })), - resourceCpuLimit: service.resourceCpuLimit, - resourceMemoryLimitMb: service.resourceMemoryLimitMb, + ports, + publishLocalPorts: creationSpec.publishLocalPorts, + env, + startCommand: creationSpec.startCommand, + healthCheck: creationSpec.healthCheck, + volumes, + resourceCpuLimit: creationSpec.resourceCpuLimit, + resourceMemoryLimitMb: creationSpec.resourceMemoryLimitMb, }, ]; }); @@ -331,42 +479,39 @@ export function buildExpectedContainersFromRows({ async function buildServerlessExpectedState( server: Server, - allServices: Service[], + allServices: RuntimeServiceRevision[], containers: ExpectedContainer[], ): Promise<{ routes: ServerlessRoute[] }> { if (!server.isProxy) { return { routes: [] }; } - const serverlessServices = allServices.filter(isDeployedServerlessService); + const serverlessServices = allServices.filter( + (service) => service.specification.serverless.enabled, + ); if (serverlessServices.length === 0) return { routes: [] }; const serviceIds = serverlessServices.map((service) => service.id); - const [ports, deploymentRows] = await Promise.all([ - db - .select() - .from(servicePorts) - .where(inArray(servicePorts.serviceId, serviceIds)), - db - .select({ - id: deployments.id, - serviceId: deployments.serviceId, - serverId: deployments.serverId, - ipAddress: deployments.ipAddress, - runtimeDesiredState: deployments.runtimeDesiredState, - trafficState: deployments.trafficState, - observedPhase: deployments.observedPhase, - serverIsProxy: servers.isProxy, - }) - .from(deployments) - .innerJoin(servers, eq(deployments.serverId, servers.id)) - .where( - and( - inArray(deployments.serviceId, serviceIds), - inArray(deployments.runtimeDesiredState, runtimeExpectedStates), - ), + const deploymentRows = await db + .select({ + id: deployments.id, + serviceId: deployments.serviceId, + serverId: deployments.serverId, + ipAddress: deployments.ipAddress, + runtimeDesiredState: deployments.runtimeDesiredState, + trafficState: deployments.trafficState, + observedPhase: deployments.observedPhase, + serverIsProxy: servers.isProxy, + }) + .from(deployments) + .innerJoin(servers, eq(deployments.serverId, servers.id)) + .where( + and( + inArray(deployments.serviceId, serviceIds), + inArray(deployments.runtimeDesiredState, runtimeExpectedStates), ), - ]); + ); + const ports = buildRuntimeRoutePorts(serverlessServices); return { routes: buildServerlessRoutesFromRows({ @@ -387,7 +532,7 @@ export function buildServerlessRoutesFromRows({ containers, }: { serverId: string; - services: Service[]; + services: RuntimeServiceRevision[]; ports: RouteServicePort[]; deployments: ServerlessDeploymentRow[]; containers: ExpectedContainer[]; @@ -403,10 +548,10 @@ export function buildServerlessRoutesFromRows({ return serviceRows .flatMap((service) => - getRuntimeServicePortsForRoutes( + (portsByServiceId.get(service.id) ?? []).map((port) => ({ service, - portsByServiceId.get(service.id) ?? [], - ).map((port) => ({ service, port })), + port, + })), ) .sort((a, b) => compareServicePorts(a.port, b.port)) .flatMap(({ service, port }) => { @@ -451,10 +596,9 @@ export function buildServerlessRoutesFromRows({ serviceId: service.id, domain: port.domain, port: port.port, - sleepAfterSeconds: - getDeployedServerlessConfig(service).sleepAfterSeconds, + sleepAfterSeconds: service.specification.serverless.sleepAfterSeconds, wakeTimeoutSeconds: - getDeployedServerlessConfig(service).wakeTimeoutSeconds, + service.specification.serverless.wakeTimeoutSeconds, localDeploymentIds, upstreams, }, @@ -462,7 +606,7 @@ export function buildServerlessRoutesFromRows({ }); } -async function buildDnsRecords(allServices: Service[]) { +async function buildDnsRecords(allServices: RuntimeServiceRevision[]) { const serviceIds = allServices.map((service) => service.id); if (serviceIds.length === 0) return []; @@ -495,25 +639,21 @@ async function buildDnsRecords(allServices: Service[]) { if (ips.length === 0) return []; - const hostname = service.hostname || slugify(service.name); - return [{ name: `${hostname}.internal`, ips }]; + return [{ name: `${service.specification.hostname}.internal`, ips }]; }) .sort((a, b) => a.name.localeCompare(b.name)); } -async function buildTraefikConfig(server: Server, allServices: Service[]) { +async function buildTraefikConfig( + server: Server, + allServices: RuntimeServiceRevision[], +) { const emptyConfig = { httpRoutes: [], tcpRoutes: [], udpRoutes: [] }; if (!server.isProxy) return emptyConfig; const serviceIds = allServices.map((service) => service.id); - const [ports, routableDeployments, proxyHostedServerlessDeployments] = + const [routableDeployments, proxyHostedServerlessDeployments] = await Promise.all([ - serviceIds.length > 0 - ? db - .select() - .from(servicePorts) - .where(inArray(servicePorts.serviceId, serviceIds)) - : Promise.resolve([]), serviceIds.length > 0 ? db .select({ @@ -556,7 +696,7 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { proxyHostedServerlessDeployments, }); - const routePorts = buildRuntimeRoutePorts(allServices, ports); + const routePorts = buildRuntimeRoutePorts(allServices); const routes = buildTraefikRoutes({ serverId: server.id, ports: routePorts, @@ -687,7 +827,7 @@ export function buildServerlessTraefikRouteSets({ proxyHostedServerlessDeployments, }: { serverId: string; - services: Service[]; + services: RuntimeServiceRevision[]; proxyHostedServerlessDeployments: ProxyHostedServerlessDeploymentRow[]; }) { const proxyHostedServerlessServiceIds = new Set( @@ -702,7 +842,7 @@ export function buildServerlessTraefikRouteSets({ serviceRows .filter( (service) => - isDeployedServerlessService(service) && + service.specification.serverless.enabled && proxyHostedServerlessServiceIds.has(service.id), ) .map((service) => service.id), @@ -732,19 +872,7 @@ export function buildTraefikCertificateDomains(ports: RouteServicePort[]) { ).sort(); } -function buildHealthCheck(service: Service): ExpectedContainer["healthCheck"] { - if (!service.healthCheckCmd) return null; - - return { - cmd: service.healthCheckCmd, - interval: service.healthCheckInterval ?? 10, - timeout: service.healthCheckTimeout ?? 5, - retries: service.healthCheckRetries ?? 3, - startPeriod: service.healthCheckStartPeriod ?? 30, - }; -} - -function buildEnv(secretRows: SecretRow[]) { +function buildEnv(secretRows: ServiceRevisionSecret[]) { const env: Record = {}; for (const secret of secretRows .slice() @@ -754,6 +882,23 @@ function buildEnv(secretRows: SecretRow[]) { return env; } +function hashContainerCreationSpec(specification: { + image: string; + ipAddress: string | null; + ports: ExpectedContainer["ports"]; + publishLocalPorts: boolean; + env: Record; + startCommand: string | null; + healthCheck: ServiceRevisionSpec["healthCheck"]; + volumes: ExpectedContainer["volumes"]; + resourceCpuLimit: number | null; + resourceMemoryLimitMb: number | null; +}) { + return createHash("sha256") + .update(JSON.stringify(specification)) + .digest("hex"); +} + function upstreamUrls(deployments: RoutableDeploymentRow[], port: number) { return deployments .map((d) => d.ipAddress) @@ -796,44 +941,20 @@ function groupBy(items: T[], keyFn: (item: T) => K) { } export function buildRuntimeRoutePorts( - serviceRows: Service[], - portRows: ServicePort[], + serviceRows: RuntimeServiceRevision[], ): RouteServicePort[] { - const portsByServiceId = groupBy(portRows, (port) => port.serviceId); return serviceRows.flatMap((service) => - getRuntimeServicePortsForRoutes( - service, - portsByServiceId.get(service.id) ?? [], - ), - ); -} - -function getRuntimeServicePortsForRoutes( - service: Service, - livePorts: RouteServicePort[], -): RouteServicePort[] { - const ports = isDeployedServerlessService(service) - ? getDeployedServicePorts(service, livePorts) - : livePorts; - return ports.map((port, index) => { - const routePort = port as Partial; - return { - id: - typeof routePort.id === "string" - ? routePort.id - : `${service.id}:deployed:${index}`, + service.specification.ports.map((port, index) => ({ + id: `${service.revisionId}:${index}`, serviceId: service.id, - port: port.port, + port: port.containerPort, isPublic: port.isPublic, domain: port.domain, - protocol: port.protocol ?? "http", - externalPort: - typeof routePort.externalPort === "number" - ? routePort.externalPort - : null, - tlsPassthrough: routePort.tlsPassthrough ?? false, - }; - }); + protocol: port.protocol, + externalPort: port.externalPort, + tlsPassthrough: port.tlsPassthrough, + })), + ); } function compareServicePorts(a: RouteServicePort, b: RouteServicePort) { diff --git a/web/lib/cli-service.ts b/web/lib/cli-service.ts index a3191c1..9929ad7 100644 --- a/web/lib/cli-service.ts +++ b/web/lib/cli-service.ts @@ -716,7 +716,7 @@ export async function deployManifest(manifest: TechulusManifest) { manifest.service.replicas.count, ); - const result = await deployServiceInternal(service.id); + const result = await deployServiceInternal(service.id, { trigger: "cli" }); return { serviceId: service.id, diff --git a/web/lib/deploy-service.ts b/web/lib/deploy-service.ts index 4f78e9d..98ebc74 100644 --- a/web/lib/deploy-service.ts +++ b/web/lib/deploy-service.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { db } from "@/db"; @@ -7,8 +6,13 @@ import { rollouts, serviceReplicas } from "@/db/schema"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { startMigrationInternal } from "@/lib/migrations"; +import type { RevisionSourceMetadata } from "@/lib/service-revisions"; +import { createRolloutWithServiceRevision } from "@/lib/service-revisions"; -export async function deployServiceInternal(serviceId: string) { +export async function deployServiceInternal( + serviceId: string, + sourceMetadata: RevisionSourceMetadata = {}, +) { const service = await getService(serviceId); if (!service) { throw new Error("Service not found"); @@ -48,14 +52,10 @@ export async function deployServiceInternal(serviceId: string) { } } - const rolloutId = randomUUID(); - - await db.insert(rollouts).values({ - id: rolloutId, + const { rolloutId } = await createRolloutWithServiceRevision( serviceId, - status: "queued", - currentStage: "queued", - }); + sourceMetadata, + ); try { await inngest.send( diff --git a/web/lib/deployment-status.ts b/web/lib/deployment-status.ts index efbbfb9..165144c 100644 --- a/web/lib/deployment-status.ts +++ b/web/lib/deployment-status.ts @@ -11,14 +11,19 @@ export type DeploymentState = { observedPhase: ObservedPhase; }; -export const runtimeExpectedStates = ["running", "stopped"] as const satisfies - readonly RuntimeDesiredState[]; +export const runtimeExpectedStates = [ + "running", + "stopped", +] as const satisfies readonly RuntimeDesiredState[]; -export const activeTrafficStates = ["active"] as const satisfies - readonly TrafficState[]; +export const activeTrafficStates = [ + "active", +] as const satisfies readonly TrafficState[]; -export const observedReadyPhases = ["healthy", "running"] as const satisfies - readonly ObservedPhase[]; +export const observedReadyPhases = [ + "healthy", + "running", +] as const satisfies readonly ObservedPhase[]; export const observedStartingPhases = [ "pending", @@ -87,3 +92,27 @@ export function markDeploymentFailedRemoved(failedStage: string) { failedStage, }; } + +export function selectNewestRevisionId( + revisions: Array<{ serviceRevisionId: string; revisionNumber: number }>, +): string | null { + const revisionNumbers = new Map(); + for (const revision of revisions) { + revisionNumbers.set( + revision.serviceRevisionId, + Math.max( + revisionNumbers.get(revision.serviceRevisionId) ?? 0, + revision.revisionNumber, + ), + ); + } + + return ( + [...revisionNumbers] + .sort( + ([firstId, firstNumber], [secondId, secondNumber]) => + secondNumber - firstNumber || secondId.localeCompare(firstId), + ) + .at(0)?.[0] ?? null + ); +} diff --git a/web/lib/inngest/functions/build-workflow.ts b/web/lib/inngest/functions/build-workflow.ts index b66ebb0..34c4dd8 100644 --- a/web/lib/inngest/functions/build-workflow.ts +++ b/web/lib/inngest/functions/build-workflow.ts @@ -78,12 +78,15 @@ export const buildWorkflow = inngest.createFunction( return { status: "failed", reason: result.data.error, buildId }; } - const manifestAlreadyCompleted = await step.run("check-existing-manifest", async () => { - return hasCompletedManifestWorkItem({ - serviceId, - buildId, - }); - }); + const manifestAlreadyCompleted = await step.run( + "check-existing-manifest", + async () => { + return hasCompletedManifestWorkItem({ + serviceId, + buildId, + }); + }, + ); if (!manifestAlreadyCompleted) { const manifestResult = await step.waitForEvent("wait-manifest", { @@ -114,7 +117,7 @@ export const buildWorkflow = inngest.createFunction( if (shouldDeploy) { await step.run("trigger-deploy", async () => { - await deployServiceInternal(serviceId); + await deployServiceInternal(serviceId, { trigger: "build", buildId }); }); } @@ -163,12 +166,15 @@ export const buildWorkflow = inngest.createFunction( return { status: "failed", reason: "build_failed", buildGroupId }; } - const manifestAlreadyCompleted = await step.run("check-existing-group-manifest", async () => { - return hasCompletedManifestWorkItem({ - serviceId, - buildGroupId, - }); - }); + const manifestAlreadyCompleted = await step.run( + "check-existing-group-manifest", + async () => { + return hasCompletedManifestWorkItem({ + serviceId, + buildGroupId, + }); + }, + ); if (!manifestAlreadyCompleted) { const manifestResult = await step.waitForEvent("wait-group-manifest", { @@ -199,7 +205,10 @@ export const buildWorkflow = inngest.createFunction( if (shouldDeploy) { await step.run("trigger-deploy-group", async () => { - await deployServiceInternal(serviceId); + await deployServiceInternal(serviceId, { + trigger: "build_group", + buildGroupId, + }); }); } diff --git a/web/lib/inngest/functions/migration-workflow.ts b/web/lib/inngest/functions/migration-workflow.ts index bdc0327..4ab5482 100644 --- a/web/lib/inngest/functions/migration-workflow.ts +++ b/web/lib/inngest/functions/migration-workflow.ts @@ -115,59 +115,61 @@ export const migrationWorkflow = inngest.createFunction( const backupResults = await Promise.all( backupIds.map((backupId) => - group.parallel(async (): Promise<{ - status: "completed" | "failed" | "pending" | "timed_out"; - error?: string; - }> => { - const readBackup = async () => - db - .select({ - status: volumeBackups.status, - errorMessage: volumeBackups.errorMessage, - }) - .from(volumeBackups) - .where(eq(volumeBackups.id, backupId)) - .then((r) => r[0]); - - const before = await step.run( - `check-backup-${backupId}-before`, - readBackup, - ); - if (before?.status === "completed") { - return { status: "completed" as const }; - } - if (before?.status === "failed") { - return { - status: "failed" as const, - error: before.errorMessage || "Backup failed", - }; - } - - const wakeup = await step.waitForEvent( - `wait-backup-status-${backupId}`, - { - event: inngestEvents.resourceStatusChanged, - timeout: "30m", - if: `async.data.type == "backup" && async.data.id == "${backupId}"`, - }, - ); - - const after = await step.run( - `check-backup-${backupId}-after`, - readBackup, - ); - if (after?.status === "completed") { - return { status: "completed" as const }; - } - if (after?.status === "failed") { - return { - status: "failed" as const, - error: after.errorMessage || "Backup failed", - }; - } - - return { status: wakeup ? "pending" : "timed_out" } as const; - }), + group.parallel( + async (): Promise<{ + status: "completed" | "failed" | "pending" | "timed_out"; + error?: string; + }> => { + const readBackup = async () => + db + .select({ + status: volumeBackups.status, + errorMessage: volumeBackups.errorMessage, + }) + .from(volumeBackups) + .where(eq(volumeBackups.id, backupId)) + .then((r) => r[0]); + + const before = await step.run( + `check-backup-${backupId}-before`, + readBackup, + ); + if (before?.status === "completed") { + return { status: "completed" as const }; + } + if (before?.status === "failed") { + return { + status: "failed" as const, + error: before.errorMessage || "Backup failed", + }; + } + + const wakeup = await step.waitForEvent( + `wait-backup-status-${backupId}`, + { + event: inngestEvents.resourceStatusChanged, + timeout: "30m", + if: `async.data.type == "backup" && async.data.id == "${backupId}"`, + }, + ); + + const after = await step.run( + `check-backup-${backupId}-after`, + readBackup, + ); + if (after?.status === "completed") { + return { status: "completed" as const }; + } + if (after?.status === "failed") { + return { + status: "failed" as const, + error: after.errorMessage || "Backup failed", + }; + } + + return { status: wakeup ? "pending" : "timed_out" } as const; + }, + ), ), ); @@ -185,7 +187,9 @@ export const migrationWorkflow = inngest.createFunction( return { status: "failed", reason: "backup_timeout" }; } - const backupStillPending = backupResults.some((r) => r.status === "pending"); + const backupStillPending = backupResults.some( + (r) => r.status === "pending", + ); if (backupStillPending) { await step.run("handle-backup-still-pending", async () => { await db @@ -287,8 +291,7 @@ export const migrationWorkflow = inngest.createFunction( .update(services) .set({ migrationStatus: "failed", - migrationError: - restoreFailure.data.error || "Restore failed", + migrationError: restoreFailure.data.error || "Restore failed", }) .where(eq(services.id, serviceId)); }); @@ -317,7 +320,10 @@ export const migrationWorkflow = inngest.createFunction( .set({ lockedServerId: targetServerId }) .where(eq(services.id, serviceId)); - await deployServiceInternal(serviceId); + await deployServiceInternal(serviceId, { + trigger: "migration", + targetServerId, + }); }); await step.run("finalize-migration", async () => { diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 20273c2..99488e7 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -1,20 +1,17 @@ import { randomUUID } from "node:crypto"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; import { deploymentPorts, deployments, - secrets, + rollouts, servers, - servicePorts, - serviceReplicas, services, - serviceVolumes, } from "@/db/schema"; import { getCertificate, issueCertificate } from "@/lib/acme-manager"; -import { buildCurrentConfig, getDeployedStateful } from "@/lib/service-config"; -import { assignContainerIp } from "@/lib/wireguard"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; +import { findAvailableContainerIp } from "@/lib/wireguard"; import { enqueueWork } from "@/lib/work-queue"; const PORT_RANGE_START = 30000; @@ -23,7 +20,8 @@ const PORT_RANGE_END = 32767; export type Placement = { serverId: string; replicas: number }; export type DeploymentContext = { - service: NonNullable>>; + revisionId: string; + specification: ServiceRevisionSpec; placements: Placement[]; serverMap: Map< string, @@ -33,17 +31,9 @@ export type DeploymentContext = { isRollingUpdate: boolean; }; -export function isActiveDeploymentForRollout( - deployment: { trafficState: string }, - service: { - deployedConfig?: string | null; - stateful?: boolean | null; - serverlessEnabled?: boolean | null; - serverlessSleepAfterSeconds?: number | null; - serverlessWakeTimeoutSeconds?: number | null; - }, -) { - void service; +export function isActiveDeploymentForRollout(deployment: { + trafficState: string; +}) { return deployment.trafficState === "active"; } @@ -57,21 +47,11 @@ export function normalizeImage(image: string): string { return image; } -async function getUsedPorts(serverId: string): Promise> { - const existingPorts = await db - .select({ hostPort: deploymentPorts.hostPort }) - .from(deploymentPorts) - .innerJoin(deployments, eq(deploymentPorts.deploymentId, deployments.id)) - .where(eq(deployments.serverId, serverId)); - - return new Set(existingPorts.map((p) => p.hostPort)); -} - -export async function allocateHostPorts( - serverId: string, +export function findAvailableHostPorts( + usedPorts: Iterable, count: number, -): Promise { - const usedPorts = await getUsedPorts(serverId); +): number[] { + const unavailablePorts = new Set(usedPorts); const allocated: number[] = []; for ( @@ -79,7 +59,7 @@ export async function allocateHostPorts( port <= PORT_RANGE_END && allocated.length < count; port++ ) { - if (!usedPorts.has(port)) { + if (!unavailablePorts.has(port)) { allocated.push(port); } } @@ -91,22 +71,16 @@ export async function allocateHostPorts( return allocated; } -export async function calculateServicePlacements( - service: NonNullable>>, -): Promise<{ +export function calculateRevisionPlacements( + specification: ServiceRevisionSpec, +): { placements: Placement[]; totalReplicas: number; - migrationNeeded?: { targetServerId: string }; -}> { - const configuredReplicas = await db - .select({ - serverId: serviceReplicas.serverId, - replicas: serviceReplicas.count, - }) - .from(serviceReplicas) - .where(eq(serviceReplicas.serviceId, service.id)); - - const placements = configuredReplicas.filter((p) => p.replicas > 0); +} { + const placements = specification.placements.map((placement) => ({ + serverId: placement.serverId, + replicas: placement.count, + })); const totalReplicas = placements.reduce((sum, p) => sum + p.replicas, 0); if (totalReplicas < 1) { @@ -116,7 +90,7 @@ export async function calculateServicePlacements( throw new Error("Maximum 10 replicas allowed"); } - if (service.stateful) { + if (specification.stateful) { if (totalReplicas !== 1) { throw new Error("Stateful services can only have exactly 1 replica"); } @@ -127,14 +101,6 @@ export async function calculateServicePlacements( "Stateful services must be deployed to exactly one server", ); } - - const targetServerId = serverIds[0]; - if (service.lockedServerId && service.lockedServerId !== targetServerId) { - if (service.migrationStatus) { - throw new Error("Migration already in progress"); - } - return { placements, totalReplicas, migrationNeeded: { targetServerId } }; - } } return { placements, totalReplicas }; @@ -192,7 +158,7 @@ export async function prepareRollingUpdate( .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter((d) => - isActiveDeploymentForRollout(d, service), + isActiveDeploymentForRollout(d), ); return { deploymentIds: runningDeployments.map((d) => d.id) }; @@ -201,40 +167,25 @@ export async function prepareRollingUpdate( export async function cleanupTerminalDeployments( serviceId: string, ): Promise { - const terminalDeployments = await db - .select() - .from(deployments) + await db + .delete(deployments) .where( and( eq(deployments.serviceId, serviceId), eq(deployments.runtimeDesiredState, "removed"), ), ); - - for (const dep of terminalDeployments) { - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, dep.id)); - await db.delete(deployments).where(eq(deployments.id, dep.id)); - } } export async function cleanupExistingDeployments( serviceId: string, ): Promise<{ deletedCount: number }> { - const existingDeployments = await db - .select() - .from(deployments) - .where(eq(deployments.serviceId, serviceId)); - - for (const dep of existingDeployments) { - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, dep.id)); - await db.delete(deployments).where(eq(deployments.id, dep.id)); - } + const deletedDeployments = await db + .delete(deployments) + .where(eq(deployments.serviceId, serviceId)) + .returning({ id: deployments.id }); - return { deletedCount: existingDeployments.length }; + return { deletedCount: deletedDeployments.length }; } export type CertificateProvisioningResult = { @@ -244,17 +195,12 @@ export type CertificateProvisioningResult = { failedDomains: string[]; }; -export async function issueCertificatesForService( - serviceId: string, +export async function issueCertificatesForRevision( + specification: ServiceRevisionSpec, ): Promise { - const servicePortsList = await db - .select() - .from(servicePorts) - .where(eq(servicePorts.serviceId, serviceId)); - const domainsNeedingCerts = Array.from( new Set( - servicePortsList + specification.ports .filter((p) => p.isPublic && p.domain) .map((p) => (p.domain as string).trim()) .filter(Boolean), @@ -304,40 +250,9 @@ export async function createDeploymentRecords( serviceId: string, context: DeploymentContext, ): Promise<{ deploymentIds: string[] }> { - const { service, placements, serverMap, totalReplicas } = context; - - const servicePortsList = await db - .select() - .from(servicePorts) - .where(eq(servicePorts.serviceId, serviceId)); - - const serviceSecrets = await db - .select() - .from(secrets) - .where(eq(secrets.serviceId, serviceId)); - - const volumes = await db - .select() - .from(serviceVolumes) - .where(eq(serviceVolumes.serviceId, serviceId)); - - const env: Record = {}; - for (const secret of serviceSecrets) { - env[secret.key] = secret.encryptedValue; - } - - const updateData: { replicas: number; lockedServerId?: string } = { - replicas: totalReplicas, - }; - - if (service.stateful && !service.lockedServerId) { - updateData.lockedServerId = placements.map((p) => p.serverId)[0]; - } - - await db.update(services).set(updateData).where(eq(services.id, serviceId)); + const { revisionId, specification, placements, serverMap } = context; const deploymentIds: string[] = []; - let replicaIndex = 0; for (const placement of placements) { if (placement.replicas <= 0) continue; @@ -348,70 +263,109 @@ export async function createDeploymentRecords( } for (let i = 0; i < placement.replicas; i++) { - replicaIndex++; - const hostPorts = await allocateHostPorts( - server.id, - servicePortsList.length, - ); - const ipAddress = await assignContainerIp(server.id); - const deploymentId = randomUUID(); - deploymentIds.push(deploymentId); - await db.insert(deployments).values({ - id: deploymentId, - serviceId, - serverId: server.id, - ipAddress, - runtimeDesiredState: "running", - trafficState: "candidate", - observedPhase: "pending", - rolloutId, - }); - - const portMappings: { containerPort: number; hostPort: number }[] = []; - for (let j = 0; j < servicePortsList.length; j++) { - const sp = servicePortsList[j]; - const hostPort = hostPorts[j]; - - await db.insert(deploymentPorts).values({ - id: randomUUID(), - deploymentId, - servicePortId: sp.id, - hostPort, + await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${`deployment-allocation:${server.id}`}))`, + ); + + if (specification.stateful) { + const lockedService = await tx + .select({ lockedServerId: services.lockedServerId }) + .from(services) + .where(eq(services.id, serviceId)) + .for("update") + .then((rows) => rows[0]); + if (!lockedService) { + throw new Error("Service not found"); + } + if ( + lockedService.lockedServerId && + lockedService.lockedServerId !== server.id + ) { + throw new Error( + `Stateful service is locked to server ${lockedService.lockedServerId}`, + ); + } + if (!lockedService.lockedServerId) { + await tx + .update(services) + .set({ lockedServerId: server.id }) + .where(eq(services.id, serviceId)); + } + } + + const [usedPortRows, subnet, usedIpRows] = await Promise.all([ + tx + .select({ hostPort: deploymentPorts.hostPort }) + .from(deploymentPorts) + .innerJoin( + deployments, + eq(deploymentPorts.deploymentId, deployments.id), + ) + .where(eq(deployments.serverId, server.id)), + tx + .select({ subnetId: servers.subnetId }) + .from(servers) + .where(eq(servers.id, server.id)) + .then((rows) => rows[0]), + tx + .select({ ipAddress: deployments.ipAddress }) + .from(deployments) + .where( + and( + eq(deployments.serverId, server.id), + isNotNull(deployments.ipAddress), + ), + ), + ]); + + if (!subnet?.subnetId) { + throw new Error(`Server ${server.name} has no subnet assigned`); + } + + const hostPorts = findAvailableHostPorts( + usedPortRows.map((port) => port.hostPort), + specification.ports.length, + ); + const ipAddress = findAvailableContainerIp( + subnet.subnetId, + usedIpRows.flatMap((deployment) => + deployment.ipAddress ? [deployment.ipAddress] : [], + ), + ); + + await tx.insert(deployments).values({ + id: deploymentId, + serviceId, + serviceRevisionId: revisionId, + serverId: server.id, + ipAddress, + runtimeDesiredState: "running", + trafficState: "candidate", + observedPhase: "pending", + rolloutId, }); - portMappings.push({ containerPort: sp.port, hostPort }); - } - - const healthCheck = service.healthCheckCmd - ? { - cmd: service.healthCheckCmd, - interval: service.healthCheckInterval ?? 10, - timeout: service.healthCheckTimeout ?? 5, - retries: service.healthCheckRetries ?? 3, - startPeriod: service.healthCheckStartPeriod ?? 30, - } - : null; + if (specification.ports.length > 0) { + await tx.insert(deploymentPorts).values( + specification.ports.map((port, index) => ({ + id: randomUUID(), + deploymentId, + containerPort: port.containerPort, + hostPort: hostPorts[index], + })), + ); + } + }); - const volumeMounts = volumes.map((v) => ({ - name: v.name, - containerPath: v.containerPath, - })); + deploymentIds.push(deploymentId); await enqueueWork(server.id, "reconcile", { reason: "rollout_deployment_created", deploymentId, - serviceId, - serviceName: service.name, - image: normalizeImage(service.image), - portMappings, - wireguardIp: server.wireguardIp, - ipAddress, - name: `${serviceId}-${replicaIndex}`, - healthCheck, - env, - volumeMounts, + revisionId, }); } } @@ -419,65 +373,75 @@ export async function createDeploymentRecords( return { deploymentIds }; } -export async function saveDeployedConfig( +export async function completeRolloutWithRevision( + rolloutId: string, serviceId: string, - context: DeploymentContext, -): Promise { - const { placements, serverMap } = context; - - const servicePortsList = await db - .select() - .from(servicePorts) - .where(eq(servicePorts.serviceId, serviceId)); - - const serviceSecrets = await db - .select() - .from(secrets) - .where(eq(secrets.serviceId, serviceId)); - - const volumes = await db - .select() - .from(serviceVolumes) - .where(eq(serviceVolumes.serviceId, serviceId)); - - const replicaConfigs = placements - .filter((p) => p.replicas > 0) - .map((p) => ({ - serverId: p.serverId, - serverName: serverMap.get(p.serverId)?.name ?? "Unknown", - count: p.replicas, - })); - - const portConfigs = servicePortsList.map((p) => ({ - port: p.port, - isPublic: p.isPublic, - domain: p.domain, - protocol: p.protocol, - tlsPassthrough: p.tlsPassthrough, - })); - - const updatedService = await getService(serviceId); - if (updatedService) { - const deployedConfig = buildCurrentConfig( - updatedService, - replicaConfigs, - portConfigs, - serviceSecrets, - volumes, - ); + context: Omit, +): Promise<{ completed: boolean; stoppedCount: number }> { + const { + placements, + revisionId, + specification, + totalReplicas, + isRollingUpdate, + } = context; + const lockedServerId = specification.stateful + ? placements[0]?.serverId + : undefined; + + return db.transaction(async (tx) => { + const rollout = await tx + .select({ status: rollouts.status }) + .from(rollouts) + .where(eq(rollouts.id, rolloutId)) + .for("update") + .then((rows) => rows[0]); + if (rollout?.status !== "in_progress") { + return { completed: false, stoppedCount: 0 }; + } - await db + const stoppedDeployments = isRollingUpdate + ? await tx + .update(deployments) + .set({ + runtimeDesiredState: "removed", + trafficState: "inactive", + }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + ), + ) + .returning({ id: deployments.id }) + : []; + + await tx .update(services) - .set({ deployedConfig: JSON.stringify(deployedConfig) }) + .set({ + activeRevisionId: revisionId, + replicas: totalReplicas, + ...(lockedServerId ? { lockedServerId } : {}), + }) .where(eq(services.id, serviceId)); - } + + await tx + .update(rollouts) + .set({ + status: "completed", + currentStage: "completed", + completedAt: new Date(), + }) + .where(eq(rollouts.id, rolloutId)); + return { completed: true, stoppedCount: stoppedDeployments.length }; + }); } export async function checkForRollingUpdate( serviceId: string, + specification: ServiceRevisionSpec, ): Promise { - const service = await getService(serviceId); - if (!service || getDeployedStateful(service)) { + if (specification.stateful) { return false; } @@ -487,7 +451,7 @@ export async function checkForRollingUpdate( .where(eq(deployments.serviceId, serviceId)); const runningDeployments = existingDeployments.filter((d) => - isActiveDeploymentForRollout(d, service), + isActiveDeploymentForRollout(d), ); return runningDeployments.length > 0; diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 30f4432..55abead 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -1,42 +1,113 @@ import { and, eq, ne } from "drizzle-orm"; import { db } from "@/db"; -import { deployments, rollouts } from "@/db/schema"; -import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; +import { deployments, rollouts, serviceRevisions, services } from "@/db/schema"; +import { + markDeploymentFailedRemoved, + selectNewestRevisionId, +} from "@/lib/deployment-status"; import { sendDeploymentFailureAlert } from "@/lib/email"; -export async function restoreDrainingDeploymentsForRollback(serviceId: string) { - await db - .update(deployments) - .set({ trafficState: "active" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "draining"), - ), - ); -} - export async function handleRolloutFailure( rolloutId: string, serviceId: string, reason: string, isRollingUpdate: boolean, ): Promise { - const rolloutDeployments = await db - .select() - .from(deployments) - .where(eq(deployments.rolloutId, rolloutId)); + let serverId: string | null = null; + let hasRolloutDeployments = false; + let handled = false; + + await db.transaction(async (tx) => { + const rollout = await tx + .select({ status: rollouts.status }) + .from(rollouts) + .where(eq(rollouts.id, rolloutId)) + .for("update") + .then((rows) => rows[0]); + + if ( + !rollout || + (rollout.status !== "queued" && rollout.status !== "in_progress") + ) { + return; + } + handled = true; + + const rolloutDeployments = await tx + .select() + .from(deployments) + .where(eq(deployments.rolloutId, rolloutId)); + + hasRolloutDeployments = rolloutDeployments.length > 0; + serverId = rolloutDeployments[0]?.serverId ?? null; + + await tx + .update(rollouts) + .set({ + status: hasRolloutDeployments ? "rolled_back" : "failed", + currentStage: reason, + completedAt: new Date(), + }) + .where(eq(rollouts.id, rolloutId)); + + if (!hasRolloutDeployments) return; + + if (isRollingUpdate) { + await tx + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + ), + ); + } + + await tx + .update(deployments) + .set(markDeploymentFailedRemoved(reason)) + .where( + and( + eq(deployments.rolloutId, rolloutId), + ne(deployments.runtimeDesiredState, "removed"), + ), + ); + + const activeRevisionRows = await tx + .select({ + serviceRevisionId: deployments.serviceRevisionId, + revisionNumber: serviceRevisions.revisionNumber, + }) + .from(deployments) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "active"), + ne(deployments.runtimeDesiredState, "removed"), + ), + ); + const activeRevisionCount = new Set( + activeRevisionRows.map((row) => row.serviceRevisionId), + ).size; + if (activeRevisionCount > 1) { + console.error( + `[rollout:${rolloutId}] rollback found ${activeRevisionCount} active revisions for ${serviceId}; selecting the newest`, + ); + } + await tx + .update(services) + .set({ activeRevisionId: selectNewestRevisionId(activeRevisionRows) }) + .where(eq(services.id, serviceId)); + }); - await db - .update(rollouts) - .set({ - status: rolloutDeployments.length === 0 ? "failed" : "rolled_back", - currentStage: reason, - completedAt: new Date(), - }) - .where(eq(rollouts.id, rolloutId)); + if (!handled) return; - if (rolloutDeployments.length === 0) { + if (!hasRolloutDeployments) { sendDeploymentFailureAlert({ serviceId, serverId: null, @@ -50,22 +121,6 @@ export async function handleRolloutFailure( return; } - const serverId = rolloutDeployments[0].serverId; - - if (isRollingUpdate) { - await restoreDrainingDeploymentsForRollback(serviceId); - } - - await db - .update(deployments) - .set(markDeploymentFailedRemoved(reason)) - .where( - and( - eq(deployments.rolloutId, rolloutId), - ne(deployments.runtimeDesiredState, "removed"), - ), - ); - sendDeploymentFailureAlert({ serviceId, serverId, diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index d2f6d5e..b056a54 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -1,19 +1,21 @@ import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; -import { deployments, rollouts, servers } from "@/db/schema"; +import { deployments, rollouts, servers, services } from "@/db/schema"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; +import { getRolloutServiceRevision } from "@/lib/service-revisions"; import { ingestRolloutLog } from "@/lib/victoria-logs"; import { inngest } from "../client"; import { inngestEvents } from "../events"; import { - calculateServicePlacements, + calculateRevisionPlacements, checkForRollingUpdate, cleanupExistingDeployments, cleanupTerminalDeployments, + completeRolloutWithRevision, createDeploymentRecords, - issueCertificatesForService, + issueCertificatesForRevision, prepareRollingUpdate, - saveDeployedConfig, validateServers, } from "./rollout-helpers"; import { handleRolloutFailure } from "./rollout-utils"; @@ -118,25 +120,28 @@ export const rolloutWorkflow = inngest.createFunction( { event: inngestEvents.rolloutCancelled, match: "data.rolloutId" }, ], onFailure: async ({ event }) => { - const { rolloutId } = event.data.event.data as { + const { rolloutId, serviceId } = event.data.event.data as { rolloutId?: string; + serviceId?: string; }; if (!rolloutId) return; - await db - .update(rollouts) - .set({ - status: "failed", - currentStage: "workflow_failed", - completedAt: new Date(), - }) - .where( - and( - eq(rollouts.id, rolloutId), - inArray(rollouts.status, ["queued", "in_progress"]), - ), - ); + const targetServiceId = + serviceId ?? + (await db + .select({ serviceId: rollouts.serviceId }) + .from(rollouts) + .where(eq(rollouts.id, rolloutId)) + .then((rows) => rows[0]?.serviceId)); + if (!targetServiceId) return; + + await handleRolloutFailure( + rolloutId, + targetServiceId, + "workflow_failed", + true, + ); }, }, async ({ event, step }) => { @@ -193,6 +198,23 @@ export const rolloutWorkflow = inngest.createFunction( return { status: "failed", rolloutId, reason: "queue_timeout" }; } + const revision = await step.run("load-service-revision", async () => { + const rolloutRevision = await getRolloutServiceRevision(rolloutId); + if (rolloutRevision.serviceId !== serviceId) { + throw new Error("Rollout revision does not belong to service"); + } + const executionSpecification: ServiceRevisionSpec = { + ...rolloutRevision.specification, + secrets: [], + }; + return { + id: rolloutRevision.id, + serviceId: rolloutRevision.serviceId, + specification: executionSpecification, + }; + }); + const specification = revision.specification; + await step.run("log-rollout-started", async () => { await ingestRolloutLog( rolloutId, @@ -203,12 +225,8 @@ export const rolloutWorkflow = inngest.createFunction( }); const placementResult = await step.run("load-placements", async () => { - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } try { - const result = await calculateServicePlacements(service); + const result = calculateRevisionPlacements(specification); await ingestRolloutLog( rolloutId, serviceId, @@ -286,7 +304,7 @@ export const rolloutWorkflow = inngest.createFunction( }); const isRollingUpdate = await step.run("check-rolling-update", async () => { - return checkForRollingUpdate(serviceId); + return checkForRollingUpdate(serviceId, specification); }); if (isRollingUpdate) { @@ -319,7 +337,7 @@ export const rolloutWorkflow = inngest.createFunction( .set({ currentStage: "certificates" }) .where(eq(rollouts.id, rolloutId)); try { - const result = await issueCertificatesForService(serviceId); + const result = await issueCertificatesForRevision(specification); if (result.issuedDomains.length > 0) { await ingestRolloutLog( rolloutId, @@ -355,20 +373,25 @@ export const rolloutWorkflow = inngest.createFunction( } const { deploymentIds } = await step.run("create-deployments", async () => { + await db + .delete(deployments) + .where( + and( + eq(deployments.rolloutId, rolloutId), + eq(deployments.trafficState, "candidate"), + ), + ); + await db .update(rollouts) .set({ currentStage: "deploying" }) .where(eq(rollouts.id, rolloutId)); - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - const serverMap = await validateServers(placements); const result = await createDeploymentRecords(rolloutId, serviceId, { - service, + revisionId: revision.id, + specification, placements, serverMap, totalReplicas, @@ -386,8 +409,7 @@ export const rolloutWorkflow = inngest.createFunction( }); await step.run("start-health-check", async () => { - const service = await getService(serviceId); - const hasHealthCheck = service?.healthCheckCmd != null; + const hasHealthCheck = specification.healthCheck != null; await db .update(rollouts) @@ -488,8 +510,19 @@ export const rolloutWorkflow = inngest.createFunction( }; } - await step.run("start-dns-sync", async () => { - await db.transaction(async (tx) => { + const trafficPromoted = await step.run("start-dns-sync", async () => { + const promoted = await db.transaction(async (tx) => { + const activeRollout = await tx + .select({ status: rollouts.status }) + .from(rollouts) + .where(eq(rollouts.id, rolloutId)) + .for("update") + .then((rows) => rows[0]); + + if (activeRollout?.status !== "in_progress") { + return false; + } + await tx .update(rollouts) .set({ currentStage: "dns_sync" }) @@ -519,7 +552,15 @@ export const rolloutWorkflow = inngest.createFunction( ), ), ); + + await tx + .update(services) + .set({ activeRevisionId: revision.id }) + .where(eq(services.id, serviceId)); + + return true; }); + if (!promoted) return false; await ingestRolloutLog( rolloutId, @@ -527,8 +568,13 @@ export const rolloutWorkflow = inngest.createFunction( "dns_sync", "Routing traffic to new deployments", ); + return true; }); + if (!trafficPromoted) { + return { status: "cancelled", rolloutId }; + } + const dnsResults = await Promise.all( serverIds.map((serverId) => step.waitForEvent(`wait-dns-${serverId}`, { @@ -583,84 +629,34 @@ export const rolloutWorkflow = inngest.createFunction( return { status: "rolled_back", rolloutId, reason: "dns_sync_timeout" }; } - if (isRollingUpdate) { - await step.run("stop-old-deployments", async () => { - const stoppedDeploymentsWithoutContainers = await db - .update(deployments) - .set({ - runtimeDesiredState: "removed", - trafficState: "inactive", - }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "draining"), - isNull(deployments.containerId), - ), - ) - .returning({ id: deployments.id }); - - const stoppingDeployments = await db - .update(deployments) - .set({ - runtimeDesiredState: "removed", - trafficState: "inactive", - }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "draining"), - ), - ) - .returning({ id: deployments.id }); - - const stoppedCount = - stoppedDeploymentsWithoutContainers.length + - stoppingDeployments.length; - if (stoppedCount > 0) { - await ingestRolloutLog( - rolloutId, - serviceId, - "dns_sync", - `Stopping ${stoppedCount} old deployment(s) after DNS sync`, - ); - } - }); - } - - await step.run("save-deployed-config", async () => { - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - - const serverMap = await validateServers(placements); - - await saveDeployedConfig(serviceId, { - service, + const rolloutCompleted = await step.run("complete-rollout", async () => { + const result = await completeRolloutWithRevision(rolloutId, serviceId, { + revisionId: revision.id, + specification, placements, - serverMap, totalReplicas, isRollingUpdate, }); - }); - - await step.run("complete-rollout", async () => { - await db - .update(rollouts) - .set({ - status: "completed", - currentStage: "completed", - completedAt: new Date(), - }) - .where(eq(rollouts.id, rolloutId)); + if (!result.completed) return false; + if (result.stoppedCount > 0) { + await ingestRolloutLog( + rolloutId, + serviceId, + "dns_sync", + `Stopping ${result.stoppedCount} old deployment(s) after DNS sync`, + ); + } await ingestRolloutLog( rolloutId, serviceId, "completed", "Rollout completed successfully", ); + return true; }); + if (!rolloutCompleted) { + return { status: "cancelled", rolloutId }; + } return { status: "completed", rolloutId }; }, diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index 640e533..dd39d58 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -5,8 +5,8 @@ import { deleteBackup } from "@/actions/backups"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { - deploymentPorts, deployments, + rollouts, secrets, services, serviceVolumes, @@ -14,7 +14,10 @@ import { } from "@/db/schema"; import { addUtcDays, toDate } from "@/lib/date"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentRemoved } from "@/lib/deployment-status"; +import { + markDeploymentFailedRemoved, + markDeploymentRemoved, +} from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -241,10 +244,6 @@ export const serviceDeletionWorkflow = inngest.createFunction( containerId: deployment.containerId, }); } - - await db - .delete(deploymentPorts) - .where(eq(deploymentPorts.deploymentId, deployment.id)); } await db @@ -421,7 +420,9 @@ export const serviceRestoreWorkflow = inngest.createFunction( .where(eq(services.id, serviceId)); try { - const result = await deployServiceInternal(serviceId); + const result = await deployServiceInternal(serviceId, { + trigger: "restore", + }); if (!("rolloutId" in result) || !result.rolloutId) { throw new Error("Restore could not start a deployment"); } @@ -483,19 +484,33 @@ export const serviceRestoreWorkflow = inngest.createFunction( if (!healthyDeployment || failedDeployment) { await step.run("mark-restore-deployment-failed", async () => { - await db - .update(services) - .set({ - deletedAt: toDate(setup.service.deletedAt), - purgeAfter: toDate(setup.service.purgeAfter), - hostname: null, - originalHostname: setup.service.originalHostname, - deletionStatus: "failed", - deletionError: - failedDeployment?.failedStage || - "Restore deployment did not become healthy", - }) - .where(eq(services.id, serviceId)); + await db.transaction(async (tx) => { + await tx + .update(deployments) + .set(markDeploymentFailedRemoved("restore_failed")) + .where(eq(deployments.rolloutId, deployResult.rolloutId)); + await tx + .update(rollouts) + .set({ + status: "failed", + currentStage: "restore_failed", + completedAt: new Date(), + }) + .where(eq(rollouts.id, deployResult.rolloutId)); + await tx + .update(services) + .set({ + deletedAt: toDate(setup.service.deletedAt), + purgeAfter: toDate(setup.service.purgeAfter), + hostname: null, + originalHostname: setup.service.originalHostname, + deletionStatus: "failed", + deletionError: + failedDeployment?.failedStage || + "Restore deployment did not become healthy", + }) + .where(eq(services.id, serviceId)); + }); }); return { status: "failed", reason: "deployment" }; } diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index 700f623..aba878a 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -216,7 +216,7 @@ export async function checkAndRunScheduledDeployments(): Promise { if (service.sourceType === "github") { await triggerBuild(service.id, "scheduled"); } else { - await deployServiceInternal(service.id); + await deployServiceInternal(service.id, { trigger: "scheduled" }); } console.log( diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 742aa2a..cd52d80 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -1,3 +1,8 @@ +import { + getDefaultServiceHostname, + type ServiceRevisionSpec, +} from "@/lib/service-revision-spec"; + export type ReplicaConfig = { serverId: string; serverName: string; @@ -27,7 +32,8 @@ export type SourceConfig = { export type SecretConfig = { key: string; - updatedAt: string; + updatedAt?: string; + fingerprint?: string; }; export type VolumeConfig = { @@ -78,6 +84,7 @@ export type ConfigChange = { export function buildCurrentConfig( service: { + name: string; image: string; hostname: string | null; healthCheckCmd: string | null; @@ -102,7 +109,11 @@ export function buildCurrentConfig( protocol?: "http" | "tcp" | "udp" | null; tlsPassthrough?: boolean | null; }[], - secrets?: { key: string; updatedAt: Date | string }[], + secrets?: { + key: string; + updatedAt: Date | string; + fingerprint?: string; + }[], volumes?: { name: string; containerPath: string }[], ): DeployedConfig { const hasResourceLimits = @@ -113,7 +124,7 @@ export function buildCurrentConfig( type: "image", image: service.image, }, - hostname: service.hostname ?? undefined, + hostname: service.hostname ?? getDefaultServiceHostname(service.name), stateful: service.stateful ?? false, placement: { replicas: replicas.reduce((sum, r) => sum + r.count, 0), @@ -151,6 +162,7 @@ export function buildCurrentConfig( key: s.key, updatedAt: s.updatedAt instanceof Date ? s.updatedAt.toISOString() : s.updatedAt, + fingerprint: s.fingerprint, })), volumes: (volumes ?? []).map((v) => ({ name: v.name, @@ -481,7 +493,11 @@ export function diffConfigs( from: "(none)", to: key, }); - } else if (deployedSecret.updatedAt !== currentSecret.updatedAt) { + } else if ( + deployedSecret.fingerprint && currentSecret.fingerprint + ? deployedSecret.fingerprint !== currentSecret.fingerprint + : deployedSecret.updatedAt !== currentSecret.updatedAt + ) { changes.push({ field: "Secret", from: key, @@ -537,15 +553,50 @@ export function diffConfigs( return changes; } -export function parseDeployedConfig( - json: string | null, -): DeployedConfig | null { - if (!json) return null; - try { - return JSON.parse(json) as DeployedConfig; - } catch { - return null; - } +export function revisionSpecToDeployedConfig( + specification: ServiceRevisionSpec, + serverNames: Record, + secretFingerprints: Record, +): DeployedConfig { + return { + source: { type: "image", image: specification.image }, + hostname: specification.hostname, + stateful: specification.stateful, + placement: { + replicas: specification.placements.reduce( + (total, placement) => total + placement.count, + 0, + ), + }, + replicas: specification.placements.map((placement) => ({ + serverId: placement.serverId, + serverName: serverNames[placement.serverId] ?? placement.serverId, + count: placement.count, + })), + healthCheck: specification.healthCheck, + startCommand: specification.startCommand, + resourceLimits: + specification.resourceLimits.cpuCores != null || + specification.resourceLimits.memoryMb != null + ? { + cpuCores: specification.resourceLimits.cpuCores, + memoryMb: specification.resourceLimits.memoryMb, + } + : undefined, + ports: specification.ports.map((port) => ({ + port: port.containerPort, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol, + tlsPassthrough: port.tlsPassthrough, + })), + serverless: specification.serverless, + secrets: specification.secrets.map((secret) => ({ + key: secret.key, + fingerprint: secretFingerprints[secret.key], + })), + volumes: specification.volumes, + }; } export function normalizeServerlessConfig( @@ -579,50 +630,3 @@ export function getCurrentServerlessConfig(service: { DEFAULT_SERVERLESS_WAKE_TIMEOUT_SECONDS, }; } - -export function getDeployedServerlessConfig(service: { - deployedConfig?: string | null; - serverlessEnabled?: boolean | null; - serverlessSleepAfterSeconds?: number | null; - serverlessWakeTimeoutSeconds?: number | null; -}): ServerlessConfig { - const deployed = parseDeployedConfig(service.deployedConfig ?? null); - return deployed?.serverless - ? normalizeServerlessConfig(deployed.serverless) - : getCurrentServerlessConfig(service); -} - -export function getDeployedStateful(service: { - deployedConfig?: string | null; - stateful?: boolean | null; -}) { - const deployed = parseDeployedConfig(service.deployedConfig ?? null); - return deployed?.stateful ?? service.stateful ?? false; -} - -export function isDeployedServerlessService(service: { - deployedConfig?: string | null; - stateful?: boolean | null; - serverlessEnabled?: boolean | null; - serverlessSleepAfterSeconds?: number | null; - serverlessWakeTimeoutSeconds?: number | null; -}) { - return getDeployedServerlessConfig(service).enabled; -} - -export function getDeployedServicePorts( - service: { id: string; deployedConfig?: string | null }, - livePorts: PortConfig[], -): PortConfig[] { - const deployed = parseDeployedConfig(service.deployedConfig ?? null); - if (!deployed?.ports) { - return livePorts; - } - return deployed.ports.map((port) => ({ - port: port.port, - isPublic: port.isPublic, - domain: port.domain, - protocol: port.protocol ?? "http", - tlsPassthrough: port.tlsPassthrough, - })); -} diff --git a/web/lib/service-revision-cutover.ts b/web/lib/service-revision-cutover.ts new file mode 100644 index 0000000..cdbeb42 --- /dev/null +++ b/web/lib/service-revision-cutover.ts @@ -0,0 +1,168 @@ +import { + buildServiceRevisionSpec, + type ServiceRevisionDraft, +} from "./service-revision-spec"; + +export type CutoverDeployedConfig = { + source?: { image?: string }; + hostname?: string | null; + stateful?: boolean; + replicas?: Array<{ serverId: string; count: number }>; + healthCheck?: { + cmd: string; + interval: number; + timeout: number; + retries: number; + startPeriod: number; + } | null; + startCommand?: string | null; + resourceLimits?: { + cpuCores?: number | null; + memoryMb?: number | null; + }; + ports?: Array<{ + port: number; + isPublic: boolean; + domain: string | null; + protocol?: "http" | "tcp" | "udp"; + tlsPassthrough?: boolean; + }>; + serverless?: { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + }; + secrets?: Array<{ + key: string; + updatedAt?: string; + }>; + secretKeys?: string[]; + volumes?: Array<{ name: string; containerPath: string }>; +}; + +function assertSecretsMatchDeployedSnapshot( + liveSecrets: ServiceRevisionDraft["secrets"], + deployedConfig: CutoverDeployedConfig, +) { + const deployedSecrets = deployedConfig.secrets; + if (!deployedSecrets) { + if ( + liveSecrets.length > 0 || + (deployedConfig.secretKeys?.length ?? 0) > 0 + ) { + throw new Error( + "Cannot verify deployed secrets because the deployed snapshot has no secret timestamps", + ); + } + return; + } + + const liveSecretsByKey = new Map( + liveSecrets.map((secret) => [secret.key, secret]), + ); + if (liveSecretsByKey.size !== deployedSecrets.length) { + throw new Error("Live secrets differ from the deployed snapshot"); + } + + for (const deployedSecret of deployedSecrets) { + const liveSecret = liveSecretsByKey.get(deployedSecret.key); + const liveUpdatedAt = liveSecret?.updatedAt + ? new Date(liveSecret.updatedAt).toISOString() + : null; + const deployedUpdatedAt = deployedSecret.updatedAt + ? new Date(deployedSecret.updatedAt).toISOString() + : null; + if (!liveSecret || !liveUpdatedAt || liveUpdatedAt !== deployedUpdatedAt) { + throw new Error( + `Secret ${deployedSecret.key} differs from the deployed snapshot`, + ); + } + } +} + +export function buildCutoverServiceRevisionSpec({ + liveDraft, + deployedConfig, +}: { + liveDraft: ServiceRevisionDraft; + deployedConfig: CutoverDeployedConfig | null; +}) { + if (deployedConfig) { + assertSecretsMatchDeployedSnapshot(liveDraft.secrets, deployedConfig); + } + + const currentPortsByIdentity = new Map( + liveDraft.ports.map((port) => [ + `${port.port}:${port.protocol ?? "http"}`, + port, + ]), + ); + const deployedPorts = Array.isArray(deployedConfig?.ports) + ? deployedConfig.ports.map((port) => { + const currentPort = currentPortsByIdentity.get( + `${port.port}:${port.protocol ?? "http"}`, + ); + return { + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? null, + externalPort: currentPort?.externalPort ?? null, + tlsPassthrough: port.tlsPassthrough ?? null, + }; + }) + : null; + const deployedHealthCheck = deployedConfig + ? (deployedConfig.healthCheck ?? null) + : undefined; + + return buildServiceRevisionSpec({ + service: { + ...liveDraft.service, + image: deployedConfig?.source?.image ?? liveDraft.service.image, + hostname: deployedConfig?.hostname ?? liveDraft.service.hostname, + stateful: deployedConfig?.stateful ?? liveDraft.service.stateful, + serverlessEnabled: + deployedConfig?.serverless?.enabled ?? + liveDraft.service.serverlessEnabled, + serverlessSleepAfterSeconds: + deployedConfig?.serverless?.sleepAfterSeconds ?? + liveDraft.service.serverlessSleepAfterSeconds, + serverlessWakeTimeoutSeconds: + deployedConfig?.serverless?.wakeTimeoutSeconds ?? + liveDraft.service.serverlessWakeTimeoutSeconds, + healthCheckCmd: deployedConfig + ? (deployedHealthCheck?.cmd ?? null) + : liveDraft.service.healthCheckCmd, + healthCheckInterval: deployedConfig + ? (deployedHealthCheck?.interval ?? null) + : liveDraft.service.healthCheckInterval, + healthCheckTimeout: deployedConfig + ? (deployedHealthCheck?.timeout ?? null) + : liveDraft.service.healthCheckTimeout, + healthCheckRetries: deployedConfig + ? (deployedHealthCheck?.retries ?? null) + : liveDraft.service.healthCheckRetries, + healthCheckStartPeriod: deployedConfig + ? (deployedHealthCheck?.startPeriod ?? null) + : liveDraft.service.healthCheckStartPeriod, + startCommand: deployedConfig + ? (deployedConfig.startCommand ?? null) + : liveDraft.service.startCommand, + resourceCpuLimit: deployedConfig + ? (deployedConfig.resourceLimits?.cpuCores ?? null) + : liveDraft.service.resourceCpuLimit, + resourceMemoryLimitMb: deployedConfig + ? (deployedConfig.resourceLimits?.memoryMb ?? null) + : liveDraft.service.resourceMemoryLimitMb, + }, + placements: Array.isArray(deployedConfig?.replicas) + ? deployedConfig.replicas + : liveDraft.placements, + ports: deployedPorts ?? liveDraft.ports, + secrets: liveDraft.secrets, + volumes: Array.isArray(deployedConfig?.volumes) + ? deployedConfig.volumes + : liveDraft.volumes, + }); +} diff --git a/web/lib/service-revision-spec.ts b/web/lib/service-revision-spec.ts new file mode 100644 index 0000000..dd23ed6 --- /dev/null +++ b/web/lib/service-revision-spec.ts @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; + +export const SERVICE_REVISION_SCHEMA_VERSION = 1 as const; + +export function getDefaultServiceHostname(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, ""); +} + +export type ServiceRevisionHealthCheck = { + cmd: string; + interval: number; + timeout: number; + retries: number; + startPeriod: number; +}; + +export type ServiceRevisionPlacement = { + serverId: string; + count: number; +}; + +export type ServiceRevisionPort = { + containerPort: number; + isPublic: boolean; + domain: string | null; + protocol: "http" | "tcp" | "udp"; + externalPort: number | null; + tlsPassthrough: boolean; +}; + +export type ServiceRevisionSecret = { + key: string; + encryptedValue: string; +}; + +export type ServiceRevisionVolume = { + name: string; + containerPath: string; +}; + +export type ServiceRevisionSpec = { + schemaVersion: typeof SERVICE_REVISION_SCHEMA_VERSION; + serviceId: string; + image: string; + hostname: string; + stateful: boolean; + serverless: { + enabled: boolean; + sleepAfterSeconds: number; + wakeTimeoutSeconds: number; + }; + healthCheck: ServiceRevisionHealthCheck | null; + startCommand: string | null; + resourceLimits: { + cpuCores: number | null; + memoryMb: number | null; + }; + placements: ServiceRevisionPlacement[]; + ports: ServiceRevisionPort[]; + secrets: ServiceRevisionSecret[]; + volumes: ServiceRevisionVolume[]; +}; + +export type ServiceRevisionDraft = { + service: { + id: string; + name: string; + image: string; + hostname: string | null; + stateful: boolean | null; + serverlessEnabled: boolean | null; + serverlessSleepAfterSeconds: number | null; + serverlessWakeTimeoutSeconds: number | null; + healthCheckCmd: string | null; + healthCheckInterval: number | null; + healthCheckTimeout: number | null; + healthCheckRetries: number | null; + healthCheckStartPeriod: number | null; + startCommand: string | null; + resourceCpuLimit: number | null; + resourceMemoryLimitMb: number | null; + }; + placements: Array<{ serverId: string; count: number }>; + ports: Array<{ + port: number; + isPublic: boolean; + domain: string | null; + protocol: "http" | "tcp" | "udp" | null; + externalPort: number | null; + tlsPassthrough: boolean | null; + }>; + secrets: Array<{ + key: string; + encryptedValue: string; + updatedAt?: Date | string; + }>; + volumes: Array<{ name: string; containerPath: string }>; +}; + +function compareStrings(a: string, b: string) { + return a.localeCompare(b, "en"); +} + +export function buildServiceRevisionSpec( + draft: ServiceRevisionDraft, +): ServiceRevisionSpec { + const { service } = draft; + + return { + schemaVersion: SERVICE_REVISION_SCHEMA_VERSION, + serviceId: service.id, + image: service.image.trim(), + hostname: + service.hostname?.trim() || getDefaultServiceHostname(service.name), + stateful: service.stateful ?? false, + serverless: { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: Math.max( + service.serverlessSleepAfterSeconds ?? 300, + 120, + ), + wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, + }, + healthCheck: service.healthCheckCmd + ? { + cmd: service.healthCheckCmd, + interval: service.healthCheckInterval ?? 10, + timeout: service.healthCheckTimeout ?? 5, + retries: service.healthCheckRetries ?? 3, + startPeriod: service.healthCheckStartPeriod ?? 30, + } + : null, + startCommand: service.startCommand?.trim() || null, + resourceLimits: { + cpuCores: service.resourceCpuLimit, + memoryMb: service.resourceMemoryLimitMb, + }, + placements: draft.placements + .filter((placement) => placement.count > 0) + .map((placement) => ({ + serverId: placement.serverId, + count: placement.count, + })) + .sort((a, b) => compareStrings(a.serverId, b.serverId)), + ports: draft.ports + .map((port) => ({ + containerPort: port.port, + isPublic: port.isPublic, + domain: port.domain?.trim() || null, + protocol: port.protocol ?? "http", + externalPort: port.externalPort, + tlsPassthrough: port.tlsPassthrough ?? false, + })) + .sort( + (a, b) => + a.containerPort - b.containerPort || + compareStrings(a.protocol, b.protocol) || + compareStrings(a.domain ?? "", b.domain ?? "") || + (a.externalPort ?? 0) - (b.externalPort ?? 0), + ), + secrets: draft.secrets + .map((secret) => ({ + key: secret.key, + encryptedValue: secret.encryptedValue, + })) + .sort((a, b) => compareStrings(a.key, b.key)), + volumes: draft.volumes + .map((volume) => ({ + name: volume.name, + containerPath: volume.containerPath, + })) + .sort( + (a, b) => + compareStrings(a.name, b.name) || + compareStrings(a.containerPath, b.containerPath), + ), + }; +} + +export function hashServiceRevisionSpec(spec: ServiceRevisionSpec): string { + return createHash("sha256").update(JSON.stringify(spec)).digest("hex"); +} diff --git a/web/lib/service-revisions.ts b/web/lib/service-revisions.ts new file mode 100644 index 0000000..01374b4 --- /dev/null +++ b/web/lib/service-revisions.ts @@ -0,0 +1,186 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import { db } from "@/db"; +import { + rollouts, + secrets, + servicePorts, + serviceReplicas, + serviceRevisions, + services, + serviceVolumes, +} from "@/db/schema"; +import { + buildServiceRevisionSpec, + hashServiceRevisionSpec, + SERVICE_REVISION_SCHEMA_VERSION, + type ServiceRevisionSpec, +} from "@/lib/service-revision-spec"; + +export type RevisionSourceMetadata = Record< + string, + string | number | boolean | null | undefined +>; + +function validateRevisionSpec(spec: ServiceRevisionSpec) { + const totalReplicas = spec.placements.reduce( + (sum, placement) => sum + placement.count, + 0, + ); + + if (totalReplicas < 1) { + throw new Error("At least one replica is required"); + } + if (totalReplicas > 10) { + throw new Error("Maximum 10 replicas allowed"); + } + if (spec.stateful && totalReplicas !== 1) { + throw new Error("Stateful services can only have exactly 1 replica"); + } + if (spec.stateful && spec.placements.length !== 1) { + throw new Error("Stateful services must be deployed to exactly one server"); + } +} + +function compactSourceMetadata(metadata: RevisionSourceMetadata) { + return Object.fromEntries( + Object.entries(metadata).filter((entry) => entry[1] !== undefined), + ) as Record; +} + +export async function createRolloutWithServiceRevision( + serviceId: string, + sourceMetadata: RevisionSourceMetadata = {}, +) { + return db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); + + const service = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .then((rows) => rows[0]); + + if (!service) { + throw new Error("Service not found"); + } + + const placements = await tx + .select({ + serverId: serviceReplicas.serverId, + count: serviceReplicas.count, + }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const ports = await tx + .select() + .from(servicePorts) + .where(eq(servicePorts.serviceId, serviceId)); + const revisionSecrets = await tx + .select({ + key: secrets.key, + encryptedValue: secrets.encryptedValue, + }) + .from(secrets) + .where(eq(secrets.serviceId, serviceId)); + const volumes = await tx + .select({ + name: serviceVolumes.name, + containerPath: serviceVolumes.containerPath, + }) + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)); + + const specification = buildServiceRevisionSpec({ + service, + placements, + ports, + secrets: revisionSecrets, + volumes, + }); + validateRevisionSpec(specification); + + const contentHash = hashServiceRevisionSpec(specification); + let revision = await tx + .select() + .from(serviceRevisions) + .where( + and( + eq(serviceRevisions.serviceId, serviceId), + eq(serviceRevisions.contentHash, contentHash), + ), + ) + .then((rows) => rows[0]); + + if (!revision) { + const [{ nextRevisionNumber }] = await tx + .select({ + nextRevisionNumber: sql`coalesce(max(${serviceRevisions.revisionNumber}), 0) + 1`, + }) + .from(serviceRevisions) + .where(eq(serviceRevisions.serviceId, serviceId)); + + revision = await tx + .insert(serviceRevisions) + .values({ + id: randomUUID(), + serviceId, + revisionNumber: nextRevisionNumber, + schemaVersion: SERVICE_REVISION_SCHEMA_VERSION, + specification, + contentHash, + sourceMetadata: compactSourceMetadata({ + sourceType: service.sourceType, + ...sourceMetadata, + }), + }) + .returning() + .then((rows) => rows[0]); + } + if (!revision) { + throw new Error("Failed to create service revision"); + } + + const rolloutId = randomUUID(); + await tx.insert(rollouts).values({ + id: rolloutId, + serviceId, + serviceRevisionId: revision.id, + status: "queued", + currentStage: "queued", + }); + + return { rolloutId, revision }; + }); +} + +export async function getRolloutServiceRevision(rolloutId: string) { + const result = await db + .select({ + rolloutServiceId: rollouts.serviceId, + revision: serviceRevisions, + }) + .from(rollouts) + .innerJoin( + serviceRevisions, + eq(rollouts.serviceRevisionId, serviceRevisions.id), + ) + .where(eq(rollouts.id, rolloutId)) + .then((rows) => rows[0]); + + if (!result) { + throw new Error("Rollout revision not found"); + } + if (result.rolloutServiceId !== result.revision.serviceId) { + throw new Error("Rollout revision does not belong to service"); + } + if ( + result.revision.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || + result.revision.specification.schemaVersion !== + SERVICE_REVISION_SCHEMA_VERSION + ) { + throw new Error("Unsupported service revision schema version"); + } + + return result.revision; +} diff --git a/web/lib/wireguard.ts b/web/lib/wireguard.ts index cb564a7..5f47dd6 100644 --- a/web/lib/wireguard.ts +++ b/web/lib/wireguard.ts @@ -1,8 +1,8 @@ -import { db } from "@/db"; -import { servers, deployments } from "@/db/schema"; -import { eq, isNotNull, and, ne } from "drizzle-orm"; -import { WIREGUARD_SUBNET_PREFIX, CONTAINER_SUBNET_PREFIX } from "./constants"; +import { and, isNotNull, ne } from "drizzle-orm"; import { Address4 } from "ip-address"; +import { db } from "@/db"; +import { servers } from "@/db/schema"; +import { CONTAINER_SUBNET_PREFIX, WIREGUARD_SUBNET_PREFIX } from "./constants"; function sameSubnet(ip1: string, ip2: string, prefix: number = 16): boolean { if (!ip1 || !ip2) return false; @@ -36,28 +36,14 @@ export async function assignSubnet(): Promise<{ throw new Error("No available subnets"); } -export async function assignContainerIp(serverId: string): Promise { - const server = await db - .select({ subnetId: servers.subnetId }) - .from(servers) - .where(eq(servers.id, serverId)) - .then((r) => r[0]); - - if (!server?.subnetId) { - throw new Error("Server does not have a subnet assigned"); - } - - const existingDeployments = await db - .select({ ipAddress: deployments.ipAddress }) - .from(deployments) - .where( - and(eq(deployments.serverId, serverId), isNotNull(deployments.ipAddress)), - ); - - const usedIps = new Set(existingDeployments.map((d) => d.ipAddress)); +export function findAvailableContainerIp( + subnetId: number, + existingIps: Iterable, +): string { + const usedIps = new Set(existingIps); for (let hostPart = 2; hostPart <= 254; hostPart++) { - const ip = `${CONTAINER_SUBNET_PREFIX}.${server.subnetId}.${hostPart}`; + const ip = `${CONTAINER_SUBNET_PREFIX}.${subnetId}.${hostPart}`; if (!usedIps.has(ip)) { return ip; } diff --git a/web/package.json b/web/package.json index 8bae61e..9e2e355 100644 --- a/web/package.json +++ b/web/package.json @@ -12,6 +12,7 @@ "admin:create": "node scripts/admin.mjs --create", "admin:reset-password": "node scripts/admin.mjs --reset-password", "db:push": "drizzle-kit push", + "db:cutover-service-revisions": "tsx scripts/cutover-service-revisions.ts", "db:studio": "drizzle-kit studio" }, "dependencies": { diff --git a/web/scripts/cutover-service-revisions.ts b/web/scripts/cutover-service-revisions.ts new file mode 100644 index 0000000..931b060 --- /dev/null +++ b/web/scripts/cutover-service-revisions.ts @@ -0,0 +1,434 @@ +import { randomUUID } from "node:crypto"; +import { Pool, type PoolClient } from "pg"; +import { + buildCutoverServiceRevisionSpec, + type CutoverDeployedConfig, +} from "../lib/service-revision-cutover"; +import { + hashServiceRevisionSpec, + SERVICE_REVISION_SCHEMA_VERSION, +} from "../lib/service-revision-spec"; + +const connectionString = process.env.DATABASE_URL; +if (!connectionString) { + throw new Error("DATABASE_URL is required"); +} + +const pool = new Pool({ connectionString }); + +async function baseSchemaExists(client: PoolClient) { + const result = await client.query<{ exists: boolean }>( + `SELECT to_regclass('services') IS NOT NULL AS exists`, + ); + return result.rows[0]?.exists ?? false; +} + +async function cutoverIsRequired(client: PoolClient) { + const result = await client.query<{ required: boolean }>(` + SELECT + to_regclass('service_revisions') IS NULL + OR NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'deployments' + AND column_name = 'service_revision_id' + ) + OR NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'rollouts' + AND column_name = 'service_revision_id' + ) + OR NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'services' + AND column_name = 'active_revision_id' + ) + OR NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'deployment_ports' + AND column_name = 'container_port' + ) + OR EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'deployment_ports' + AND column_name = 'service_port_id' + ) AS required + `); + return result.rows[0]?.required ?? true; +} + +async function prepareSchema(client: PoolClient) { + await client.query(` + CREATE TABLE IF NOT EXISTS service_revisions ( + id text PRIMARY KEY, + service_id text NOT NULL CONSTRAINT service_revisions_service_id_services_id_fk + REFERENCES services(id) ON DELETE CASCADE, + revision_number integer NOT NULL, + schema_version integer NOT NULL, + specification jsonb NOT NULL, + content_hash text NOT NULL, + source_metadata jsonb, + created_at timestamptz NOT NULL DEFAULT now() + ) + `); + await client.query(` + ALTER TABLE services ADD COLUMN IF NOT EXISTS active_revision_id text; + ALTER TABLE deployments ADD COLUMN IF NOT EXISTS service_revision_id text; + ALTER TABLE rollouts ADD COLUMN IF NOT EXISTS service_revision_id text; + ALTER TABLE deployment_ports ADD COLUMN IF NOT EXISTS container_port integer + `); + await client.query(` + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'deployment_ports' + AND column_name = 'service_port_id' + ) THEN + UPDATE deployment_ports AS deployment_port + SET container_port = service_port.port + FROM service_ports AS service_port + WHERE deployment_port.service_port_id = service_port.id + AND deployment_port.container_port IS NULL; + END IF; + END $$ + `); +} + +async function captureBootstrapRevision(client: PoolClient, serviceId: string) { + const serviceResult = await client.query( + `SELECT * FROM services WHERE id = $1 FOR UPDATE`, + [serviceId], + ); + const service = serviceResult.rows[0]; + if (!service) throw new Error(`Service ${serviceId} not found`); + let deployedConfig: CutoverDeployedConfig | null = null; + if (service.deployed_config) { + try { + deployedConfig = JSON.parse( + service.deployed_config, + ) as CutoverDeployedConfig; + } catch { + throw new Error(`Service ${serviceId} has invalid deployed_config JSON`); + } + } + + const [placementResult, portResult, secretResult, volumeResult] = + await Promise.all([ + client.query( + `SELECT server_id, count FROM service_replicas WHERE service_id = $1`, + [serviceId], + ), + client.query( + `SELECT port, is_public, domain, protocol, external_port, tls_passthrough + FROM service_ports WHERE service_id = $1`, + [serviceId], + ), + client.query( + `SELECT key, encrypted_value, updated_at FROM secrets WHERE service_id = $1`, + [serviceId], + ), + client.query( + `SELECT name, container_path FROM service_volumes WHERE service_id = $1`, + [serviceId], + ), + ]); + + const specification = buildCutoverServiceRevisionSpec({ + deployedConfig, + liveDraft: { + service: { + id: service.id, + name: service.name, + image: service.image, + hostname: service.hostname, + stateful: service.stateful, + serverlessEnabled: service.serverless_enabled, + serverlessSleepAfterSeconds: service.serverless_sleep_after_seconds, + serverlessWakeTimeoutSeconds: service.serverless_wake_timeout_seconds, + healthCheckCmd: service.health_check_cmd, + healthCheckInterval: service.health_check_interval, + healthCheckTimeout: service.health_check_timeout, + healthCheckRetries: service.health_check_retries, + healthCheckStartPeriod: service.health_check_start_period, + startCommand: service.start_command, + resourceCpuLimit: service.resource_cpu_limit, + resourceMemoryLimitMb: service.resource_memory_limit_mb, + }, + placements: placementResult.rows.map((placement) => ({ + serverId: placement.server_id, + count: placement.count, + })), + ports: portResult.rows.map((port) => ({ + port: port.port, + isPublic: port.is_public, + domain: port.domain, + protocol: port.protocol, + externalPort: port.external_port, + tlsPassthrough: port.tls_passthrough, + })), + secrets: secretResult.rows.map((secret) => ({ + key: secret.key, + encryptedValue: secret.encrypted_value, + updatedAt: secret.updated_at, + })), + volumes: volumeResult.rows.map((volume) => ({ + name: volume.name, + containerPath: volume.container_path, + })), + }, + }); + + const totalReplicas = specification.placements.reduce( + (total, placement) => total + placement.count, + 0, + ); + const activeDeploymentResult = await client.query<{ active: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM deployments + WHERE service_id = $1 + AND runtime_desired_state <> 'removed' + AND traffic_state <> 'inactive' + ) AS active`, + [serviceId], + ); + const hasActiveDeployment = activeDeploymentResult.rows[0]?.active ?? false; + if (hasActiveDeployment && (totalReplicas < 1 || totalReplicas > 10)) { + throw new Error( + `Service ${serviceId} has invalid replica count ${totalReplicas}`, + ); + } + if ( + hasActiveDeployment && + specification.stateful && + (totalReplicas !== 1 || specification.placements.length !== 1) + ) { + throw new Error(`Stateful service ${serviceId} has invalid placement`); + } + if (hasActiveDeployment) { + const activeDeployments = await client.query<{ id: string }>( + `SELECT id FROM deployments + WHERE service_id = $1 + AND runtime_desired_state <> 'removed' + AND traffic_state <> 'inactive'`, + [serviceId], + ); + const expectedPorts = specification.ports + .map((port) => port.containerPort) + .sort((a, b) => a - b); + for (const deployment of activeDeployments.rows) { + const allocatedPorts = await client.query<{ container_port: number }>( + `SELECT container_port FROM deployment_ports + WHERE deployment_id = $1 ORDER BY container_port`, + [deployment.id], + ); + if ( + JSON.stringify( + allocatedPorts.rows.map((port) => port.container_port), + ) !== JSON.stringify(expectedPorts) + ) { + throw new Error( + `Deployment ${deployment.id} has incomplete runtime port allocation`, + ); + } + } + } + + const contentHash = hashServiceRevisionSpec(specification); + const existingRevision = await client.query( + `SELECT id FROM service_revisions WHERE service_id = $1 AND content_hash = $2`, + [serviceId, contentHash], + ); + let revisionId = existingRevision.rows[0]?.id as string | undefined; + if (!revisionId) { + revisionId = randomUUID(); + await client.query( + `INSERT INTO service_revisions ( + id, service_id, revision_number, schema_version, specification, + content_hash, source_metadata + ) VALUES ( + $1, $2, + (SELECT coalesce(max(revision_number), 0) + 1 FROM service_revisions WHERE service_id = $2), + $3, $4, $5, $6 + )`, + [ + revisionId, + serviceId, + SERVICE_REVISION_SCHEMA_VERSION, + JSON.stringify(specification), + contentHash, + JSON.stringify({ sourceType: "cutover_bootstrap" }), + ], + ); + } + + await client.query( + `UPDATE deployments SET service_revision_id = $1 + WHERE service_id = $2 AND service_revision_id IS NULL`, + [revisionId, serviceId], + ); + await client.query( + `UPDATE rollouts SET service_revision_id = $1 + WHERE service_id = $2 AND service_revision_id IS NULL`, + [revisionId, serviceId], + ); + await client.query( + `UPDATE services SET active_revision_id = $1 + WHERE id = $2 AND active_revision_id IS NULL + AND EXISTS ( + SELECT 1 FROM deployments + WHERE deployments.service_id = services.id + AND deployments.runtime_desired_state <> 'removed' + AND deployments.traffic_state <> 'inactive' + )`, + [revisionId, serviceId], + ); +} + +async function finalizeSchema(client: PoolClient) { + await client.query(` + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM deployments WHERE service_revision_id IS NULL) THEN + RAISE EXCEPTION 'deployments.service_revision_id backfill incomplete'; + END IF; + IF EXISTS (SELECT 1 FROM rollouts WHERE service_revision_id IS NULL) THEN + RAISE EXCEPTION 'rollouts.service_revision_id backfill incomplete'; + END IF; + IF EXISTS (SELECT 1 FROM deployment_ports WHERE container_port IS NULL) THEN + RAISE EXCEPTION 'deployment_ports.container_port backfill incomplete'; + END IF; + END $$ + `); + const duplicateIps = await client.query<{ + server_id: string; + ip_address: string; + deployment_ids: string[]; + }>(` + SELECT server_id, ip_address, array_agg(id ORDER BY id) AS deployment_ids + FROM deployments + WHERE ip_address IS NOT NULL + GROUP BY server_id, ip_address + HAVING count(*) > 1 + ORDER BY server_id, ip_address + LIMIT 20 + `); + if (duplicateIps.rows.length > 0) { + const conflicts = duplicateIps.rows + .map( + (row) => + `server ${row.server_id} IP ${row.ip_address}: ${row.deployment_ids.join(", ")}`, + ) + .join("; "); + throw new Error( + `Duplicate container IP allocations prevent cutover: ${conflicts}`, + ); + } + await client.query(` + CREATE UNIQUE INDEX IF NOT EXISTS service_revisions_service_revision_number_idx + ON service_revisions(service_id, revision_number); + CREATE UNIQUE INDEX IF NOT EXISTS service_revisions_service_content_hash_idx + ON service_revisions(service_id, content_hash); + CREATE INDEX IF NOT EXISTS service_revisions_service_id_idx + ON service_revisions(service_id); + CREATE INDEX IF NOT EXISTS deployments_service_revision_id_idx + ON deployments(service_revision_id); + CREATE UNIQUE INDEX IF NOT EXISTS deployments_server_ip_address_idx + ON deployments(server_id, ip_address); + CREATE INDEX IF NOT EXISTS rollouts_service_revision_id_idx + ON rollouts(service_revision_id) + `); + await client.query(` + ALTER TABLE deployments ALTER COLUMN service_revision_id SET NOT NULL; + ALTER TABLE rollouts ALTER COLUMN service_revision_id SET NOT NULL; + ALTER TABLE deployment_ports ALTER COLUMN container_port SET NOT NULL + `); + await client.query(` + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'service_revisions_id_service_id_unique') THEN + ALTER TABLE service_revisions ADD CONSTRAINT service_revisions_id_service_id_unique + UNIQUE (id, service_id); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'services_active_revision_id_service_revisions_id_fk') THEN + ALTER TABLE services ADD CONSTRAINT services_active_revision_id_service_revisions_id_fk + FOREIGN KEY (active_revision_id) REFERENCES service_revisions(id) ON DELETE SET NULL; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'deployments_service_revision_service_fk') THEN + ALTER TABLE deployments ADD CONSTRAINT deployments_service_revision_service_fk + FOREIGN KEY (service_revision_id, service_id) + REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'rollouts_service_revision_service_fk') THEN + ALTER TABLE rollouts ADD CONSTRAINT rollouts_service_revision_service_fk + FOREIGN KEY (service_revision_id, service_id) + REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; + END IF; + END $$ + `); + await client.query( + `ALTER TABLE deployment_ports DROP COLUMN IF EXISTS service_port_id`, + ); +} + +async function main() { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await client.query( + "SELECT pg_advisory_xact_lock(hashtext('service-revisions-cutover'))", + ); + if (!(await baseSchemaExists(client))) { + await client.query("COMMIT"); + console.log( + "Fresh database detected; schema push will create service revisions.", + ); + return; + } + if (!(await cutoverIsRequired(client))) { + await client.query("COMMIT"); + console.log("Service revision cutover already complete; skipping."); + return; + } + const activeRollouts = await client.query<{ count: string }>( + `SELECT count(*)::text AS count FROM rollouts WHERE status IN ('queued', 'in_progress')`, + ); + if (Number(activeRollouts.rows[0]?.count ?? 0) > 0) { + throw new Error("Cutover requires zero queued or in-progress rollouts"); + } + await prepareSchema(client); + + const serviceIds = await client.query<{ service_id: string }>(` + SELECT DISTINCT service_id FROM deployments + UNION + SELECT DISTINCT service_id FROM rollouts + `); + for (const { service_id: serviceId } of serviceIds.rows) { + try { + await captureBootstrapRevision(client, serviceId); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Service ${serviceId}: ${message}`); + } + } + + await finalizeSchema(client); + await client.query("COMMIT"); + console.log( + `Service revision cutover complete for ${serviceIds.rowCount ?? 0} services.`, + ); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + await pool.end(); + } +} + +await main(); diff --git a/web/tests/agent-capabilities.test.ts b/web/tests/agent-capabilities.test.ts new file mode 100644 index 0000000..388df2f --- /dev/null +++ b/web/tests/agent-capabilities.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { + getAgentCompatibilityStatus, + SERVICE_REVISION_CAPABILITY, +} from "@/lib/agent-capabilities"; + +describe("agent compatibility", () => { + it("requires an explicit service revision capability", () => { + expect(getAgentCompatibilityStatus(null)).toBe("upgrade_required"); + expect( + getAgentCompatibilityStatus({ + version: "1.0.0", + uptimeSecs: 10, + capabilities: ["serverless_gateway"], + }), + ).toBe("upgrade_required"); + }); + + it("accepts agents advertising the revision contract", () => { + expect( + getAgentCompatibilityStatus({ + version: "1.0.0", + uptimeSecs: 10, + capabilities: [SERVICE_REVISION_CAPABILITY], + }), + ).toBe("compatible"); + }); +}); diff --git a/web/tests/deployment-status.test.ts b/web/tests/deployment-status.test.ts index 06d958c..6bb8226 100644 --- a/web/tests/deployment-status.test.ts +++ b/web/tests/deployment-status.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest"; import { isDeploymentRoutable, - isRuntimeExpected, isObservedReady, + isRuntimeExpected, markDeploymentRemoved, + selectNewestRevisionId, } from "@/lib/deployment-status"; describe("deployment state helpers", () => { @@ -46,3 +47,20 @@ describe("deployment state helpers", () => { }); }); }); + +describe("selectNewestRevisionId", () => { + it("selects the highest revision deterministically", () => { + expect( + selectNewestRevisionId([ + { serviceRevisionId: "revision-1", revisionNumber: 1 }, + { serviceRevisionId: "revision-3", revisionNumber: 3 }, + { serviceRevisionId: "revision-2", revisionNumber: 2 }, + { serviceRevisionId: "revision-3", revisionNumber: 3 }, + ]), + ).toBe("revision-3"); + }); + + it("returns null when there are no active revisions", () => { + expect(selectNewestRevisionId([])).toBeNull(); + }); +}); diff --git a/web/tests/expected-state-route.test.ts b/web/tests/expected-state-route.test.ts new file mode 100644 index 0000000..8a39e51 --- /dev/null +++ b/web/tests/expected-state-route.test.ts @@ -0,0 +1,93 @@ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + buildAgentExpectedState: vi.fn(), + getServer: vi.fn(), + verifyAgentRequest: vi.fn(), +})); + +vi.mock("@/lib/agent/expected-state", () => ({ + buildAgentExpectedState: mocks.buildAgentExpectedState, + getServer: mocks.getServer, +})); +vi.mock("@/lib/agent-auth", () => ({ + verifyAgentRequest: mocks.verifyAgentRequest, +})); + +import { GET } from "@/app/api/v1/agent/expected-state/route"; + +describe("agent expected-state route", () => { + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.EXPECTED_STATE_MAINTENANCE_MODE; + mocks.verifyAgentRequest.mockResolvedValue({ + success: true, + serverId: "server-1", + }); + mocks.getServer.mockResolvedValue({ + id: "server-1", + agentHealth: { version: "old", uptimeSecs: 60, capabilities: [] }, + }); + }); + + it("returns a structured 426 for an incompatible agent", async () => { + const response = await GET( + new NextRequest("http://localhost/api/v1/agent/expected-state"), + ); + + expect(response.status).toBe(426); + expect(await response.json()).toEqual({ + error: "Agent upgrade required", + code: "AGENT_UPGRADE_REQUIRED", + requiredCapabilities: ["service_revision_v1"], + }); + expect(mocks.buildAgentExpectedState).not.toHaveBeenCalled(); + }); + + it("pauses expected state during the maintenance window", async () => { + process.env.EXPECTED_STATE_MAINTENANCE_MODE = "true"; + mocks.getServer.mockResolvedValue({ + id: "server-1", + agentHealth: { + version: "new", + uptimeSecs: 60, + capabilities: ["service_revision_v1"], + }, + }); + + const response = await GET( + new NextRequest("http://localhost/api/v1/agent/expected-state"), + ); + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("30"); + expect(await response.json()).toMatchObject({ + code: "EXPECTED_STATE_MAINTENANCE", + }); + }); + + it("returns a structured retryable error when state construction fails", async () => { + mocks.getServer.mockResolvedValue({ + id: "server-1", + agentHealth: { + version: "new", + uptimeSecs: 60, + capabilities: ["service_revision_v1"], + }, + }); + mocks.buildAgentExpectedState.mockRejectedValue( + new Error("incomplete port allocation"), + ); + + const response = await GET( + new NextRequest("http://localhost/api/v1/agent/expected-state"), + ); + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("15"); + expect(await response.json()).toMatchObject({ + code: "EXPECTED_STATE_BUILD_FAILED", + }); + }); +}); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 4848957..eaf13dc 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -13,28 +13,56 @@ import { buildServerlessTraefikRouteSets, buildTraefikCertificateDomains, buildTraefikRoutes, + selectRuntimeServiceRevisions, } from "@/lib/agent/expected-state"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; describe("expected-state pure builders", () => { - const deployedServerlessConfig = JSON.stringify({ - source: { type: "image", image: "nginx" }, - stateful: false, - replicas: [], - healthCheck: null, - ports: [ - { - port: 3000, - isPublic: true, - domain: "sleepy.example.com", - protocol: "http", + function revision( + serviceId: string, + overrides: Partial = {}, + ) { + return { + id: `rev_${serviceId}`, + name: serviceId, + serviceId, + schemaVersion: 1, + revisionId: `rev_${serviceId}`, + specification: { + schemaVersion: 1, + serviceId, + image: "nginx", + hostname: serviceId, + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [], + ports: [], + secrets: [], + volumes: [], + ...overrides, }, - ], - serverless: { - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - }, - }); + } as any; + } + + function runtimeRevision( + serviceId: string, + overrides: Partial = {}, + ) { + const revisionRow = revision(serviceId, overrides); + return { + id: serviceId, + name: serviceId, + revisionId: revisionRow.id, + specification: revisionRow.specification, + }; + } it("groups container inputs by deployment and service deterministically", () => { const containers = buildExpectedContainersFromRows({ @@ -42,6 +70,7 @@ describe("expected-state pure builders", () => { { id: "dep_bbbbbbbb", serviceId: "svc_1", + serviceRevisionId: "rev_svc_1", ipAddress: "10.0.0.2", runtimeDesiredState: "stopped", trafficState: "active", @@ -50,12 +79,57 @@ describe("expected-state pure builders", () => { { id: "dep_aaaaaaaa", serviceId: "svc_1", + serviceRevisionId: "rev_svc_1", ipAddress: "10.0.0.1", runtimeDesiredState: "running", trafficState: "active", observedPhase: "running", }, ] as any, + revisions: [ + revision("svc_1", { + startCommand: "npm start", + healthCheck: { + cmd: "curl -f /health", + interval: 10, + timeout: 5, + retries: 3, + startPeriod: 30, + }, + resourceLimits: { cpuCores: 1, memoryMb: 512 }, + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + ports: [ + { + containerPort: 80, + isPublic: true, + domain: "api.example.com", + protocol: "http", + externalPort: null, + tlsPassthrough: false, + }, + { + containerPort: 3000, + isPublic: false, + domain: null, + protocol: "http", + externalPort: null, + tlsPassthrough: false, + }, + ], + secrets: [ + { key: "ZED", encryptedValue: "last" }, + { key: "ALPHA", encryptedValue: "first" }, + ], + volumes: [ + { name: "cache", containerPath: "/var/cache" }, + { name: "data", containerPath: "/data" }, + ], + }), + ], services: [ { id: "svc_1", @@ -75,14 +149,8 @@ describe("expected-state pure builders", () => { deploymentPorts: [ { deploymentId: "dep_aaaaaaaa", hostPort: 30001, containerPort: 3000 }, { deploymentId: "dep_aaaaaaaa", hostPort: 80, containerPort: 80 }, - ] as any, - secrets: [ - { serviceId: "svc_1", key: "ZED", encryptedValue: "last" }, - { serviceId: "svc_1", key: "ALPHA", encryptedValue: "first" }, - ] as any, - volumes: [ - { serviceId: "svc_1", name: "cache", containerPath: "/var/cache" }, - { serviceId: "svc_1", name: "data", containerPath: "/data" }, + { deploymentId: "dep_bbbbbbbb", hostPort: 30002, containerPort: 3000 }, + { deploymentId: "dep_bbbbbbbb", hostPort: 81, containerPort: 80 }, ] as any, }); @@ -118,6 +186,176 @@ describe("expected-state pure builders", () => { }); }); + it("keeps the spec hash stable for duplicate container ports", () => { + const build = (hostPorts: number[]) => + buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_dns", + serviceId: "svc_dns", + serviceRevisionId: "rev_svc_dns", + runtimeDesiredState: "running", + }, + ] as any, + services: [{ id: "svc_dns", name: "dns" }] as any, + revisions: [ + revision("svc_dns", { + ports: [ + { + containerPort: 53, + isPublic: true, + domain: null, + protocol: "tcp", + externalPort: null, + tlsPassthrough: false, + }, + { + containerPort: 53, + isPublic: true, + domain: null, + protocol: "udp", + externalPort: null, + tlsPassthrough: false, + }, + ], + }), + ], + deploymentPorts: hostPorts.map((hostPort) => ({ + deploymentId: "dep_dns", + containerPort: 53, + hostPort, + })) as any, + })[0]; + + const first = build([30002, 30001]); + const second = build([30001, 30002]); + + expect(first.ports).toEqual([ + { containerPort: 53, hostPort: 30001 }, + { containerPort: 53, hostPort: 30002 }, + ]); + expect(first.containerSpecHash).toBe(second.containerSpecHash); + }); + + it("rejects partial expected state when a deployment revision is missing", () => { + expect(() => + buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_missing_revision", + serviceId: "svc_1", + serviceRevisionId: "rev_missing", + runtimeDesiredState: "running", + }, + ] as any, + services: [{ id: "svc_1", name: "api" }] as any, + revisions: [], + deploymentPorts: [], + }), + ).toThrow("Deployment dep_missing_revision has no service revision"); + }); + + it("omits deployments whose service was soft-deleted", () => { + expect( + buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_deleted_service", + serviceId: "svc_1", + serviceRevisionId: "rev_svc_1", + runtimeDesiredState: "running", + }, + ] as any, + services: [], + revisions: [revision("svc_1")], + deploymentPorts: [], + }), + ).toEqual([]); + }); + + it("rejects partial expected state when deployment ports are incomplete", () => { + expect(() => + buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_incomplete_ports", + serviceId: "svc_1", + serviceRevisionId: "rev_svc_1", + runtimeDesiredState: "running", + }, + ] as any, + services: [{ id: "svc_1", name: "api" }] as any, + revisions: [ + revision("svc_1", { + ports: [ + { + containerPort: 3000, + isPublic: false, + domain: null, + protocol: "http", + externalPort: null, + tlsPassthrough: false, + }, + ], + }), + ], + deploymentPorts: [], + }), + ).toThrow("Deployment dep_incomplete_ports has incomplete port allocation"); + }); + + it("contains multiple active revisions to the authoritative revision", () => { + const specification = runtimeRevision("svc_1").specification; + const result = selectRuntimeServiceRevisions([ + { + deploymentId: "dep_old", + serviceId: "svc_1", + serviceName: "api", + serviceActiveRevisionId: "rev_old", + revisionId: "rev_old", + revisionServiceId: "svc_1", + revisionSchemaVersion: 1, + specification, + }, + { + deploymentId: "dep_new", + serviceId: "svc_1", + serviceName: "api", + serviceActiveRevisionId: "rev_old", + revisionId: "rev_new", + revisionServiceId: "svc_1", + revisionSchemaVersion: 1, + specification, + }, + ]); + + expect(result.services).toHaveLength(1); + expect(result.services[0]?.revisionId).toBe("rev_old"); + expect(result.errors[0]).toContain("multiple active revisions"); + }); + + it("excludes desired running state from the container creation hash", () => { + const build = (runtimeDesiredState: "running" | "stopped") => + buildExpectedContainersFromRows({ + deployments: [ + { + id: "dep_1", + serviceId: "svc_1", + serviceRevisionId: "rev_svc_1", + ipAddress: "10.0.0.1", + runtimeDesiredState, + }, + ] as any, + services: [{ id: "svc_1", name: "api" }] as any, + revisions: [revision("svc_1")], + deploymentPorts: [], + })[0]; + + expect(build("running").containerSpecHash).toBe( + build("stopped").containerSpecHash, + ); + }); + it("keeps HTTP local upstreams before remote upstreams", () => { const routes = buildTraefikRoutes({ serverId: "server_local", @@ -162,12 +400,22 @@ describe("expected-state pure builders", () => { { id: "dep_sleeping", serviceId: "svc_private", + serviceRevisionId: "rev_svc_private", ipAddress: "10.0.0.10", runtimeDesiredState: "running", trafficState: "active", observedPhase: "running", }, ] as any, + revisions: [ + revision("svc_private", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + }), + ], services: [ { id: "svc_private", @@ -177,8 +425,6 @@ describe("expected-state pure builders", () => { }, ] as any, deploymentPorts: [], - secrets: [], - volumes: [], }); expect(containers[0]).toMatchObject({ @@ -193,12 +439,14 @@ describe("expected-state pure builders", () => { { id: "dep_stopped", serviceId: "svc_private", + serviceRevisionId: "rev_svc_private", ipAddress: "10.0.0.10", runtimeDesiredState: "stopped", trafficState: "active", observedPhase: "sleeping", }, ] as any, + revisions: [revision("svc_private")], services: [ { id: "svc_private", @@ -208,8 +456,6 @@ describe("expected-state pure builders", () => { }, ] as any, deploymentPorts: [], - secrets: [], - volumes: [], }); expect(containers[0]).toMatchObject({ @@ -225,12 +471,22 @@ describe("expected-state pure builders", () => { { id: "dep_sleeping", serviceId: "svc_public", + serviceRevisionId: "rev_svc_public", ipAddress: "10.0.0.10", runtimeDesiredState: "stopped", trafficState: "active", observedPhase: "sleeping", }, ] as any, + revisions: [ + revision("svc_public", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + }), + ], services: [ { id: "svc_public", @@ -240,8 +496,6 @@ describe("expected-state pure builders", () => { }, ] as any, deploymentPorts: [], - secrets: [], - volumes: [], }); expect(containers[0]).toMatchObject({ @@ -251,30 +505,37 @@ describe("expected-state pure builders", () => { }); }); - it("keeps deployed serverless sleeping while live serverless settings are disabled", () => { + it("keeps revision serverless behavior while draft settings are disabled", () => { const containers = buildExpectedContainersFromRows({ deployments: [ { id: "dep_sleeping", serviceId: "svc_public", + serviceRevisionId: "rev_svc_public", ipAddress: "10.0.0.10", runtimeDesiredState: "stopped", trafficState: "active", observedPhase: "sleeping", }, ] as any, + revisions: [ + revision("svc_public", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + }), + ], services: [ { id: "svc_public", name: "public-api", image: "nginx", serverlessEnabled: false, - deployedConfig: deployedServerlessConfig, }, ] as any, deploymentPorts: [], - secrets: [], - volumes: [], }); expect(containers[0]).toMatchObject({ @@ -290,6 +551,7 @@ describe("expected-state pure builders", () => { { id: "dep_draining", serviceId: "svc_public", + serviceRevisionId: "rev_svc_public", ipAddress: "10.0.0.10", runtimeDesiredState: "stopped", trafficState: "draining", @@ -297,6 +559,15 @@ describe("expected-state pure builders", () => { containerId: null, }, ] as any, + revisions: [ + revision("svc_public", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + }), + ], services: [ { id: "svc_public", @@ -306,8 +577,6 @@ describe("expected-state pure builders", () => { }, ] as any, deploymentPorts: [], - secrets: [], - volumes: [], }); expect(containers[0]).toMatchObject({ @@ -347,12 +616,14 @@ describe("expected-state pure builders", () => { const services: Parameters< typeof buildServerlessTraefikRouteSets >[0]["services"] = [ - { - id: "svc_serverless", - serverlessEnabled: true, - stateful: false, - }, - ] as Parameters[0]["services"]; + runtimeRevision("svc_serverless", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + }), + ]; const ports: Parameters[0]["ports"] = [ { id: "port_1", @@ -536,14 +807,14 @@ describe("expected-state pure builders", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", services: [ - { - id: "svc_1", - serverlessEnabled: true, - stateful: false, - serverlessSleepAfterSeconds: 300, - serverlessWakeTimeoutSeconds: 120, - }, - ] as any, + runtimeRevision("svc_1", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + }), + ], ports: [ { id: "port_1", @@ -619,14 +890,15 @@ describe("expected-state pure builders", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", services: [ - { - id: "svc_stateful", - serverlessEnabled: true, + runtimeRevision("svc_stateful", { stateful: true, - serverlessSleepAfterSeconds: 300, - serverlessWakeTimeoutSeconds: 120, - }, - ] as any, + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + }), + ], ports: [ { id: "port_1", @@ -674,14 +946,14 @@ describe("expected-state pure builders", () => { const routes = buildServerlessRoutesFromRows({ serverId: "proxy_1", services: [ - { - id: "svc_1", - serverlessEnabled: true, - stateful: false, - serverlessSleepAfterSeconds: 300, - serverlessWakeTimeoutSeconds: 120, - }, - ] as any, + runtimeRevision("svc_1", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + }), + ], ports: [ { id: "port_1", @@ -726,20 +998,28 @@ describe("expected-state pure builders", () => { ).toEqual(["dep_new"]); }); - it("keeps wake metadata from deployed serverless config while live settings are disabled", () => { - const routes = buildServerlessRoutesFromRows({ - serverId: "proxy_1", - services: [ + it("keeps wake metadata and ports pinned to the active revision", () => { + const service = runtimeRevision("svc_1", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + ports: [ { - id: "svc_1", - serverlessEnabled: false, - stateful: false, - serverlessSleepAfterSeconds: 60, - serverlessWakeTimeoutSeconds: 60, - deployedConfig: deployedServerlessConfig, + containerPort: 3000, + isPublic: true, + domain: "sleepy.example.com", + protocol: "http", + externalPort: null, + tlsPassthrough: false, }, - ] as any, - ports: [], + ], + }); + const routes = buildServerlessRoutesFromRows({ + serverId: "proxy_1", + services: [service], + ports: buildRuntimeRoutePorts([service]), deployments: [ { id: "dep_sleeping", @@ -770,18 +1050,26 @@ describe("expected-state pure builders", () => { ]); }); - it("keeps Traefik gateway route from deployed serverless ports while live ports are removed", () => { - const ports = buildRuntimeRoutePorts( - [ - { - id: "svc_public", - serverlessEnabled: false, - stateful: false, - deployedConfig: deployedServerlessConfig, + it("keeps Traefik gateway routes pinned to revision ports", () => { + const ports = buildRuntimeRoutePorts([ + runtimeRevision("svc_public", { + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, }, - ] as any, - [], - ); + ports: [ + { + containerPort: 3000, + isPublic: true, + domain: "sleepy.example.com", + protocol: "http", + externalPort: null, + tlsPassthrough: false, + }, + ], + }), + ]); const routes = buildTraefikRoutes({ serverId: "proxy_1", diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts index 65f9695..f894642 100644 --- a/web/tests/rollout-helpers.test.ts +++ b/web/tests/rollout-helpers.test.ts @@ -7,33 +7,31 @@ vi.mock("@/lib/acme-manager", () => ({ issueCertificate: vi.fn(), })); vi.mock("@/lib/wireguard", () => ({ - assignContainerIp: vi.fn(), + findAvailableContainerIp: vi.fn(), })); vi.mock("@/lib/work-queue", () => ({ enqueueWork: vi.fn(), })); -import { isActiveDeploymentForRollout } from "@/lib/inngest/functions/rollout-helpers"; +import { + findAvailableHostPorts, + isActiveDeploymentForRollout, +} from "@/lib/inngest/functions/rollout-helpers"; describe("rollout helpers", () => { it("treats active traffic deployments as the live rollout version", () => { - expect( - isActiveDeploymentForRollout( - { trafficState: "active" }, - { serverlessEnabled: true }, - ), - ).toBe(true); - expect( - isActiveDeploymentForRollout( - { trafficState: "candidate" }, - { serverlessEnabled: true }, - ), - ).toBe(false); - expect( - isActiveDeploymentForRollout( - { trafficState: "draining" }, - { serverlessEnabled: false }, - ), - ).toBe(false); + expect(isActiveDeploymentForRollout({ trafficState: "active" })).toBe(true); + expect(isActiveDeploymentForRollout({ trafficState: "candidate" })).toBe( + false, + ); + expect(isActiveDeploymentForRollout({ trafficState: "draining" })).toBe( + false, + ); + }); + + it("allocates host ports around ports already in use", () => { + expect(findAvailableHostPorts([30000, 30002], 3)).toEqual([ + 30001, 30003, 30004, + ]); }); }); diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index 9544414..332744e 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -2,11 +2,11 @@ import { describe, expect, it } from "vitest"; import { type DeployedConfig, diffConfigs, - getDeployedServerlessConfig, getCurrentServerlessConfig, - isDeployedServerlessService, MIN_SERVERLESS_SLEEP_AFTER_SECONDS, + revisionSpecToDeployedConfig, } from "@/lib/service-config"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; function deployedConfig( overrides: Partial = {}, @@ -27,24 +27,7 @@ function deployedConfig( } describe("service config", () => { - it("uses deployed serverless settings as the runtime mode", () => { - const service = { - serverlessEnabled: false, - serverlessSleepAfterSeconds: 60, - serverlessWakeTimeoutSeconds: 60, - stateful: true, - deployedConfig: JSON.stringify(deployedConfig()), - }; - - expect(getDeployedServerlessConfig(service)).toMatchObject({ - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - }); - expect(isDeployedServerlessService(service)).toBe(true); - }); - - it("enforces the minimum serverless sleep timeout for legacy config", () => { + it("enforces the minimum serverless sleep timeout for draft config", () => { expect( getCurrentServerlessConfig({ serverlessEnabled: true, @@ -54,33 +37,41 @@ describe("service config", () => { ).toMatchObject({ sleepAfterSeconds: MIN_SERVERLESS_SLEEP_AFTER_SECONDS, }); - expect( - getDeployedServerlessConfig({ - deployedConfig: JSON.stringify( - deployedConfig({ - serverless: { - enabled: true, - sleepAfterSeconds: 60, - wakeTimeoutSeconds: 120, - }, - }), - ), - }), - ).toMatchObject({ - sleepAfterSeconds: MIN_SERVERLESS_SLEEP_AFTER_SECONDS, - }); }); - it("allows deployed stateful services to be serverless", () => { - const service = { - serverlessEnabled: true, + it("converts an immutable revision into the pending-change baseline", () => { + const specification: ServiceRevisionSpec = { + schemaVersion: 1, + serviceId: "service-1", + image: "nginx", + hostname: "api", stateful: true, - serverlessSleepAfterSeconds: 300, - serverlessWakeTimeoutSeconds: 120, - deployedConfig: JSON.stringify(deployedConfig({ stateful: true })), + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "server-1", count: 1 }], + ports: [], + secrets: [{ key: "TOKEN", encryptedValue: "ciphertext" }], + volumes: [], }; - expect(isDeployedServerlessService(service)).toBe(true); + expect( + revisionSpecToDeployedConfig( + specification, + { "server-1": "Sydney" }, + { TOKEN: "fingerprint" }, + ), + ).toMatchObject({ + stateful: true, + serverless: { enabled: true }, + replicas: [{ serverId: "server-1", serverName: "Sydney", count: 1 }], + secrets: [{ key: "TOKEN", fingerprint: "fingerprint" }], + }); }); it("reports serverless changes as pending config", () => { diff --git a/web/tests/service-revision-cutover.test.ts b/web/tests/service-revision-cutover.test.ts new file mode 100644 index 0000000..0927c4b --- /dev/null +++ b/web/tests/service-revision-cutover.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { buildCutoverServiceRevisionSpec } from "@/lib/service-revision-cutover"; +import type { ServiceRevisionDraft } from "@/lib/service-revision-spec"; + +function liveDraft(): ServiceRevisionDraft { + return { + service: { + id: "service-1", + name: "API", + image: "current:image", + hostname: "current-host", + stateful: false, + serverlessEnabled: false, + serverlessSleepAfterSeconds: 600, + serverlessWakeTimeoutSeconds: 600, + healthCheckCmd: "current-health", + healthCheckInterval: 10, + healthCheckTimeout: 5, + healthCheckRetries: 3, + healthCheckStartPeriod: 30, + startCommand: "current-command", + resourceCpuLimit: 2, + resourceMemoryLimitMb: 1024, + }, + placements: [{ serverId: "current-server", count: 2 }], + ports: [ + { + port: 8080, + isPublic: true, + domain: "current.example.com", + protocol: "tcp", + externalPort: 443, + tlsPassthrough: true, + }, + ], + secrets: [ + { + key: "TOKEN", + encryptedValue: "ciphertext", + updatedAt: "2026-07-01T00:00:00.000Z", + }, + ], + volumes: [{ name: "current", containerPath: "/current" }], + }; +} + +describe("service revision cutover", () => { + it("uses the legacy deployed snapshot while retaining runtime-only values", () => { + const specification = buildCutoverServiceRevisionSpec({ + liveDraft: liveDraft(), + deployedConfig: { + source: { image: "deployed:image" }, + hostname: "deployed-host", + stateful: true, + replicas: [{ serverId: "deployed-server", count: 1 }], + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: 1, memoryMb: 512 }, + ports: [ + { + port: 8080, + isPublic: false, + domain: null, + protocol: "tcp", + tlsPassthrough: false, + }, + ], + serverless: { + enabled: true, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 120, + }, + secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], + volumes: [{ name: "deployed", containerPath: "/data" }], + }, + }); + + expect(specification).toMatchObject({ + image: "deployed:image", + hostname: "deployed-host", + stateful: true, + placements: [{ serverId: "deployed-server", count: 1 }], + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: 1, memoryMb: 512 }, + serverless: { enabled: true }, + ports: [{ containerPort: 8080, externalPort: 443 }], + secrets: [{ key: "TOKEN", encryptedValue: "ciphertext" }], + volumes: [{ name: "deployed", containerPath: "/data" }], + }); + }); + + it("keeps deployed resource limits unset instead of using live draft limits", () => { + const specification = buildCutoverServiceRevisionSpec({ + liveDraft: liveDraft(), + deployedConfig: { + secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], + }, + }); + + expect(specification.resourceLimits).toEqual({ + cpuCores: null, + memoryMb: null, + }); + }); + + it("rejects undeployed secret changes", () => { + const draft = liveDraft(); + draft.secrets[0].updatedAt = "2026-07-02T00:00:00.000Z"; + + expect(() => + buildCutoverServiceRevisionSpec({ + liveDraft: draft, + deployedConfig: { + secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], + }, + }), + ).toThrow("Secret TOKEN differs from the deployed snapshot"); + }); + + it("uses live configuration for a service that has never been deployed", () => { + const draft = liveDraft(); + const specification = buildCutoverServiceRevisionSpec({ + liveDraft: draft, + deployedConfig: null, + }); + + expect(specification).toMatchObject({ + image: draft.service.image, + hostname: draft.service.hostname, + placements: draft.placements, + healthCheck: { cmd: "current-health" }, + startCommand: "current-command", + ports: [{ containerPort: 8080, externalPort: 443 }], + volumes: draft.volumes, + }); + }); +}); diff --git a/web/tests/service-revision-spec.test.ts b/web/tests/service-revision-spec.test.ts new file mode 100644 index 0000000..fed67ad --- /dev/null +++ b/web/tests/service-revision-spec.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { + buildServiceRevisionSpec, + hashServiceRevisionSpec, + type ServiceRevisionDraft, +} from "@/lib/service-revision-spec"; + +function draft( + overrides: Partial = {}, +): ServiceRevisionDraft { + return { + service: { + id: "service-1", + name: "API Service", + image: "nginx:latest", + hostname: "api.internal", + stateful: false, + serverlessEnabled: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 300, + healthCheckCmd: "curl -f http://localhost/health", + healthCheckInterval: 10, + healthCheckTimeout: 5, + healthCheckRetries: 3, + healthCheckStartPeriod: 30, + startCommand: null, + resourceCpuLimit: null, + resourceMemoryLimitMb: null, + }, + placements: [ + { serverId: "server-b", count: 1 }, + { serverId: "server-a", count: 2 }, + ], + ports: [ + { + port: 443, + isPublic: true, + domain: "api.example.com", + protocol: "tcp", + externalPort: 443, + tlsPassthrough: true, + }, + { + port: 80, + isPublic: false, + domain: null, + protocol: "http", + externalPort: null, + tlsPassthrough: false, + }, + ], + secrets: [ + { key: "TOKEN", encryptedValue: "ciphertext-2" }, + { key: "API_KEY", encryptedValue: "ciphertext-1" }, + ], + volumes: [ + { name: "logs", containerPath: "/logs" }, + { name: "data", containerPath: "/data" }, + ], + ...overrides, + }; +} + +describe("service revision specification", () => { + it("produces the same hash regardless of draft row ordering", () => { + const first = buildServiceRevisionSpec(draft()); + const reorderedDraft = draft(); + reorderedDraft.placements.reverse(); + reorderedDraft.ports.reverse(); + reorderedDraft.secrets.reverse(); + reorderedDraft.volumes.reverse(); + const second = buildServiceRevisionSpec(reorderedDraft); + + expect(second).toEqual(first); + expect(hashServiceRevisionSpec(second)).toBe( + hashServiceRevisionSpec(first), + ); + }); + + it("normalizes defaults once", () => { + const input = draft(); + input.service.serverlessSleepAfterSeconds = 30; + input.service.healthCheckInterval = null; + input.ports[0].protocol = null; + input.ports[0].tlsPassthrough = null; + + const spec = buildServiceRevisionSpec(input); + + expect(spec.serverless.sleepAfterSeconds).toBe(120); + expect(spec.healthCheck?.interval).toBe(10); + expect(spec.ports[1]).toMatchObject({ + protocol: "http", + tlsPassthrough: false, + }); + }); + + it("changes identity when encrypted secret content changes", () => { + const first = buildServiceRevisionSpec(draft()); + const changedDraft = draft(); + changedDraft.secrets[0].encryptedValue = "new-ciphertext"; + const changed = buildServiceRevisionSpec(changedDraft); + + expect(hashServiceRevisionSpec(changed)).not.toBe( + hashServiceRevisionSpec(first), + ); + }); +}); From eadf61ac18f84c71b37dcef3231e7a19cbd1f914 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:21:12 +1000 Subject: [PATCH 06/10] Simplify immutable service revisions --- agent/internal/agent/agent.go | 5 - agent/internal/agent/drift.go | 131 +------ agent/internal/agent/reconcile_failures.go | 160 -------- .../internal/agent/reconcile_failures_test.go | 88 ----- agent/internal/agent/reporting.go | 19 +- .../internal/agent/revision_reconcile_test.go | 226 ----------- agent/internal/agent/run.go | 47 +-- agent/internal/agent/run_test.go | 36 -- agent/internal/agent/serverless.go | 17 +- agent/internal/container/runtime.go | 4 - agent/internal/container/runtime_test.go | 30 -- agent/internal/container/types.go | 4 - agent/internal/http/client.go | 95 +---- agent/internal/http/client_test.go | 54 --- agent/internal/reconcile/reconcile.go | 2 - deployment/README.md | 53 +-- deployment/compose.postgres.yml | 2 - deployment/compose.production.yml | 2 - web/actions/projects.ts | 158 +++----- .../[serviceId]/rollouts/[rolloutId]/page.tsx | 18 +- .../dashboard/servers/[id]/page.tsx | 1 - web/app/api/projects/[id]/services/route.ts | 75 ++-- web/app/api/services/[id]/rollouts/route.ts | 22 +- .../api/v1/agent/builds/[id]/status/route.ts | 5 +- web/app/api/v1/agent/expected-state/route.ts | 50 +-- .../server/server-health-details.tsx | 42 +-- web/components/server/server-services.tsx | 12 +- .../service/details/rollout-details.tsx | 8 - .../service/details/rollout-history.tsx | 6 - web/db/queries.ts | 19 +- web/db/schema.ts | 35 +- web/db/types.ts | 11 +- web/lib/agent-capabilities.ts | 13 - web/lib/agent-status.ts | 2 - web/lib/agent/expected-state.ts | 151 +------- web/lib/cli-service.ts | 2 +- web/lib/deploy-service.ts | 11 +- web/lib/deployment-status.ts | 41 +- web/lib/inngest/functions/build-workflow.ts | 37 +- .../inngest/functions/migration-workflow.ts | 120 +++--- web/lib/inngest/functions/rollout-helpers.ts | 157 ++------ web/lib/inngest/functions/rollout-utils.ts | 141 +++---- web/lib/inngest/functions/rollout-workflow.ts | 92 +---- .../functions/service-deletion-workflow.ts | 55 +-- web/lib/scheduler.ts | 2 +- web/lib/service-config.ts | 107 +++--- web/lib/service-revision-cutover.ts | 128 ++----- web/lib/service-revision-spec.ts | 40 +- web/lib/service-revisions.ts | 201 ++++------ web/lib/wireguard.ts | 34 +- web/scripts/cutover-service-revisions.ts | 352 ++++++------------ web/tests/agent-capabilities.test.ts | 28 -- web/tests/deployment-status.test.ts | 20 +- web/tests/expected-state-route.test.ts | 93 ----- web/tests/expected-state.test.ts | 118 +----- web/tests/rollout-helpers.test.ts | 37 -- web/tests/service-config.test.ts | 58 ++- web/tests/service-revision-cutover.test.ts | 27 +- web/tests/service-revision-spec.test.ts | 30 +- 59 files changed, 727 insertions(+), 2807 deletions(-) delete mode 100644 agent/internal/agent/reconcile_failures.go delete mode 100644 agent/internal/agent/reconcile_failures_test.go delete mode 100644 agent/internal/agent/revision_reconcile_test.go delete mode 100644 agent/internal/agent/run_test.go delete mode 100644 agent/internal/http/client_test.go delete mode 100644 web/lib/agent-capabilities.ts delete mode 100644 web/tests/agent-capabilities.test.ts delete mode 100644 web/tests/expected-state-route.test.ts delete mode 100644 web/tests/rollout-helpers.test.ts diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index bfece7f..fa85f72 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -64,10 +64,6 @@ type Agent struct { pendingDeploymentErrors []agenthttp.DeploymentError deployLockMutex sync.Mutex deploymentDeployLocks map[string]*sync.Mutex - reconcileFailureMutex sync.Mutex - reconcileFailures map[string]reconcileActionFailure - legacyCutoverHealthWait string - legacyCutoverHealthWaitSince time.Time serverlessMutex sync.Mutex pendingServerlessTransitions []agenthttp.ServerlessTransition pendingServerlessSleep map[string]serverlessTransitionGuard @@ -124,7 +120,6 @@ func NewAgent( IsProxy: isProxy, DisableDNS: disableDNS, deploymentDeployLocks: map[string]*sync.Mutex{}, - reconcileFailures: map[string]reconcileActionFailure{}, pendingServerlessSleep: map[string]serverlessTransitionGuard{}, pendingServerlessWake: map[string]serverlessTransitionGuard{}, } diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 63025bf..18216d8 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -26,15 +26,12 @@ const ( actionStopExpectedContainer reconcileActionKind = "stop_expected_container" actionStartContainer reconcileActionKind = "start_container" actionRedeployContainer reconcileActionKind = "redeploy_container" - actionWaitLegacyCutover reconcileActionKind = "wait_legacy_cutover" actionUpdateDNS reconcileActionKind = "update_dns" actionUpdateTraefik reconcileActionKind = "update_traefik" actionUpdateCertificates reconcileActionKind = "update_certificates" actionWriteChallengeRoute reconcileActionKind = "write_challenge_route" actionUpdateWireGuard reconcileActionKind = "update_wireguard" actionStartWireGuard reconcileActionKind = "start_wireguard" - legacyCutoverStabilizationDelay = 30 * time.Second - legacyCutoverMaxWait = 5 * time.Minute ) type reconcileAction struct { @@ -147,124 +144,25 @@ func (a *Agent) handleProcessing() { a.updateDnsInSync(a.expectedState, actual) actions := a.planReconcile(a.expectedState, actual) - actions, waitingForCutoverHealth := a.gateLegacyCutoverRecreations( - actions, - actual, - ) if len(actions) == 0 { - if waitingForCutoverHealth { - return - } - a.clearReconcileFailures() log.Printf("[processing] state converged, transitioning to IDLE") a.transitionToIdle() return } - action, eligible, nextRetryAt := a.nextEligibleReconcileAction(actions, time.Now()) - if !eligible { - log.Printf("[processing] all %d pending actions are backed off; next retry at %s", len(actions), nextRetryAt.Format(time.RFC3339)) - return - } + action := actions[0] if err := a.applyReconcileAction(action); err != nil { - failure := a.recordReconcileFailure(action, err, time.Now()) - log.Printf("[processing] reconciliation action failed (attempt %d); continuing with remaining actions, retry at %s: %v", failure.Attempts, failure.NextRetryAt.Format(time.RFC3339), err) + log.Printf("[processing] reconciliation failed: %v, transitioning to IDLE", err) a.RecordDeploymentError(action.DeploymentID, err) a.RequestStatusReport("reconcile failed") + a.transitionToIdle() return } - a.clearReconcileFailure(action) - if a.legacyCutoverHealthWait == action.DeploymentID && - (action.Kind == actionDeployMissingContainer || - action.Kind == actionStartContainer || - action.Kind == actionRedeployContainer) { - a.legacyCutoverHealthWaitSince = time.Now() - } - if isLegacyCutoverRecreation(action) { - a.legacyCutoverHealthWait = action.DeploymentID - a.legacyCutoverHealthWaitSince = time.Now() - log.Printf("[processing] waiting for deployment %s to stabilize before recreating another legacy container", action.DeploymentID) - } a.RequestStatusReport("reconcile completed") } -func isLegacyCutoverRecreation(action reconcileAction) bool { - return action.Kind == actionRedeployContainer && - action.Actual != nil && - action.Actual.SpecHash == "" && - action.Expected != nil && - action.Expected.ContainerSpecHash != "" -} - -func (a *Agent) gateLegacyCutoverRecreations( - actions []reconcileAction, - actual *ActualState, -) ([]reconcileAction, bool) { - if a.legacyCutoverHealthWait == "" { - return actions, false - } - var waitingExpected *agenthttp.ExpectedContainer - for _, expectedContainer := range a.expectedState.Containers { - if expectedContainer.DeploymentID == a.legacyCutoverHealthWait { - copy := expectedContainer - waitingExpected = © - break - } - } - if waitingExpected == nil { - a.legacyCutoverHealthWait = "" - a.legacyCutoverHealthWaitSince = time.Time{} - return actions, false - } - - var waitingContainer *container.Container - for i := range actual.Containers { - if actual.Containers[i].DeploymentID == a.legacyCutoverHealthWait { - waitingContainer = &actual.Containers[i] - break - } - } - ready := false - if waitingContainer != nil && waitingContainer.State == "running" { - if waitingExpected.HealthCheck != nil { - ready = container.GetHealthStatus(waitingContainer.ID) == "healthy" - } else { - ready = time.Since(a.legacyCutoverHealthWaitSince) >= legacyCutoverStabilizationDelay - } - } - if ready { - log.Printf("[processing] deployment %s is stable; legacy recreation may continue", a.legacyCutoverHealthWait) - a.legacyCutoverHealthWait = "" - a.legacyCutoverHealthWaitSince = time.Time{} - return actions, false - } - - if time.Since(a.legacyCutoverHealthWaitSince) >= legacyCutoverMaxWait { - deploymentID := a.legacyCutoverHealthWait - log.Printf("[processing] deployment %s did not stabilize; releasing legacy recreation gate", deploymentID) - a.legacyCutoverHealthWait = "" - a.legacyCutoverHealthWaitSince = time.Time{} - return []reconcileAction{{ - Kind: actionWaitLegacyCutover, - DeploymentID: deploymentID, - Description: fmt.Sprintf("WAIT for legacy deployment %s to stabilize", deploymentID), - Expected: waitingExpected, - Actual: waitingContainer, - }}, false - } - - filtered := make([]reconcileAction, 0, len(actions)) - for _, action := range actions { - if isLegacyCutoverRecreation(action) { - continue - } - filtered = append(filtered, action) - } - return filtered, true -} - func (a *Agent) updateDnsInSync(expected *agenthttp.ExpectedState, actual *ActualState) { if a.DisableDNS { a.dnsInSync = true @@ -389,19 +287,7 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS continue } - if exp.ContainerSpecHash == "" || act.SpecHash != exp.ContainerSpecHash { - reason := "container specification changed" - if act.SpecHash == "" { - reason = "legacy container is missing revision labels" - } - actions = append(actions, reconcileAction{ - Kind: actionRedeployContainer, - Description: fmt.Sprintf("REDEPLOY %s (%s)", exp.Name, reason), - DeploymentID: id, - Expected: &expectedContainer, - Actual: &actualContainer, - }) - } else if normalizeImage(exp.Image) != normalizeImage(act.Image) { + if normalizeImage(exp.Image) != normalizeImage(act.Image) { actions = append(actions, reconcileAction{ Kind: actionRedeployContainer, Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), @@ -597,6 +483,9 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { } if err := container.Start(action.Actual.ID); err != nil { log.Printf("[reconcile] start failed, will redeploy: %v", err) + if err := container.Stop(action.Actual.ID); err != nil { + log.Printf("[reconcile] warning: failed to stop old container: %v", err) + } if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } @@ -607,6 +496,9 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { if action.Actual == nil || action.Expected == nil { return fmt.Errorf("missing container state for %s", action.Kind) } + if err := container.Stop(action.Actual.ID); err != nil { + log.Printf("[reconcile] warning: failed to stop old container: %v", err) + } if err := a.DeployExpectedContainer(*action.Expected); err != nil { return fmt.Errorf("failed to redeploy container: %w", err) } @@ -664,9 +556,6 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { } return nil - case actionWaitLegacyCutover: - return fmt.Errorf("deployment %s did not stabilize within %s", action.DeploymentID, legacyCutoverMaxWait) - default: return fmt.Errorf("unknown reconcile action: %s", action.Kind) } diff --git a/agent/internal/agent/reconcile_failures.go b/agent/internal/agent/reconcile_failures.go deleted file mode 100644 index 6ab86a1..0000000 --- a/agent/internal/agent/reconcile_failures.go +++ /dev/null @@ -1,160 +0,0 @@ -package agent - -import ( - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" - "fmt" - "sort" - "time" - - agenthttp "techulus/cloud-agent/internal/http" -) - -const ( - reconcileFailureBaseBackoff = 5 * time.Second - reconcileFailureMaxBackoff = 5 * time.Minute -) - -type reconcileActionFailure struct { - Key string - Kind reconcileActionKind - DeploymentID string - Description string - LastError string - Attempts int - NextRetryAt time.Time -} - -func (a *Agent) reconcileActionKey(action reconcileAction) string { - target := action.DeploymentID - if target == "" && action.Actual != nil { - target = action.Actual.ID - } - - var desired any - switch action.Kind { - case actionDeployMissingContainer, actionStopExpectedContainer, actionStartContainer, actionRedeployContainer: - desired = action.Expected - case actionUpdateDNS: - desired = a.expectedState.Dns - case actionUpdateTraefik: - desired = a.expectedState.Traefik - case actionUpdateCertificates: - desired = a.expectedState.Traefik.Certificates - case actionWriteChallengeRoute: - desired = a.expectedState.Traefik.ChallengeRoute - case actionUpdateWireGuard, actionStartWireGuard: - desired = a.expectedState.Wireguard - default: - desired = action.Description - } - - payload, _ := json.Marshal(desired) - digest := sha256.Sum256(payload) - return fmt.Sprintf("%s:%s:%s", action.Kind, target, hex.EncodeToString(digest[:8])) -} - -func reconcileFailureBackoff(attempts int, key string) time.Duration { - backoff := reconcileFailureBaseBackoff - for i := 1; i < attempts && backoff < reconcileFailureMaxBackoff; i++ { - backoff *= 2 - } - if backoff >= reconcileFailureMaxBackoff { - return reconcileFailureMaxBackoff - } - digest := sha256.Sum256([]byte(key)) - jitterWindow := backoff / 5 - jitter := time.Duration( - binary.BigEndian.Uint64(digest[:8]) % uint64(jitterWindow+1), - ) - if backoff+jitter > reconcileFailureMaxBackoff { - return reconcileFailureMaxBackoff - } - return backoff + jitter -} - -func (a *Agent) recordReconcileFailure(action reconcileAction, err error, now time.Time) reconcileActionFailure { - key := a.reconcileActionKey(action) - a.reconcileFailureMutex.Lock() - defer a.reconcileFailureMutex.Unlock() - - failure := a.reconcileFailures[key] - failure.Key = key - failure.Kind = action.Kind - failure.DeploymentID = action.DeploymentID - failure.Description = action.Description - failure.LastError = err.Error() - failure.Attempts++ - failure.NextRetryAt = now.Add(reconcileFailureBackoff(failure.Attempts, key)) - a.reconcileFailures[key] = failure - return failure -} - -func (a *Agent) clearReconcileFailure(action reconcileAction) { - key := a.reconcileActionKey(action) - a.reconcileFailureMutex.Lock() - defer a.reconcileFailureMutex.Unlock() - delete(a.reconcileFailures, key) -} - -func (a *Agent) clearReconcileFailures() { - a.reconcileFailureMutex.Lock() - defer a.reconcileFailureMutex.Unlock() - clear(a.reconcileFailures) -} - -func (a *Agent) nextEligibleReconcileAction(actions []reconcileAction, now time.Time) (reconcileAction, bool, time.Time) { - activeKeys := make(map[string]struct{}, len(actions)) - keys := make([]string, len(actions)) - for i, action := range actions { - keys[i] = a.reconcileActionKey(action) - activeKeys[keys[i]] = struct{}{} - } - - a.reconcileFailureMutex.Lock() - defer a.reconcileFailureMutex.Unlock() - for key := range a.reconcileFailures { - if _, active := activeKeys[key]; !active { - delete(a.reconcileFailures, key) - } - } - - var earliestRetry time.Time - for i, action := range actions { - failure, failed := a.reconcileFailures[keys[i]] - if !failed || !now.Before(failure.NextRetryAt) { - return action, true, earliestRetry - } - if earliestRetry.IsZero() || failure.NextRetryAt.Before(earliestRetry) { - earliestRetry = failure.NextRetryAt - } - } - - return reconcileAction{}, false, earliestRetry -} - -func (a *Agent) SnapshotReconciliationFailures() []agenthttp.ReconciliationFailure { - a.reconcileFailureMutex.Lock() - defer a.reconcileFailureMutex.Unlock() - - failures := make([]agenthttp.ReconciliationFailure, 0, len(a.reconcileFailures)) - for _, failure := range a.reconcileFailures { - failures = append(failures, agenthttp.ReconciliationFailure{ - Action: string(failure.Kind), - DeploymentID: failure.DeploymentID, - Description: failure.Description, - LastError: failure.LastError, - Attempts: failure.Attempts, - NextRetryAt: failure.NextRetryAt, - }) - } - sort.Slice(failures, func(i, j int) bool { - if failures[i].Action != failures[j].Action { - return failures[i].Action < failures[j].Action - } - return failures[i].DeploymentID < failures[j].DeploymentID - }) - return failures -} diff --git a/agent/internal/agent/reconcile_failures_test.go b/agent/internal/agent/reconcile_failures_test.go deleted file mode 100644 index c6fe9db..0000000 --- a/agent/internal/agent/reconcile_failures_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package agent - -import ( - "errors" - "testing" - "time" - - agenthttp "techulus/cloud-agent/internal/http" -) - -func TestFailedReconcileActionDoesNotBlockFollowingActions(t *testing.T) { - now := time.Now() - agent := &Agent{ - expectedState: &agenthttp.ExpectedState{}, - reconcileFailures: map[string]reconcileActionFailure{}, - } - badContainer := agenthttp.ExpectedContainer{ - DeploymentID: "bad-deployment", - ContainerSpecHash: "bad-spec", - } - badAction := reconcileAction{ - Kind: actionDeployMissingContainer, - Description: "deploy image that cannot be pulled", - DeploymentID: badContainer.DeploymentID, - Expected: &badContainer, - } - clusterAction := reconcileAction{ - Kind: actionUpdateDNS, - Description: "update DNS", - } - - agent.recordReconcileFailure(badAction, errors.New("image pull failed"), now) - selected, eligible, _ := agent.nextEligibleReconcileAction( - []reconcileAction{badAction, clusterAction}, - now, - ) - - if !eligible { - t.Fatal("expected the independent action to remain eligible") - } - if selected.Kind != actionUpdateDNS { - t.Fatalf("expected DNS action, got %s", selected.Kind) - } - - selected, eligible, _ = agent.nextEligibleReconcileAction( - []reconcileAction{badAction, clusterAction}, - now.Add(2*reconcileFailureBaseBackoff), - ) - if !eligible || selected.Kind != actionDeployMissingContainer { - t.Fatalf("expected failed action to become eligible after backoff, got %s", selected.Kind) - } -} - -func TestChangedContainerSpecClearsActionBackoff(t *testing.T) { - now := time.Now() - agent := &Agent{ - expectedState: &agenthttp.ExpectedState{}, - reconcileFailures: map[string]reconcileActionFailure{}, - } - original := agenthttp.ExpectedContainer{ - DeploymentID: "deployment", - ContainerSpecHash: "original-spec", - } - originalAction := reconcileAction{ - Kind: actionRedeployContainer, - Description: "redeploy original", - DeploymentID: original.DeploymentID, - Expected: &original, - } - agent.recordReconcileFailure(originalAction, errors.New("image pull failed"), now) - - updated := original - updated.ContainerSpecHash = "updated-spec" - updatedAction := originalAction - updatedAction.Description = "redeploy updated" - updatedAction.Expected = &updated - - selected, eligible, _ := agent.nextEligibleReconcileAction( - []reconcileAction{updatedAction}, - now, - ) - if !eligible || selected.Expected.ContainerSpecHash != "updated-spec" { - t.Fatal("expected a changed container specification to bypass stale backoff") - } - if failures := agent.SnapshotReconciliationFailures(); len(failures) != 0 { - t.Fatalf("expected stale failure to be pruned, got %d", len(failures)) - } -} diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index b9236fe..70f75cc 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -18,10 +18,7 @@ import ( var Version = "dev" -const ( - serverlessGatewayCapability = "serverless_gateway" - serviceRevisionCapability = "service_revision_v1" -) +const serverlessGatewayCapability = "serverless_gateway" var ( agentStartTime = time.Now() @@ -36,10 +33,9 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport Containers: []agenthttp.ContainerStatus{}, DnsInSync: a.dnsInSync, AgentHealth: &agenthttp.AgentHealth{ - Version: Version, - UptimeSecs: int64(time.Since(agentStartTime).Seconds()), - Capabilities: a.agentCapabilities(), - ReconciliationFailures: a.SnapshotReconciliationFailures(), + Version: Version, + UptimeSecs: int64(time.Since(agentStartTime).Seconds()), + Capabilities: a.agentCapabilities(), }, } @@ -120,11 +116,10 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport } func (a *Agent) agentCapabilities() []string { - capabilities := []string{serviceRevisionCapability} - if a.IsProxy && a.serverlessGatewayRunning.Load() { - capabilities = append(capabilities, serverlessGatewayCapability) + if !a.IsProxy || !a.serverlessGatewayRunning.Load() { + return nil } - return capabilities + return []string{serverlessGatewayCapability} } func (a *Agent) RecordDeploymentError(deploymentID string, err error) { diff --git a/agent/internal/agent/revision_reconcile_test.go b/agent/internal/agent/revision_reconcile_test.go deleted file mode 100644 index a6cb0c8..0000000 --- a/agent/internal/agent/revision_reconcile_test.go +++ /dev/null @@ -1,226 +0,0 @@ -package agent - -import ( - "testing" - "time" - - "techulus/cloud-agent/internal/container" - agenthttp "techulus/cloud-agent/internal/http" -) - -func TestContainerRevisionHashControlsRecreation(t *testing.T) { - tests := []struct { - name string - actualHash string - wantAction bool - }{ - {name: "matching hash", actualHash: "spec-v1", wantAction: false}, - {name: "changed hash", actualHash: "spec-v0", wantAction: true}, - {name: "legacy missing hash", actualHash: "", wantAction: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - agent := &Agent{DisableDNS: true} - expected := &agenthttp.ExpectedState{ - Containers: []agenthttp.ExpectedContainer{ - { - DeploymentID: "deployment", - ServiceID: "service", - Name: "service-deployment", - Image: "docker.io/library/nginx", - DesiredState: "running", - ContainerSpecHash: "spec-v1", - }, - }, - } - actual := &ActualState{ - Containers: []container.Container{ - { - ID: "container", - DeploymentID: "deployment", - Image: "docker.io/library/nginx", - State: "running", - SpecHash: tt.actualHash, - }, - }, - } - - found := false - for _, action := range agent.planReconcile(expected, actual) { - if action.DeploymentID == "deployment" && action.Kind == actionRedeployContainer { - found = true - } - } - if found != tt.wantAction { - t.Fatalf("redeploy action = %v, want %v", found, tt.wantAction) - } - }) - } -} - -func TestRunningContainerOnlySkipsDeployWhenRevisionSpecMatches(t *testing.T) { - expected := agenthttp.ExpectedContainer{ - RevisionID: "revision-v2", - ContainerSpecHash: "spec-v2", - } - - tests := []struct { - name string - actual container.Container - want bool - }{ - { - name: "matching revision and spec", - actual: container.Container{ - RevisionID: "revision-v2", - SpecHash: "spec-v2", - }, - want: true, - }, - { - name: "legacy container without revision labels", - actual: container.Container{ - Image: "docker.io/library/nginx", - }, - }, - { - name: "changed spec", - actual: container.Container{ - RevisionID: "revision-v2", - SpecHash: "spec-v1", - }, - }, - { - name: "changed revision", - actual: container.Container{ - RevisionID: "revision-v1", - SpecHash: "spec-v2", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := runningContainerMatchesExpectedRevision(tt.actual, expected); got != tt.want { - t.Fatalf("revision/spec match = %v, want %v", got, tt.want) - } - }) - } -} - -func TestLegacyCutoverHealthWaitOnlyDefersOtherLegacyRecreations(t *testing.T) { - healthCheck := &agenthttp.HealthCheck{Cmd: "check"} - agent := &Agent{ - expectedState: &agenthttp.ExpectedState{ - Containers: []agenthttp.ExpectedContainer{ - {DeploymentID: "first", HealthCheck: healthCheck}, - }, - }, - legacyCutoverHealthWait: "first", - legacyCutoverHealthWaitSince: time.Now(), - } - legacyActual := container.Container{DeploymentID: "second"} - legacyExpected := agenthttp.ExpectedContainer{ - DeploymentID: "second", - ContainerSpecHash: "spec-v1", - } - actions := []reconcileAction{ - { - Kind: actionRedeployContainer, - DeploymentID: "second", - Actual: &legacyActual, - Expected: &legacyExpected, - }, - {Kind: actionUpdateDNS}, - } - - filtered, waiting := agent.gateLegacyCutoverRecreations(actions, &ActualState{}) - if !waiting { - t.Fatal("expected cutover health gate to remain active") - } - if len(filtered) != 1 || filtered[0].Kind != actionUpdateDNS { - t.Fatalf("expected cluster reconciliation to continue, got %+v", filtered) - } -} - -func TestLegacyCutoverStabilizationTimeoutBecomesVisibleFailure(t *testing.T) { - agent := &Agent{ - expectedState: &agenthttp.ExpectedState{ - Containers: []agenthttp.ExpectedContainer{{DeploymentID: "first"}}, - }, - legacyCutoverHealthWait: "first", - legacyCutoverHealthWaitSince: time.Now().Add(-legacyCutoverMaxWait), - } - - filtered, waiting := agent.gateLegacyCutoverRecreations( - []reconcileAction{{ - Kind: actionRedeployContainer, - DeploymentID: "second", - Actual: &container.Container{DeploymentID: "second"}, - Expected: &agenthttp.ExpectedContainer{ - DeploymentID: "second", - ContainerSpecHash: "spec-v1", - }, - }}, - &ActualState{}, - ) - - if waiting || len(filtered) != 1 || filtered[0].Kind != actionWaitLegacyCutover { - t.Fatalf("expected a visible stabilization failure action, got %+v", filtered) - } - if agent.legacyCutoverHealthWait != "" { - t.Fatal("expected timed-out legacy recreation gate to be released") - } - - nextActions, waiting := agent.gateLegacyCutoverRecreations( - []reconcileAction{{ - Kind: actionRedeployContainer, - DeploymentID: "second", - Actual: &container.Container{DeploymentID: "second"}, - Expected: &agenthttp.ExpectedContainer{ - DeploymentID: "second", - ContainerSpecHash: "spec-v1", - }, - }}, - &ActualState{}, - ) - if waiting || len(nextActions) != 1 || nextActions[0].Kind != actionRedeployContainer { - t.Fatalf("expected remaining legacy recreation to proceed, got %+v", nextActions) - } -} - -func TestLegacyCutoverWithoutHealthCheckWaitsForStabilization(t *testing.T) { - agent := &Agent{ - expectedState: &agenthttp.ExpectedState{ - Containers: []agenthttp.ExpectedContainer{{DeploymentID: "first"}}, - }, - legacyCutoverHealthWait: "first", - legacyCutoverHealthWaitSince: time.Now(), - } - actual := &ActualState{Containers: []container.Container{{ - ID: "container", - DeploymentID: "first", - State: "running", - }}} - actions := []reconcileAction{{ - Kind: actionRedeployContainer, - DeploymentID: "second", - Actual: &container.Container{DeploymentID: "second"}, - Expected: &agenthttp.ExpectedContainer{ - DeploymentID: "second", - ContainerSpecHash: "spec-v1", - }, - }} - - filtered, waiting := agent.gateLegacyCutoverRecreations(actions, actual) - if !waiting || len(filtered) != 0 { - t.Fatal("expected another legacy recreation to remain gated") - } - - agent.legacyCutoverHealthWaitSince = time.Now().Add(-legacyCutoverStabilizationDelay) - filtered, waiting = agent.gateLegacyCutoverRecreations(actions, actual) - if waiting || len(filtered) != len(actions) { - t.Fatal("expected recreation to continue after stabilization") - } -} diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index e826db9..fbf35ca 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -12,8 +12,6 @@ import ( "techulus/cloud-agent/internal/serverless" ) -const startupStatusReportMaxAttempts = 12 - const ( traefikMetricsURL = "http://127.0.0.1:9100/metrics" traefikMetricsInterval = 15 * time.Second @@ -65,14 +63,6 @@ func (a *Agent) Run(ctx context.Context) { cleanupTickerC = cleanupTicker.C } - if !retryStartupStatus(ctx, startupStatusReportMaxAttempts, 5*time.Second, func() bool { - return a.reportStatus("startup") - }) { - if ctx.Err() != nil { - return - } - log.Printf("[status] startup report unavailable after %d attempts; continuing with validated cached state", startupStatusReportMaxAttempts) - } go a.StatusReportLoop(ctx) a.Tick() @@ -104,36 +94,6 @@ func (a *Agent) Run(ctx context.Context) { } } -func retryStartupStatus( - ctx context.Context, - maxAttempts int, - retryDelay time.Duration, - report func() bool, -) bool { - for attempt := 1; attempt <= maxAttempts; attempt++ { - if report() { - return true - } - if attempt == maxAttempts { - return false - } - log.Printf("[status] startup report failed; retrying before first expected-state fetch") - timer := time.NewTimer(retryDelay) - select { - case <-ctx.Done(): - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } - return false - case <-timer.C: - } - } - return false -} - func (a *Agent) TraefikMetricsLoop(ctx context.Context) { ticker := time.NewTicker(traefikMetricsInterval) defer ticker.Stop() @@ -187,6 +147,8 @@ func (a *Agent) ForwardTraefikMetrics(ctx context.Context) error { } func (a *Agent) StatusReportLoop(ctx context.Context) { + a.reportStatus("startup") + timer := time.NewTimer(nextStatusReportDelay()) defer timer.Stop() @@ -219,7 +181,7 @@ func (a *Agent) RequestStatusReport(reason string) { } } -func (a *Agent) reportStatus(reason string) bool { +func (a *Agent) reportStatus(reason string) { startedAt := time.Now() report := a.BuildStatusReport(true) reportedDeploymentErrorCount := len(report.DeploymentErrors) @@ -228,7 +190,7 @@ func (a *Agent) reportStatus(reason string) bool { response, err := a.Client.ReportStatus(report, completed, active, serverlessTransitions) if err != nil { log.Printf("[status] failed to report (%s) latency=%s: %v", reason, time.Since(startedAt).Round(time.Millisecond), err) - return false + return } a.ClearReportedDeploymentErrors(reportedDeploymentErrorCount) a.AcknowledgeServerlessTransitions(response.ServerlessTransitionResults, len(serverlessTransitions)) @@ -236,7 +198,6 @@ func (a *Agent) reportStatus(reason string) bool { a.LogRejectedActiveWorkItems(response.RejectedActiveWorkItems) a.AcceptLeasedWorkItems(response.WorkItems) log.Printf("[status] reported (%s) latency=%s", reason, time.Since(startedAt).Round(time.Millisecond)) - return true } func nextStatusReportDelay() time.Duration { diff --git a/agent/internal/agent/run_test.go b/agent/internal/agent/run_test.go deleted file mode 100644 index 0448faf..0000000 --- a/agent/internal/agent/run_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package agent - -import ( - "context" - "testing" -) - -func TestRetryStartupStatusIsBounded(t *testing.T) { - attempts := 0 - reported := retryStartupStatus(context.Background(), 3, 0, func() bool { - attempts++ - return false - }) - - if reported { - t.Fatal("expected startup reporting to fall through to cached state") - } - if attempts != 3 { - t.Fatalf("attempts = %d, want 3", attempts) - } -} - -func TestRetryStartupStatusStopsAfterSuccess(t *testing.T) { - attempts := 0 - reported := retryStartupStatus(context.Background(), 3, 0, func() bool { - attempts++ - return attempts == 2 - }) - - if !reported { - t.Fatal("expected startup reporting to succeed") - } - if attempts != 2 { - t.Fatalf("attempts = %d, want 2", attempts) - } -} diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index cafeb79..7bbbc70 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -41,10 +41,12 @@ func (a *Agent) DeployServerlessContainer(expected agenthttp.ExpectedContainer) if actual.DeploymentID != expected.DeploymentID { continue } - if !runningContainerMatchesExpectedRevision(actual, expected) { + if normalizeImage(actual.Image) != normalizeImage(expected.Image) { log.Printf( - "[serverless] recreate deployment %s because revision/spec changed", + "[serverless] recreate deployment %s because image changed (%s -> %s)", Truncate(expected.DeploymentID, 8), + actual.Image, + expected.Image, ) return a.Reconciler.Deploy(expected) } @@ -79,7 +81,7 @@ func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) er if actual.DeploymentID != expected.DeploymentID || actual.State != "running" { continue } - if runningContainerMatchesExpectedRevision(actual, expected) { + if normalizeImage(actual.Image) == normalizeImage(expected.Image) { return nil } } @@ -88,15 +90,6 @@ func (a *Agent) DeployExpectedContainer(expected agenthttp.ExpectedContainer) er }) } -func runningContainerMatchesExpectedRevision( - actual container.Container, - expected agenthttp.ExpectedContainer, -) bool { - return actual.SpecHash != "" && - actual.SpecHash == expected.ContainerSpecHash && - actual.RevisionID == expected.RevisionID -} - func (a *Agent) withDeploymentDeployLock(deploymentID string, fn func() error) error { a.deployLockMutex.Lock() if a.deploymentDeployLocks == nil { diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index c86e96c..a0993d9 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -155,8 +155,6 @@ func buildPodmanRunArgs(config *DeployConfig, image string) []string { "--label", fmt.Sprintf("techulus.service.id=%s", config.ServiceID), "--label", fmt.Sprintf("techulus.service.name=%s", config.ServiceName), "--label", fmt.Sprintf("techulus.deployment.id=%s", config.DeploymentID), - "--label", fmt.Sprintf("techulus.service.revision=%s", config.RevisionID), - "--label", fmt.Sprintf("techulus.container.spec-hash=%s", config.ContainerSpecHash), ) if config.IPAddress != "" { args = append(args, "--network", NetworkName, "--ip", config.IPAddress) @@ -516,8 +514,6 @@ func List() ([]Container, error) { Labels: pc.Labels, DeploymentID: pc.Labels["techulus.deployment.id"], ServiceID: pc.Labels["techulus.service.id"], - RevisionID: pc.Labels["techulus.service.revision"], - SpecHash: pc.Labels["techulus.container.spec-hash"], } } diff --git a/agent/internal/container/runtime_test.go b/agent/internal/container/runtime_test.go index f517d1a..1fcf4b1 100644 --- a/agent/internal/container/runtime_test.go +++ b/agent/internal/container/runtime_test.go @@ -1,10 +1,7 @@ package container import ( - "os" - "path/filepath" "slices" - "strings" "testing" ) @@ -76,30 +73,3 @@ func TestBuildPodmanRunArgsDoesNotPublishStaticIPPortsByDefault(t *testing.T) { t.Fatalf("args unexpectedly publish ports: %+v", args) } } - -func TestDeployPullFailureLeavesExistingContainerUntouched(t *testing.T) { - tempDir := t.TempDir() - logPath := filepath.Join(tempDir, "podman.log") - podmanPath := filepath.Join(tempDir, "podman") - script := "#!/bin/sh\nprintf '%s\\n' \"$1\" >> \"$PODMAN_TEST_LOG\"\nif [ \"$1\" = pull ]; then exit 1; fi\n" - if err := os.WriteFile(podmanPath, []byte(script), 0755); err != nil { - t.Fatal(err) - } - t.Setenv("PATH", tempDir+string(os.PathListSeparator)+os.Getenv("PATH")) - t.Setenv("PODMAN_TEST_LOG", logPath) - - _, err := Deploy(&DeployConfig{ - Name: "existing-container", - Image: "registry.invalid/missing:latest", - }) - if err == nil { - t.Fatal("expected image pull to fail") - } - commands, readErr := os.ReadFile(logPath) - if readErr != nil { - t.Fatal(readErr) - } - if got := strings.TrimSpace(string(commands)); got != "pull" { - t.Fatalf("podman commands = %q, want only pull before failure", got) - } -} diff --git a/agent/internal/container/types.go b/agent/internal/container/types.go index a81395c..f057158 100644 --- a/agent/internal/container/types.go +++ b/agent/internal/container/types.go @@ -31,8 +31,6 @@ type DeployConfig struct { ServiceID string ServiceName string DeploymentID string - RevisionID string - ContainerSpecHash string WireGuardIP string IPAddress string PortMappings []PortMapping @@ -59,8 +57,6 @@ type Container struct { Labels map[string]string `json:"Labels"` DeploymentID string ServiceID string - RevisionID string - SpecHash string } type containerInspect struct { diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 363233b..22ee9dd 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -3,7 +3,6 @@ package http import ( "bytes" "encoding/json" - "errors" "fmt" "io" "log" @@ -11,7 +10,6 @@ import ( "os" "path/filepath" "strconv" - "sync" "time" "techulus/cloud-agent/internal/crypto" @@ -19,22 +17,12 @@ import ( ) type Client struct { - baseURL string - serverID string - keyPair *crypto.KeyPair - client *http.Client - longClient *http.Client - dataDir string - upgradeLogMutex sync.Mutex - lastUpgradeRequiredLog time.Time -} - -type UpgradeRequiredError struct { - Message string -} - -func (e *UpgradeRequiredError) Error() string { - return fmt.Sprintf("agent upgrade required: %s", e.Message) + baseURL string + serverID string + keyPair *crypto.KeyPair + client *http.Client + longClient *http.Client + dataDir string } func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string) *Client { @@ -82,8 +70,6 @@ type VolumeMount struct { type ExpectedContainer struct { DeploymentID string `json:"deploymentId"` - RevisionID string `json:"revisionId"` - ContainerSpecHash string `json:"containerSpecHash"` ServiceID string `json:"serviceId"` ServiceName string `json:"serviceName"` Name string `json:"name"` @@ -167,10 +153,9 @@ type WireGuardPeer struct { } type ExpectedState struct { - SchemaVersion int `json:"schemaVersion"` - ServerName string `json:"serverName"` - Containers []ExpectedContainer `json:"containers"` - Dns struct { + ServerName string `json:"serverName"` + Containers []ExpectedContainer `json:"containers"` + Dns struct { Records []DnsRecord `json:"records"` } `json:"dns"` Serverless struct { @@ -215,9 +200,6 @@ func (c *Client) loadCachedExpectedState() (*ExpectedState, error) { if err := json.Unmarshal(data, &state); err != nil { return nil, err } - if err := validateExpectedState(&state); err != nil { - return nil, fmt.Errorf("cached expected state is invalid: %w", err) - } return &state, nil } @@ -237,9 +219,6 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - if resp.StatusCode == http.StatusUpgradeRequired { - return nil, &UpgradeRequiredError{Message: string(body)} - } return nil, fmt.Errorf("expected state request failed with status %d: %s", resp.StatusCode, string(body)) } @@ -247,9 +226,6 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { if err := json.NewDecoder(resp.Body).Decode(&state); err != nil { return nil, fmt.Errorf("failed to decode expected state: %w", err) } - if err := validateExpectedState(&state); err != nil { - return nil, err - } if err := c.cacheExpectedState(&state); err != nil { log.Printf("[cache] failed to cache expected state: %v", err) @@ -258,38 +234,13 @@ func (c *Client) getExpectedState() (*ExpectedState, error) { return &state, nil } -func validateExpectedState(state *ExpectedState) error { - if state.SchemaVersion != 1 { - return fmt.Errorf("unsupported expected state schema version %d", state.SchemaVersion) - } - for _, expectedContainer := range state.Containers { - if expectedContainer.DeploymentID == "" { - return fmt.Errorf("expected container is missing deploymentId") - } - if expectedContainer.RevisionID == "" { - return fmt.Errorf("expected container %s is missing revisionId", expectedContainer.DeploymentID) - } - if expectedContainer.ContainerSpecHash == "" { - return fmt.Errorf("expected container %s is missing containerSpecHash", expectedContainer.DeploymentID) - } - } - return nil -} - func (c *Client) GetExpectedStateWithFallback() (*ExpectedState, bool, error) { state, err := c.getExpectedState() if err == nil { return state, false, nil } - var upgradeRequired *UpgradeRequiredError - if errors.As(err, &upgradeRequired) { - if c.shouldLogUpgradeRequired() { - log.Printf("[state] UPGRADE REQUIRED: control plane rejected this agent; serving cached state until upgraded: %v", err) - } - } else { - log.Printf("[state] CP unreachable, attempting to use cached state: %v", err) - } + log.Printf("[state] CP unreachable, attempting to use cached state: %v", err) cachedState, cacheErr := c.loadCachedExpectedState() if cacheErr != nil { return nil, false, fmt.Errorf("CP unreachable and no cached state available: %w (cache error: %v)", err, cacheErr) @@ -298,16 +249,6 @@ func (c *Client) GetExpectedStateWithFallback() (*ExpectedState, bool, error) { return cachedState, true, nil } -func (c *Client) shouldLogUpgradeRequired() bool { - c.upgradeLogMutex.Lock() - defer c.upgradeLogMutex.Unlock() - if time.Since(c.lastUpgradeRequiredLog) < time.Minute { - return false - } - c.lastUpgradeRequiredLog = time.Now() - return true -} - type ContainerStatus struct { DeploymentID string `json:"deploymentId"` ContainerID string `json:"containerId"` @@ -327,19 +268,9 @@ type Resources struct { } type AgentHealth struct { - Version string `json:"version"` - UptimeSecs int64 `json:"uptimeSecs"` - Capabilities []string `json:"capabilities,omitempty"` - ReconciliationFailures []ReconciliationFailure `json:"reconciliationFailures,omitempty"` -} - -type ReconciliationFailure struct { - Action string `json:"action"` - DeploymentID string `json:"deploymentId,omitempty"` - Description string `json:"description"` - LastError string `json:"lastError"` - Attempts int `json:"attempts"` - NextRetryAt time.Time `json:"nextRetryAt"` + Version string `json:"version"` + UptimeSecs int64 `json:"uptimeSecs"` + Capabilities []string `json:"capabilities,omitempty"` } type StatusReport struct { diff --git a/agent/internal/http/client_test.go b/agent/internal/http/client_test.go deleted file mode 100644 index c3451a7..0000000 --- a/agent/internal/http/client_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package http - -import "testing" - -func TestValidateExpectedStateRejectsPartialRevisionContract(t *testing.T) { - tests := []struct { - name string - state ExpectedState - }{ - {name: "old schema", state: ExpectedState{}}, - { - name: "missing revision", - state: ExpectedState{ - SchemaVersion: 1, - Containers: []ExpectedContainer{ - {DeploymentID: "deployment", ContainerSpecHash: "hash"}, - }, - }, - }, - { - name: "missing container hash", - state: ExpectedState{ - SchemaVersion: 1, - Containers: []ExpectedContainer{ - {DeploymentID: "deployment", RevisionID: "revision"}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validateExpectedState(&tt.state); err == nil { - t.Fatal("expected validation error") - } - }) - } -} - -func TestValidateExpectedStateAcceptsCompleteRevisionContract(t *testing.T) { - state := ExpectedState{ - SchemaVersion: 1, - Containers: []ExpectedContainer{ - { - DeploymentID: "deployment", - RevisionID: "revision", - ContainerSpecHash: "hash", - }, - }, - } - if err := validateExpectedState(&state); err != nil { - t.Fatalf("validateExpectedState() error = %v", err) - } -} diff --git a/agent/internal/reconcile/reconcile.go b/agent/internal/reconcile/reconcile.go index 4449764..65635af 100644 --- a/agent/internal/reconcile/reconcile.go +++ b/agent/internal/reconcile/reconcile.go @@ -69,8 +69,6 @@ func (r *Reconciler) Deploy(exp agenthttp.ExpectedContainer) error { ServiceID: exp.ServiceID, ServiceName: exp.ServiceName, DeploymentID: exp.DeploymentID, - RevisionID: exp.RevisionID, - ContainerSpecHash: exp.ContainerSpecHash, IPAddress: exp.IPAddress, PortMappings: portMappings, PublishLocalPorts: exp.PublishLocalPorts, diff --git a/deployment/README.md b/deployment/README.md index 6463a03..5d9ef44 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -92,51 +92,14 @@ Schema is synced automatically by the one-shot `migrate` service via `drizzle-ki ### Immutable service revision cutover -The service-revision release is a maintenance-window cutover. Do not start the -new agents across the fleet at the same time. - -1. Pause builds, schedules, migrations, restores, and rollout workers. Verify - that no rollout is `queued` or `in_progress`. Verify every service has no - pending secret additions, edits, or removals; deploy or revert those changes - before the maintenance window. -2. Set `EXPECTED_STATE_MAINTENANCE_MODE=true`. Stop every old agent so each - server keeps running its last applied container and cluster state. -3. Stop the old `web` and `inngest` services. Take and verify a PostgreSQL - backup. -4. Run the new `migrate` image. It executes - `scripts/cutover-service-revisions.ts` before `drizzle-kit push`. The script - aborts and rolls back if active rollouts exist or any runtime row cannot be - attached to a baseline revision. -5. Start the new control plane while maintenance mode remains enabled. Install - the new agent binary on every server, but leave the agents stopped. -6. Set `EXPECTED_STATE_MAINTENANCE_MODE=false` and restart the control plane. - Confirm server health identifies every stopped or old node as requiring the - `service_revision_v1` capability. -7. Start one server agent. Its synchronous startup report must register the - capability before it requests expected state. -8. Let that server recreate its pre-cutover containers one at a time. Verify - container health, volume mounts, DNS, HTTP and L4 routes, certificates, - WireGuard, and serverless behavior before starting the next server. -9. Continue through the fleet. Handle stateful and single-replica services in - an explicit maintenance order, then resume workflow producers. - -Each recreation pulls and verifies the image before removing the old container. -A failed pull leaves the old container running and backs off only that action; -other container and cluster reconciliation continues. - -The agent waits for each recreated container before starting the next legacy -recreation. Services with a health check must become healthy; services without -one must remain running through a 30-second stabilization period. - -Pull-before-remove minimizes downtime but cannot eliminate it. The replacement -uses the same static IP, so every container has a stop-to-start gap. Existing -connections to that replica may reset. Multi-replica services remain available -only through healthy replicas on other containers. Single-replica and stateful -services have bounded customer-visible downtime. Verify a healthy remaining -replica before allowing the next replica recreation. - -Keep `services.deployed_config` unchanged for the burn-in release. The -application does not read or write it; it remains only as recovery evidence. +Before deploying this release, stop rollout producers, verify no rollout is +queued or in progress, and take a PostgreSQL backup. The migrate service creates +one baseline revision for each existing deployment before applying the new +schema. Existing containers keep running unchanged; subsequent deployments use +immutable revision snapshots. Stale non-active deployment records are removed +before the baseline is captured. The cutover aborts if an active deployment +cannot be reconstructed safely from its stored deployed configuration; that +legacy column is dropped after the backfill succeeds. **Future plan:** Once the schema stabilizes, switch to `drizzle-kit generate` + `drizzle-orm migrate()` with pre-generated SQL migration files. This will eliminate the esbuild/drizzle-kit dependency from the production image. diff --git a/deployment/compose.postgres.yml b/deployment/compose.postgres.yml index 0a5226a..3a196a0 100644 --- a/deployment/compose.postgres.yml +++ b/deployment/compose.postgres.yml @@ -90,7 +90,6 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: postgres: condition: service_healthy @@ -119,7 +118,6 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: migrate: condition: service_completed_successfully diff --git a/deployment/compose.production.yml b/deployment/compose.production.yml index 1efc382..94dca63 100644 --- a/deployment/compose.production.yml +++ b/deployment/compose.production.yml @@ -72,7 +72,6 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} command: ["sh", "-c", "npx tsx scripts/cutover-service-revisions.ts && npx drizzle-kit push --force"] restart: on-failure @@ -98,7 +97,6 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - - EXPECTED_STATE_MAINTENANCE_MODE=${EXPECTED_STATE_MAINTENANCE_MODE:-false} depends_on: migrate: condition: service_completed_successfully diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 7d419a6..56c886a 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -13,6 +13,7 @@ import { getService, } from "@/db/queries"; import { + deploymentPorts, deployments, environments, githubRepos, @@ -22,7 +23,6 @@ import { servers, servicePorts, serviceReplicas, - serviceRevisions, services, serviceVolumes, volumeBackups, @@ -32,13 +32,12 @@ import { requireDeveloperRole, verifyDeleteConfirmation } from "@/lib/auth"; import { deployServiceInternal } from "@/lib/deploy-service"; import { isObservedReady, - markDeploymentFailedRemoved, markDeploymentRemoved, runtimeExpectedStates, - selectNewestRevisionId, } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; +import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; import { blocksProjectDeletion } from "@/lib/project-deletion"; import { @@ -546,6 +545,10 @@ async function hardDeleteService(serviceId: string) { containerId: dep.containerId, }); } + + await db + .delete(deploymentPorts) + .where(eq(deploymentPorts.deploymentId, dep.id)); } await db.delete(deployments).where(eq(deployments.serviceId, serviceId)); @@ -886,7 +889,7 @@ export async function updateServiceGithubRepo( export async function deployService(serviceId: string) { await requireDeveloperRole(); - return deployServiceInternal(serviceId, { trigger: "ui" }); + return deployServiceInternal(serviceId); } export async function deleteDeployments(serviceId: string) { @@ -1348,86 +1351,15 @@ export async function restartService(serviceId: string) { export async function abortRollout(serviceId: string) { await requireDeveloperRole(); - const { activeRollouts, rolloutDeployments } = await db.transaction( - async (tx) => { - const activeRollouts = await tx - .select({ id: rollouts.id, status: rollouts.status }) - .from(rollouts) - .where( - and( - eq(rollouts.serviceId, serviceId), - inArray(rollouts.status, ["queued", "in_progress"]), - ), - ) - .for("update"); - - if (activeRollouts.length === 0) { - return { activeRollouts, rolloutDeployments: [] }; - } - - const activeRolloutIds = activeRollouts.map((rollout) => rollout.id); - const rolloutDeployments = await tx - .select() - .from(deployments) - .where(inArray(deployments.rolloutId, activeRolloutIds)); - - await tx - .update(rollouts) - .set({ - status: "failed", - currentStage: "aborted", - completedAt: new Date(), - }) - .where(inArray(rollouts.id, activeRolloutIds)); - - await tx - .update(deployments) - .set({ trafficState: "active" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "draining"), - ), - ); - - await tx - .update(deployments) - .set(markDeploymentFailedRemoved("aborted")) - .where(inArray(deployments.rolloutId, activeRolloutIds)); - - const activeRevisionRows = await tx - .select({ - serviceRevisionId: deployments.serviceRevisionId, - revisionNumber: serviceRevisions.revisionNumber, - }) - .from(deployments) - .innerJoin( - serviceRevisions, - eq(deployments.serviceRevisionId, serviceRevisions.id), - ) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "active"), - inArray(deployments.runtimeDesiredState, runtimeExpectedStates), - ), - ); - const activeRevisionCount = new Set( - activeRevisionRows.map((row) => row.serviceRevisionId), - ).size; - if (activeRevisionCount > 1) { - console.error( - `[rollout:abort] found ${activeRevisionCount} active revisions for ${serviceId}; selecting the newest`, - ); - } - await tx - .update(services) - .set({ activeRevisionId: selectNewestRevisionId(activeRevisionRows) }) - .where(eq(services.id, serviceId)); - - return { activeRollouts, rolloutDeployments }; - }, - ); + const activeRollouts = await db + .select({ id: rollouts.id, status: rollouts.status }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ); if (activeRollouts.length === 0) { return { success: false, error: "No in-progress rollout found" }; @@ -1435,21 +1367,33 @@ export async function abortRollout(serviceId: string) { const activeRolloutIds = activeRollouts.map((rollout) => rollout.id); + await db + .update(rollouts) + .set({ + status: "failed", + currentStage: "aborted", + completedAt: new Date(), + }) + .where(inArray(rollouts.id, activeRolloutIds)); + for (const rolloutId of activeRolloutIds) { - await inngest - .send( - inngestEvents.rolloutCancelled.create({ - rolloutId, - }), - ) - .catch((error) => { - console.error( - `[rollout:${rolloutId}] failed to send cancellation event:`, - error, - ); - }); + await inngest.send( + inngestEvents.rolloutCancelled.create({ + rolloutId, + }), + ); } + await restoreDrainingDeploymentsForRollback(serviceId); + + const rolloutDeployments = + activeRolloutIds.length > 0 + ? await db + .select() + .from(deployments) + .where(inArray(deployments.rolloutId, activeRolloutIds)) + : []; + const serverContainers = new Map(); for (const dep of rolloutDeployments) { @@ -1464,21 +1408,19 @@ export async function abortRollout(serviceId: string) { await enqueueWork(serverId, "force_cleanup", { serviceId, containerIds, - }).catch((error) => { - console.error( - `[rollout] failed to enqueue cleanup on server ${serverId}:`, - error, - ); }); } - const rolloutDeploymentIds = rolloutDeployments.map( - (deployment) => deployment.id, - ); - if (rolloutDeploymentIds.length > 0) { + for (const dep of rolloutDeployments) { + await db + .delete(deploymentPorts) + .where(eq(deploymentPorts.deploymentId, dep.id)); + } + + if (activeRolloutIds.length > 0) { await db .delete(deployments) - .where(inArray(deployments.id, rolloutDeploymentIds)); + .where(inArray(deployments.rolloutId, activeRolloutIds)); } const pendingWork = @@ -1495,11 +1437,11 @@ export async function abortRollout(serviceId: string) { ) : []; - const rolloutDeploymentIdSet = new Set(rolloutDeploymentIds); + const rolloutDeploymentIds = new Set(rolloutDeployments.map((d) => d.id)); const workToDelete = pendingWork.filter((w) => { try { const parsed = JSON.parse(w.payload); - return rolloutDeploymentIdSet.has(parsed.deploymentId); + return rolloutDeploymentIds.has(parsed.deploymentId); } catch { return false; } diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx index fc544bc..6ecc746 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/rollouts/[rolloutId]/page.tsx @@ -3,7 +3,7 @@ import { notFound } from "next/navigation"; import { SetBreadcrumbs } from "@/components/core/breadcrumb-data"; import { RolloutDetails } from "@/components/service/details/rollout-details"; import { db } from "@/db"; -import { projects, rollouts, serviceRevisions, services } from "@/db/schema"; +import { projects, rollouts, services } from "@/db/schema"; async function getRollout( projectSlug: string, @@ -27,22 +27,8 @@ async function getRollout( if (!service) return null; const rollout = await db - .select({ - id: rollouts.id, - serviceId: rollouts.serviceId, - serviceRevisionId: rollouts.serviceRevisionId, - status: rollouts.status, - currentStage: rollouts.currentStage, - createdAt: rollouts.createdAt, - completedAt: rollouts.completedAt, - revisionNumber: serviceRevisions.revisionNumber, - contentHash: serviceRevisions.contentHash, - }) + .select() .from(rollouts) - .innerJoin( - serviceRevisions, - eq(rollouts.serviceRevisionId, serviceRevisions.id), - ) .where(and(eq(rollouts.id, rolloutId), eq(rollouts.serviceId, serviceId))) .then((r) => r[0]); diff --git a/web/app/(dashboard)/dashboard/servers/[id]/page.tsx b/web/app/(dashboard)/dashboard/servers/[id]/page.tsx index bed4841..ca9ad17 100644 --- a/web/app/(dashboard)/dashboard/servers/[id]/page.tsx +++ b/web/app/(dashboard)/dashboard/servers/[id]/page.tsx @@ -182,7 +182,6 @@ export default async function ServerDetailPage({ networkHealth: server.networkHealth, containerHealth: server.containerHealth, agentHealth: server.agentHealth, - agentCompatibilityStatus: server.agentCompatibilityStatus, }} /> diff --git a/web/app/api/projects/[id]/services/route.ts b/web/app/api/projects/[id]/services/route.ts index 9897533..3562878 100644 --- a/web/app/api/projects/[id]/services/route.ts +++ b/web/app/api/projects/[id]/services/route.ts @@ -1,6 +1,5 @@ export const dynamic = "force-dynamic"; -import { createHash } from "node:crypto"; import { and, desc, eq, inArray, isNull } from "drizzle-orm"; import { headers } from "next/headers"; import { db } from "@/db"; @@ -22,10 +21,6 @@ import { auth } from "@/lib/auth"; import { getTimestamp } from "@/lib/date"; import { revisionSpecToDeployedConfig } from "@/lib/service-config"; -function secretFingerprint(encryptedValue: string) { - return createHash("sha256").update(encryptedValue).digest("hex"); -} - export async function GET( request: Request, { params }: { params: Promise<{ id: string }> }, @@ -67,7 +62,6 @@ export async function GET( volumes, lockedServer, latestBuild, - activeRevision, ] = await Promise.all([ db .select() @@ -91,11 +85,7 @@ export async function GET( .innerJoin(servers, eq(serviceReplicas.serverId, servers.id)) .where(eq(serviceReplicas.serviceId, service.id)), db - .select({ - key: secrets.key, - updatedAt: secrets.updatedAt, - encryptedValue: secrets.encryptedValue, - }) + .select({ key: secrets.key, updatedAt: secrets.updatedAt }) .from(secrets) .where(eq(secrets.serviceId, service.id)), db @@ -124,40 +114,39 @@ export async function GET( .limit(1) .then((r) => r[0] || null) : Promise.resolve(null), - service.activeRevisionId - ? db - .select({ specification: serviceRevisions.specification }) - .from(serviceRevisions) - .where(eq(serviceRevisions.id, service.activeRevisionId)) - .then((rows) => rows[0] ?? null) - : Promise.resolve(null), ]); - const revisionServerIds = - activeRevision?.specification.placements.map( - (placement) => placement.serverId, - ) ?? []; - const revisionServers = - revisionServerIds.length > 0 - ? await db - .select({ id: servers.id, name: servers.name }) - .from(servers) - .where(inArray(servers.id, revisionServerIds)) - : []; - const serverNames = Object.fromEntries( - revisionServers.map((server) => [server.id, server.name]), - ); - const secretFingerprints = Object.fromEntries( - activeRevision?.specification.secrets.map((secret) => [ - secret.key, - secretFingerprint(secret.encryptedValue), - ]) ?? [], + const activeDeployment = serviceDeployments.find( + (deployment) => + deployment.trafficState === "active" && + deployment.runtimeDesiredState !== "removed", ); + const activeRevision = activeDeployment + ? await db + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where(eq(serviceRevisions.id, activeDeployment.serviceRevisionId)) + .then((rows) => rows[0]) + : null; + const revisionServers = activeRevision + ? await db + .select({ id: servers.id, name: servers.name }) + .from(servers) + .where( + inArray( + servers.id, + activeRevision.specification.placements.map( + (placement) => placement.serverId, + ), + ), + ) + : []; const activeConfig = activeRevision ? revisionSpecToDeployedConfig( activeRevision.specification, - serverNames, - secretFingerprints, + Object.fromEntries( + revisionServers.map((server) => [server.id, server.name]), + ), ) : null; @@ -243,16 +232,12 @@ export async function GET( ports, configuredReplicas: replicas, deployments: deploymentsWithDetails, - secrets: serviceSecrets.map((secret) => ({ - key: secret.key, - updatedAt: secret.updatedAt, - fingerprint: secretFingerprint(secret.encryptedValue), - })), - activeConfig, + secrets: serviceSecrets, rollouts: serviceRollouts, volumes, lockedServer, latestBuild, + activeConfig, deletionBackupFallback, }; }), diff --git a/web/app/api/services/[id]/rollouts/route.ts b/web/app/api/services/[id]/rollouts/route.ts index 3e1c1a4..b5053a3 100644 --- a/web/app/api/services/[id]/rollouts/route.ts +++ b/web/app/api/services/[id]/rollouts/route.ts @@ -1,33 +1,19 @@ export const dynamic = "force-dynamic"; +import { NextRequest, NextResponse } from "next/server"; import { desc, eq } from "drizzle-orm"; -import { type NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; -import { rollouts, serviceRevisions } from "@/db/schema"; +import { rollouts } from "@/db/schema"; export async function GET( - _request: NextRequest, + request: NextRequest, { params }: { params: Promise<{ id: string }> }, ) { const { id: serviceId } = await params; const rolloutsList = await db - .select({ - id: rollouts.id, - serviceId: rollouts.serviceId, - serviceRevisionId: rollouts.serviceRevisionId, - status: rollouts.status, - currentStage: rollouts.currentStage, - createdAt: rollouts.createdAt, - completedAt: rollouts.completedAt, - revisionNumber: serviceRevisions.revisionNumber, - contentHash: serviceRevisions.contentHash, - }) + .select() .from(rollouts) - .innerJoin( - serviceRevisions, - eq(rollouts.serviceRevisionId, serviceRevisions.id), - ) .where(eq(rollouts.serviceId, serviceId)) .orderBy(desc(rollouts.createdAt)) .limit(50); diff --git a/web/app/api/v1/agent/builds/[id]/status/route.ts b/web/app/api/v1/agent/builds/[id]/status/route.ts index 003abe2..0e53d83 100644 --- a/web/app/api/v1/agent/builds/[id]/status/route.ts +++ b/web/app/api/v1/agent/builds/[id]/status/route.ts @@ -352,10 +352,7 @@ export async function POST( ); try { - await deployServiceInternal(build.serviceId, { - trigger: "agent_build", - buildId, - }); + await deployServiceInternal(build.serviceId); } catch (error) { console.error("[build:complete] deployment failed:", error); await db diff --git a/web/app/api/v1/agent/expected-state/route.ts b/web/app/api/v1/agent/expected-state/route.ts index cbdc55d..57593db 100644 --- a/web/app/api/v1/agent/expected-state/route.ts +++ b/web/app/api/v1/agent/expected-state/route.ts @@ -1,10 +1,9 @@ import { type NextRequest, NextResponse } from "next/server"; -import { buildAgentExpectedState, getServer } from "@/lib/agent/expected-state"; -import { verifyAgentRequest } from "@/lib/agent-auth"; import { - getAgentCompatibilityStatus, - SERVICE_REVISION_CAPABILITY, -} from "@/lib/agent-capabilities"; + buildAgentExpectedState, + getServer, +} from "@/lib/agent/expected-state"; +import { verifyAgentRequest } from "@/lib/agent-auth"; export async function GET(request: NextRequest) { const auth = await verifyAgentRequest(request); @@ -16,45 +15,6 @@ export async function GET(request: NextRequest) { if (!server) { return NextResponse.json({ error: "Server not found" }, { status: 404 }); } - if (process.env.EXPECTED_STATE_MAINTENANCE_MODE === "true") { - return NextResponse.json( - { - error: "Expected state is paused for maintenance", - code: "EXPECTED_STATE_MAINTENANCE", - }, - { - status: 503, - headers: { "Retry-After": "30" }, - }, - ); - } - if (getAgentCompatibilityStatus(server.agentHealth) === "upgrade_required") { - return NextResponse.json( - { - error: "Agent upgrade required", - code: "AGENT_UPGRADE_REQUIRED", - requiredCapabilities: [SERVICE_REVISION_CAPABILITY], - }, - { status: 426 }, - ); - } - try { - return NextResponse.json(await buildAgentExpectedState(server)); - } catch (error) { - console.error( - `[expected-state] failed to build state for server ${server.id}:`, - error, - ); - return NextResponse.json( - { - error: "Expected state is temporarily unavailable", - code: "EXPECTED_STATE_BUILD_FAILED", - }, - { - status: 503, - headers: { "Retry-After": "15" }, - }, - ); - } + return NextResponse.json(await buildAgentExpectedState(server)); } diff --git a/web/components/server/server-health-details.tsx b/web/components/server/server-health-details.tsx index fbd774a..de6973e 100644 --- a/web/components/server/server-health-details.tsx +++ b/web/components/server/server-health-details.tsx @@ -2,7 +2,6 @@ import { Activity, - AlertTriangle, Container, Cpu, HardDrive, @@ -12,7 +11,6 @@ import { import useSWR from "swr"; import { HealthIndicator } from "@/components/cluster/health-indicator"; import { ResourceBar } from "@/components/cluster/resource-bar"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import type { HealthStats, Server } from "@/db/types"; @@ -23,7 +21,6 @@ type ServerHealthData = { networkHealth: Server["networkHealth"]; containerHealth: Server["containerHealth"]; agentHealth: Server["agentHealth"]; - agentCompatibilityStatus: "compatible" | "upgrade_required" | null; }; type ClusterHealthResponse = { @@ -59,10 +56,6 @@ export function ServerHealthDetails({ const containerHealth = serverData?.containerHealth ?? initialData.containerHealth; const agentHealth = serverData?.agentHealth ?? initialData.agentHealth; - const agentCompatibilityStatus = - serverData?.agentCompatibilityStatus ?? - initialData.agentCompatibilityStatus; - const reconciliationFailures = agentHealth?.reconciliationFailures ?? []; if (!healthStats && !networkHealth && !containerHealth && !agentHealth) { return null; @@ -74,30 +67,6 @@ export function ServerHealthDetails({ System Health - {agentCompatibilityStatus === "upgrade_required" && ( - - - Agent upgrade required - - This agent is incompatible with the current deployment protocol. - It will keep its cached state but cannot receive updates until it - is upgraded. - - - )} - {reconciliationFailures.length > 0 && ( - - - Reconciliation delayed - - {reconciliationFailures.length} action - {reconciliationFailures.length === 1 ? " is" : "s are"} being - retried without blocking other server updates. Latest:{" "} - {reconciliationFailures[0]?.description} ( - {reconciliationFailures[0]?.lastError}). - - - )} {healthStats && (
} /> } />
diff --git a/web/components/server/server-services.tsx b/web/components/server/server-services.tsx index 44441cf..1b1240f 100644 --- a/web/components/server/server-services.tsx +++ b/web/components/server/server-services.tsx @@ -1,5 +1,5 @@ -import { ArrowUpRight, Layers } from "lucide-react"; import Link from "next/link"; +import { ArrowUpRight, Layers } from "lucide-react"; import { Card, CardContent, @@ -47,14 +47,8 @@ export async function ServerServices({ serverId }: { serverId: string }) { href={`/dashboard/projects/${service.projectSlug}/${service.environmentName}/services/${service.serviceId}`} className="flex items-center justify-between gap-3 px-3 py-2.5 text-sm transition-colors hover:bg-muted" > - - - {service.serviceName} - - - revision {service.revisionNumber} ·{" "} - {service.revisionContentHash.slice(0, 12)} - + + {service.serviceName} diff --git a/web/components/service/details/rollout-details.tsx b/web/components/service/details/rollout-details.tsx index 3d286df..d6adce6 100644 --- a/web/components/service/details/rollout-details.tsx +++ b/web/components/service/details/rollout-details.tsx @@ -17,8 +17,6 @@ import { formatElapsedDurationBetween, formatRelativeTime } from "@/lib/date"; type RolloutWithDates = Omit & { createdAt: string | Date; completedAt: string | Date | null; - revisionNumber: number; - contentHash: string; }; const STATUS_CONFIG: Record< @@ -137,12 +135,6 @@ export function RolloutDetails({
-
- Immutable revision - #{rollout.revisionNumber} - · - {rollout.contentHash} -

Rollout Logs

- - Revision {rollout.revisionNumber} ·{" "} - {rollout.contentHash.slice(0, 12)} - {formatRelativeTime(rollout.createdAt)} Duration:{" "} diff --git a/web/db/queries.ts b/web/db/queries.ts index ebe04ae..7f70e81 100644 --- a/web/db/queries.ts +++ b/web/db/queries.ts @@ -5,12 +5,10 @@ import { environments, projects, servers, - serviceRevisions, services, settings, } from "@/db/schema"; import type { HealthStats } from "@/db/types"; -import { getAgentCompatibilityStatus } from "@/lib/agent-capabilities"; import type { ControlPlaneUpdateState, ControlPlaneUpgradeState, @@ -127,15 +125,7 @@ export async function getServerDetails(id: string) { .where(eq(servers.id, id)); const server = serverResults[0]; - return server - ? { - ...server, - healthStats: null, - agentCompatibilityStatus: getAgentCompatibilityStatus( - server.agentHealth, - ), - } - : null; + return server ? { ...server, healthStats: null } : null; } export async function getClusterHealth() { @@ -165,7 +155,6 @@ export async function getClusterHealth() { const serversWithHealth = allServers.map((server) => ({ ...server, healthStats: metricSnapshotToHealthStats(metricsByServer.get(server.id)), - agentCompatibilityStatus: getAgentCompatibilityStatus(server.agentHealth), })); const serversWithCurrentMetrics = serversWithHealth.filter( (server) => server.status === "online" && server.healthStats, @@ -244,8 +233,6 @@ export async function getServerServices(serverId: string) { .selectDistinctOn([services.id], { deploymentId: deployments.id, deploymentStatus: deployments.observedPhase, - revisionNumber: serviceRevisions.revisionNumber, - revisionContentHash: serviceRevisions.contentHash, serviceId: services.id, serviceName: services.name, serviceImage: services.image, @@ -256,10 +243,6 @@ export async function getServerServices(serverId: string) { }) .from(deployments) .innerJoin(services, eq(deployments.serviceId, services.id)) - .innerJoin( - serviceRevisions, - eq(deployments.serviceRevisionId, serviceRevisions.id), - ) .innerJoin(projects, eq(services.projectId, projects.id)) .innerJoin(environments, eq(services.environmentId, environments.id)) .where(eq(deployments.serverId, serverId)); diff --git a/web/db/schema.ts b/web/db/schema.ts index 7586b83..5727e28 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -1,6 +1,5 @@ import { relations, sql } from "drizzle-orm"; import { - type AnyPgColumn, bigint, boolean, foreignKey, @@ -307,14 +306,6 @@ export type AgentHealth = { version: string; uptimeSecs: number; capabilities?: string[]; - reconciliationFailures?: Array<{ - action: string; - deploymentId?: string; - description: string; - lastError: string; - attempts: number; - nextRetryAt: string; - }>; }; export type AgentUpgradeStatus = @@ -429,11 +420,6 @@ export const services = pgTable("services", { serverlessWakeTimeoutSeconds: integer("serverless_wake_timeout_seconds") .notNull() .default(300), - activeRevisionId: text("active_revision_id").references( - (): AnyPgColumn => serviceRevisions.id, - { onDelete: "set null" }, - ), - deployedConfig: text("deployed_config"), deploymentSchedule: text("deployment_schedule"), lastScheduledDeploymentRunAt: timestamp("last_scheduled_deployment_run_at", { withTimezone: true, @@ -570,29 +556,14 @@ export const serviceRevisions = pgTable( serviceId: text("service_id") .notNull() .references(() => services.id, { onDelete: "cascade" }), - revisionNumber: integer("revision_number").notNull(), - schemaVersion: integer("schema_version").notNull(), specification: jsonb("specification") .$type() .notNull(), - contentHash: text("content_hash").notNull(), - sourceMetadata: - jsonb("source_metadata").$type< - Record - >(), createdAt: timestamp("created_at", { withTimezone: true }) .defaultNow() .notNull(), }, (table) => [ - uniqueIndex("service_revisions_service_revision_number_idx").on( - table.serviceId, - table.revisionNumber, - ), - uniqueIndex("service_revisions_service_content_hash_idx").on( - table.serviceId, - table.contentHash, - ), unique("service_revisions_id_service_id_unique").on( table.id, table.serviceId, @@ -668,10 +639,6 @@ export const deployments = pgTable( index("deployments_service_id_idx").on(table.serviceId), index("deployments_service_revision_id_idx").on(table.serviceRevisionId), index("deployments_server_id_idx").on(table.serverId), - uniqueIndex("deployments_server_ip_address_idx").on( - table.serverId, - table.ipAddress, - ), index("deployments_runtime_desired_state_idx").on( table.runtimeDesiredState, ), @@ -692,7 +659,7 @@ export const rollouts = pgTable( serviceId: text("service_id") .notNull() .references(() => services.id, { onDelete: "cascade" }), - serviceRevisionId: text("service_revision_id").notNull(), + serviceRevisionId: text("service_revision_id"), status: text("status", { enum: ["queued", "in_progress", "completed", "failed", "rolled_back"], }) diff --git a/web/db/types.ts b/web/db/types.ts index 973ef96..6a8d6f4 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -13,7 +13,6 @@ import type { servers, servicePorts, serviceReplicas, - serviceRevisions, services, serviceVolumes, user, @@ -28,7 +27,6 @@ export type Service = typeof services.$inferSelect; export type ServicePort = typeof servicePorts.$inferSelect; export type ServiceVolume = typeof serviceVolumes.$inferSelect; export type ServiceReplica = typeof serviceReplicas.$inferSelect; -export type ServiceRevision = typeof serviceRevisions.$inferSelect; export type Secret = typeof secrets.$inferSelect; export type Deployment = typeof deployments.$inferSelect; export type DeploymentPort = typeof deploymentPorts.$inferSelect; @@ -57,6 +55,7 @@ export type HealthStats = { }; export type ServiceWithDetails = Service & { + activeConfig?: DeployedConfig | null; ports: ServicePort[]; configuredReplicas: Array< ServiceReplica & { serverName: string; serverIsProxy: boolean } @@ -70,13 +69,7 @@ export type ServiceWithDetails = Service & { } >; volumes?: ServiceVolume[]; - secrets?: Array< - Pick & { - updatedAt: Date | string; - fingerprint: string; - } - >; - activeConfig?: DeployedConfig | null; + secrets?: Array & { updatedAt: Date | string }>; rollouts?: Rollout[]; lockedServer?: Pick | null; latestBuild?: Pick | null; diff --git a/web/lib/agent-capabilities.ts b/web/lib/agent-capabilities.ts deleted file mode 100644 index c015ccc..0000000 --- a/web/lib/agent-capabilities.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { AgentHealth } from "@/db/schema"; - -export const SERVICE_REVISION_CAPABILITY = "service_revision_v1"; - -export type AgentCompatibilityStatus = "compatible" | "upgrade_required"; - -export function getAgentCompatibilityStatus( - agentHealth: AgentHealth | null | undefined, -): AgentCompatibilityStatus { - return agentHealth?.capabilities?.includes(SERVICE_REVISION_CAPABILITY) - ? "compatible" - : "upgrade_required"; -} diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index 000f5ae..51243b2 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -159,7 +159,6 @@ async function applyDeploymentErrors( }) .from(deployments) .innerJoin(servers, eq(deployments.serverId, servers.id)) - .innerJoin(services, eq(deployments.serviceId, services.id)) .innerJoin( serviceRevisions, eq(deployments.serviceRevisionId, serviceRevisions.id), @@ -288,7 +287,6 @@ async function applyServerlessTransitions( serverName: servers.name, }) .from(deployments) - .innerJoin(services, eq(deployments.serviceId, services.id)) .innerJoin( serviceRevisions, eq(deployments.serviceRevisionId, serviceRevisions.id), diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index adc95c7..db5c4b3 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { and, eq, inArray, isNull } from "drizzle-orm"; import { db } from "@/db"; import { @@ -46,21 +45,8 @@ export type RuntimeServiceRevision = { specification: ServiceRevisionSpec; }; -export type RuntimeServiceRevisionRow = { - deploymentId: string; - serviceId: string; - serviceName: string; - serviceActiveRevisionId: string | null; - revisionId: string; - revisionServiceId: string; - revisionSchemaVersion: number; - specification: ServiceRevisionSpec; -}; - export type ExpectedContainer = { deploymentId: string; - revisionId: string; - containerSpecHash: string; serviceId: string; serviceName: string; name: string; @@ -124,7 +110,6 @@ export type ServerlessRoute = { }; export type AgentExpectedState = { - schemaVersion: 1; serverName: string; containers: ExpectedContainer[]; dns: { records: Array<{ name: string; ips: string[] }> }; @@ -193,7 +178,6 @@ export async function buildAgentExpectedState( ); return { - schemaVersion: 1, serverName: server.name, containers, dns: { records: dnsRecords }, @@ -206,13 +190,9 @@ export async function buildAgentExpectedState( async function getRuntimeServiceRevisions(): Promise { const rows = await db .select({ - deploymentId: deployments.id, serviceId: services.id, serviceName: services.name, - serviceActiveRevisionId: services.activeRevisionId, revisionId: serviceRevisions.id, - revisionServiceId: serviceRevisions.serviceId, - revisionSchemaVersion: serviceRevisions.schemaVersion, specification: serviceRevisions.specification, }) .from(deployments) @@ -229,78 +209,23 @@ async function getRuntimeServiceRevisions(): Promise { ), ); - const { services: runtimeServices, errors } = - selectRuntimeServiceRevisions(rows); - for (const error of errors) { - console.error(`[expected-state] ${error}`); - } - return runtimeServices; -} - -export function selectRuntimeServiceRevisions( - rows: RuntimeServiceRevisionRow[], -): { services: RuntimeServiceRevision[]; errors: string[] } { - const rowsByService = groupBy(rows, (row) => row.serviceId); - const runtimeServices: RuntimeServiceRevision[] = []; - const errors: string[] = []; - - for (const [serviceId, serviceRows] of [...rowsByService.entries()].sort( - ([a], [b]) => a.localeCompare(b), - )) { - const invalidOwnership = serviceRows.find( - (row) => row.revisionServiceId !== serviceId, - ); - if (invalidOwnership) { - errors.push( - `service ${serviceId} omitted: deployment ${invalidOwnership.deploymentId} revision belongs to another service`, - ); - continue; + const runtimeServices = new Map(); + for (const row of rows) { + if (row.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION) { + throw new Error(`Service ${row.serviceId} uses an unsupported revision`); } - - const unsupported = serviceRows.find( - (row) => - row.revisionSchemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || - row.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION, - ); - if (unsupported) { - errors.push( - `service ${serviceId} omitted: deployment ${unsupported.deploymentId} uses an unsupported service revision`, - ); - continue; + const existing = runtimeServices.get(row.serviceId); + if (existing && existing.revisionId !== row.revisionId) { + throw new Error(`Service ${row.serviceId} has multiple active revisions`); } - - const rowsByRevision = new Map( - serviceRows.map((row) => [row.revisionId, row]), - ); - let selected: RuntimeServiceRevisionRow | undefined = [ - ...rowsByRevision.values(), - ][0]; - if (rowsByRevision.size > 1) { - const activeRevisionId = serviceRows[0]?.serviceActiveRevisionId; - selected = activeRevisionId - ? rowsByRevision.get(activeRevisionId) - : undefined; - if (!selected) { - errors.push( - `service ${serviceId} omitted: multiple active revisions have no authoritative active revision`, - ); - continue; - } - errors.push( - `service ${serviceId} has multiple active revisions; using authoritative revision ${selected.revisionId}`, - ); - } - - if (!selected) continue; - runtimeServices.push({ - id: serviceId, - name: selected.serviceName, - revisionId: selected.revisionId, - specification: selected.specification, + runtimeServices.set(row.serviceId, { + id: row.serviceId, + name: row.serviceName, + revisionId: row.revisionId, + specification: row.specification, }); } - - return { services: runtimeServices, errors }; + return [...runtimeServices.values()].sort((a, b) => a.id.localeCompare(b.id)); } async function buildExpectedContainers( @@ -397,13 +322,7 @@ export function buildExpectedContainersFromRows({ if (!revision) { throw new Error(`Deployment ${dep.id} has no service revision`); } - if (revision.serviceId !== dep.serviceId) { - throw new Error( - `Deployment ${dep.id} revision belongs to another service`, - ); - } if ( - revision.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || revision.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION ) { throw new Error( @@ -439,39 +358,24 @@ export function buildExpectedContainersFromRows({ name: volume.name, containerPath: volume.containerPath, })); - const creationSpec = { - image: normalizeImage(specification.image), - ipAddress: dep.ipAddress, - ports, - publishLocalPorts: specification.serverless.enabled, - env, - startCommand: specification.startCommand, - healthCheck: specification.healthCheck, - volumes, - resourceCpuLimit: specification.resourceLimits.cpuCores, - resourceMemoryLimitMb: specification.resourceLimits.memoryMb, - }; - return [ { deploymentId: dep.id, - revisionId: revision.id, - containerSpecHash: hashContainerCreationSpec(creationSpec), serviceId: dep.serviceId, serviceName: service.name, name: `${dep.serviceId}-${dep.id.slice(0, 8)}`, desiredState: dep.runtimeDesiredState === "stopped" ? "stopped" : "running", - image: creationSpec.image, + image: normalizeImage(specification.image), ipAddress: dep.ipAddress, ports, - publishLocalPorts: creationSpec.publishLocalPorts, + publishLocalPorts: specification.serverless.enabled, env, - startCommand: creationSpec.startCommand, - healthCheck: creationSpec.healthCheck, + startCommand: specification.startCommand, + healthCheck: specification.healthCheck, volumes, - resourceCpuLimit: creationSpec.resourceCpuLimit, - resourceMemoryLimitMb: creationSpec.resourceMemoryLimitMb, + resourceCpuLimit: specification.resourceLimits.cpuCores, + resourceMemoryLimitMb: specification.resourceLimits.memoryMb, }, ]; }); @@ -882,23 +786,6 @@ function buildEnv(secretRows: ServiceRevisionSecret[]) { return env; } -function hashContainerCreationSpec(specification: { - image: string; - ipAddress: string | null; - ports: ExpectedContainer["ports"]; - publishLocalPorts: boolean; - env: Record; - startCommand: string | null; - healthCheck: ServiceRevisionSpec["healthCheck"]; - volumes: ExpectedContainer["volumes"]; - resourceCpuLimit: number | null; - resourceMemoryLimitMb: number | null; -}) { - return createHash("sha256") - .update(JSON.stringify(specification)) - .digest("hex"); -} - function upstreamUrls(deployments: RoutableDeploymentRow[], port: number) { return deployments .map((d) => d.ipAddress) diff --git a/web/lib/cli-service.ts b/web/lib/cli-service.ts index 9929ad7..a3191c1 100644 --- a/web/lib/cli-service.ts +++ b/web/lib/cli-service.ts @@ -716,7 +716,7 @@ export async function deployManifest(manifest: TechulusManifest) { manifest.service.replicas.count, ); - const result = await deployServiceInternal(service.id, { trigger: "cli" }); + const result = await deployServiceInternal(service.id); return { serviceId: service.id, diff --git a/web/lib/deploy-service.ts b/web/lib/deploy-service.ts index 98ebc74..aa420c2 100644 --- a/web/lib/deploy-service.ts +++ b/web/lib/deploy-service.ts @@ -6,13 +6,9 @@ import { rollouts, serviceReplicas } from "@/db/schema"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { startMigrationInternal } from "@/lib/migrations"; -import type { RevisionSourceMetadata } from "@/lib/service-revisions"; import { createRolloutWithServiceRevision } from "@/lib/service-revisions"; -export async function deployServiceInternal( - serviceId: string, - sourceMetadata: RevisionSourceMetadata = {}, -) { +export async function deployServiceInternal(serviceId: string) { const service = await getService(serviceId); if (!service) { throw new Error("Service not found"); @@ -52,10 +48,7 @@ export async function deployServiceInternal( } } - const { rolloutId } = await createRolloutWithServiceRevision( - serviceId, - sourceMetadata, - ); + const { rolloutId } = await createRolloutWithServiceRevision(serviceId); try { await inngest.send( diff --git a/web/lib/deployment-status.ts b/web/lib/deployment-status.ts index 165144c..efbbfb9 100644 --- a/web/lib/deployment-status.ts +++ b/web/lib/deployment-status.ts @@ -11,19 +11,14 @@ export type DeploymentState = { observedPhase: ObservedPhase; }; -export const runtimeExpectedStates = [ - "running", - "stopped", -] as const satisfies readonly RuntimeDesiredState[]; +export const runtimeExpectedStates = ["running", "stopped"] as const satisfies + readonly RuntimeDesiredState[]; -export const activeTrafficStates = [ - "active", -] as const satisfies readonly TrafficState[]; +export const activeTrafficStates = ["active"] as const satisfies + readonly TrafficState[]; -export const observedReadyPhases = [ - "healthy", - "running", -] as const satisfies readonly ObservedPhase[]; +export const observedReadyPhases = ["healthy", "running"] as const satisfies + readonly ObservedPhase[]; export const observedStartingPhases = [ "pending", @@ -92,27 +87,3 @@ export function markDeploymentFailedRemoved(failedStage: string) { failedStage, }; } - -export function selectNewestRevisionId( - revisions: Array<{ serviceRevisionId: string; revisionNumber: number }>, -): string | null { - const revisionNumbers = new Map(); - for (const revision of revisions) { - revisionNumbers.set( - revision.serviceRevisionId, - Math.max( - revisionNumbers.get(revision.serviceRevisionId) ?? 0, - revision.revisionNumber, - ), - ); - } - - return ( - [...revisionNumbers] - .sort( - ([firstId, firstNumber], [secondId, secondNumber]) => - secondNumber - firstNumber || secondId.localeCompare(firstId), - ) - .at(0)?.[0] ?? null - ); -} diff --git a/web/lib/inngest/functions/build-workflow.ts b/web/lib/inngest/functions/build-workflow.ts index 34c4dd8..b66ebb0 100644 --- a/web/lib/inngest/functions/build-workflow.ts +++ b/web/lib/inngest/functions/build-workflow.ts @@ -78,15 +78,12 @@ export const buildWorkflow = inngest.createFunction( return { status: "failed", reason: result.data.error, buildId }; } - const manifestAlreadyCompleted = await step.run( - "check-existing-manifest", - async () => { - return hasCompletedManifestWorkItem({ - serviceId, - buildId, - }); - }, - ); + const manifestAlreadyCompleted = await step.run("check-existing-manifest", async () => { + return hasCompletedManifestWorkItem({ + serviceId, + buildId, + }); + }); if (!manifestAlreadyCompleted) { const manifestResult = await step.waitForEvent("wait-manifest", { @@ -117,7 +114,7 @@ export const buildWorkflow = inngest.createFunction( if (shouldDeploy) { await step.run("trigger-deploy", async () => { - await deployServiceInternal(serviceId, { trigger: "build", buildId }); + await deployServiceInternal(serviceId); }); } @@ -166,15 +163,12 @@ export const buildWorkflow = inngest.createFunction( return { status: "failed", reason: "build_failed", buildGroupId }; } - const manifestAlreadyCompleted = await step.run( - "check-existing-group-manifest", - async () => { - return hasCompletedManifestWorkItem({ - serviceId, - buildGroupId, - }); - }, - ); + const manifestAlreadyCompleted = await step.run("check-existing-group-manifest", async () => { + return hasCompletedManifestWorkItem({ + serviceId, + buildGroupId, + }); + }); if (!manifestAlreadyCompleted) { const manifestResult = await step.waitForEvent("wait-group-manifest", { @@ -205,10 +199,7 @@ export const buildWorkflow = inngest.createFunction( if (shouldDeploy) { await step.run("trigger-deploy-group", async () => { - await deployServiceInternal(serviceId, { - trigger: "build_group", - buildGroupId, - }); + await deployServiceInternal(serviceId); }); } diff --git a/web/lib/inngest/functions/migration-workflow.ts b/web/lib/inngest/functions/migration-workflow.ts index 4ab5482..bdc0327 100644 --- a/web/lib/inngest/functions/migration-workflow.ts +++ b/web/lib/inngest/functions/migration-workflow.ts @@ -115,61 +115,59 @@ export const migrationWorkflow = inngest.createFunction( const backupResults = await Promise.all( backupIds.map((backupId) => - group.parallel( - async (): Promise<{ - status: "completed" | "failed" | "pending" | "timed_out"; - error?: string; - }> => { - const readBackup = async () => - db - .select({ - status: volumeBackups.status, - errorMessage: volumeBackups.errorMessage, - }) - .from(volumeBackups) - .where(eq(volumeBackups.id, backupId)) - .then((r) => r[0]); - - const before = await step.run( - `check-backup-${backupId}-before`, - readBackup, - ); - if (before?.status === "completed") { - return { status: "completed" as const }; - } - if (before?.status === "failed") { - return { - status: "failed" as const, - error: before.errorMessage || "Backup failed", - }; - } - - const wakeup = await step.waitForEvent( - `wait-backup-status-${backupId}`, - { - event: inngestEvents.resourceStatusChanged, - timeout: "30m", - if: `async.data.type == "backup" && async.data.id == "${backupId}"`, - }, - ); - - const after = await step.run( - `check-backup-${backupId}-after`, - readBackup, - ); - if (after?.status === "completed") { - return { status: "completed" as const }; - } - if (after?.status === "failed") { - return { - status: "failed" as const, - error: after.errorMessage || "Backup failed", - }; - } - - return { status: wakeup ? "pending" : "timed_out" } as const; - }, - ), + group.parallel(async (): Promise<{ + status: "completed" | "failed" | "pending" | "timed_out"; + error?: string; + }> => { + const readBackup = async () => + db + .select({ + status: volumeBackups.status, + errorMessage: volumeBackups.errorMessage, + }) + .from(volumeBackups) + .where(eq(volumeBackups.id, backupId)) + .then((r) => r[0]); + + const before = await step.run( + `check-backup-${backupId}-before`, + readBackup, + ); + if (before?.status === "completed") { + return { status: "completed" as const }; + } + if (before?.status === "failed") { + return { + status: "failed" as const, + error: before.errorMessage || "Backup failed", + }; + } + + const wakeup = await step.waitForEvent( + `wait-backup-status-${backupId}`, + { + event: inngestEvents.resourceStatusChanged, + timeout: "30m", + if: `async.data.type == "backup" && async.data.id == "${backupId}"`, + }, + ); + + const after = await step.run( + `check-backup-${backupId}-after`, + readBackup, + ); + if (after?.status === "completed") { + return { status: "completed" as const }; + } + if (after?.status === "failed") { + return { + status: "failed" as const, + error: after.errorMessage || "Backup failed", + }; + } + + return { status: wakeup ? "pending" : "timed_out" } as const; + }), ), ); @@ -187,9 +185,7 @@ export const migrationWorkflow = inngest.createFunction( return { status: "failed", reason: "backup_timeout" }; } - const backupStillPending = backupResults.some( - (r) => r.status === "pending", - ); + const backupStillPending = backupResults.some((r) => r.status === "pending"); if (backupStillPending) { await step.run("handle-backup-still-pending", async () => { await db @@ -291,7 +287,8 @@ export const migrationWorkflow = inngest.createFunction( .update(services) .set({ migrationStatus: "failed", - migrationError: restoreFailure.data.error || "Restore failed", + migrationError: + restoreFailure.data.error || "Restore failed", }) .where(eq(services.id, serviceId)); }); @@ -320,10 +317,7 @@ export const migrationWorkflow = inngest.createFunction( .set({ lockedServerId: targetServerId }) .where(eq(services.id, serviceId)); - await deployServiceInternal(serviceId, { - trigger: "migration", - targetServerId, - }); + await deployServiceInternal(serviceId); }); await step.run("finalize-migration", async () => { diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 99488e7..3943b9f 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -1,7 +1,6 @@ import { randomUUID } from "node:crypto"; -import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; -import { getService } from "@/db/queries"; import { deploymentPorts, deployments, @@ -11,7 +10,7 @@ import { } from "@/db/schema"; import { getCertificate, issueCertificate } from "@/lib/acme-manager"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; -import { findAvailableContainerIp } from "@/lib/wireguard"; +import { assignContainerIp } from "@/lib/wireguard"; import { enqueueWork } from "@/lib/work-queue"; const PORT_RANGE_START = 30000; @@ -31,12 +30,6 @@ export type DeploymentContext = { isRollingUpdate: boolean; }; -export function isActiveDeploymentForRollout(deployment: { - trafficState: string; -}) { - return deployment.trafficState === "active"; -} - export function normalizeImage(image: string): string { if (!image.includes("/")) { return `docker.io/library/${image}`; @@ -47,11 +40,21 @@ export function normalizeImage(image: string): string { return image; } -export function findAvailableHostPorts( - usedPorts: Iterable, +async function getUsedPorts(serverId: string): Promise> { + const existingPorts = await db + .select({ hostPort: deploymentPorts.hostPort }) + .from(deploymentPorts) + .innerJoin(deployments, eq(deploymentPorts.deploymentId, deployments.id)) + .where(eq(deployments.serverId, serverId)); + + return new Set(existingPorts.map((port) => port.hostPort)); +} + +export async function allocateHostPorts( + serverId: string, count: number, -): number[] { - const unavailablePorts = new Set(usedPorts); +): Promise { + const unavailablePorts = await getUsedPorts(serverId); const allocated: number[] = []; for ( @@ -144,26 +147,6 @@ export async function validateServers( return serverMap; } -export async function prepareRollingUpdate( - serviceId: string, -): Promise<{ deploymentIds: string[] }> { - const service = await getService(serviceId); - if (!service) { - return { deploymentIds: [] }; - } - - const existingDeployments = await db - .select() - .from(deployments) - .where(eq(deployments.serviceId, serviceId)); - - const runningDeployments = existingDeployments.filter((d) => - isActiveDeploymentForRollout(d), - ); - - return { deploymentIds: runningDeployments.map((d) => d.id) }; -} - export async function cleanupTerminalDeployments( serviceId: string, ): Promise { @@ -264,78 +247,13 @@ export async function createDeploymentRecords( for (let i = 0; i < placement.replicas; i++) { const deploymentId = randomUUID(); + const hostPorts = await allocateHostPorts( + server.id, + specification.ports.length, + ); + const ipAddress = await assignContainerIp(server.id); await db.transaction(async (tx) => { - await tx.execute( - sql`SELECT pg_advisory_xact_lock(hashtext(${`deployment-allocation:${server.id}`}))`, - ); - - if (specification.stateful) { - const lockedService = await tx - .select({ lockedServerId: services.lockedServerId }) - .from(services) - .where(eq(services.id, serviceId)) - .for("update") - .then((rows) => rows[0]); - if (!lockedService) { - throw new Error("Service not found"); - } - if ( - lockedService.lockedServerId && - lockedService.lockedServerId !== server.id - ) { - throw new Error( - `Stateful service is locked to server ${lockedService.lockedServerId}`, - ); - } - if (!lockedService.lockedServerId) { - await tx - .update(services) - .set({ lockedServerId: server.id }) - .where(eq(services.id, serviceId)); - } - } - - const [usedPortRows, subnet, usedIpRows] = await Promise.all([ - tx - .select({ hostPort: deploymentPorts.hostPort }) - .from(deploymentPorts) - .innerJoin( - deployments, - eq(deploymentPorts.deploymentId, deployments.id), - ) - .where(eq(deployments.serverId, server.id)), - tx - .select({ subnetId: servers.subnetId }) - .from(servers) - .where(eq(servers.id, server.id)) - .then((rows) => rows[0]), - tx - .select({ ipAddress: deployments.ipAddress }) - .from(deployments) - .where( - and( - eq(deployments.serverId, server.id), - isNotNull(deployments.ipAddress), - ), - ), - ]); - - if (!subnet?.subnetId) { - throw new Error(`Server ${server.name} has no subnet assigned`); - } - - const hostPorts = findAvailableHostPorts( - usedPortRows.map((port) => port.hostPort), - specification.ports.length, - ); - const ipAddress = findAvailableContainerIp( - subnet.subnetId, - usedIpRows.flatMap((deployment) => - deployment.ipAddress ? [deployment.ipAddress] : [], - ), - ); - await tx.insert(deployments).values({ id: deploymentId, serviceId, @@ -365,7 +283,6 @@ export async function createDeploymentRecords( await enqueueWork(server.id, "reconcile", { reason: "rollout_deployment_created", deploymentId, - revisionId, }); } } @@ -373,18 +290,12 @@ export async function createDeploymentRecords( return { deploymentIds }; } -export async function completeRolloutWithRevision( +export async function completeRollout( rolloutId: string, serviceId: string, - context: Omit, + context: Omit, ): Promise<{ completed: boolean; stoppedCount: number }> { - const { - placements, - revisionId, - specification, - totalReplicas, - isRollingUpdate, - } = context; + const { placements, specification, totalReplicas, isRollingUpdate } = context; const lockedServerId = specification.stateful ? placements[0]?.serverId : undefined; @@ -419,7 +330,6 @@ export async function completeRolloutWithRevision( await tx .update(services) .set({ - activeRevisionId: revisionId, replicas: totalReplicas, ...(lockedServerId ? { lockedServerId } : {}), }) @@ -445,14 +355,17 @@ export async function checkForRollingUpdate( return false; } - const existingDeployments = await db - .select() + const existingDeployment = await db + .select({ id: deployments.id }) .from(deployments) - .where(eq(deployments.serviceId, serviceId)); - - const runningDeployments = existingDeployments.filter((d) => - isActiveDeploymentForRollout(d), - ); + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "active"), + ), + ) + .limit(1) + .then((rows) => rows[0]); - return runningDeployments.length > 0; + return existingDeployment != null; } diff --git a/web/lib/inngest/functions/rollout-utils.ts b/web/lib/inngest/functions/rollout-utils.ts index 55abead..30f4432 100644 --- a/web/lib/inngest/functions/rollout-utils.ts +++ b/web/lib/inngest/functions/rollout-utils.ts @@ -1,113 +1,42 @@ import { and, eq, ne } from "drizzle-orm"; import { db } from "@/db"; -import { deployments, rollouts, serviceRevisions, services } from "@/db/schema"; -import { - markDeploymentFailedRemoved, - selectNewestRevisionId, -} from "@/lib/deployment-status"; +import { deployments, rollouts } from "@/db/schema"; +import { markDeploymentFailedRemoved } from "@/lib/deployment-status"; import { sendDeploymentFailureAlert } from "@/lib/email"; +export async function restoreDrainingDeploymentsForRollback(serviceId: string) { + await db + .update(deployments) + .set({ trafficState: "active" }) + .where( + and( + eq(deployments.serviceId, serviceId), + eq(deployments.trafficState, "draining"), + ), + ); +} + export async function handleRolloutFailure( rolloutId: string, serviceId: string, reason: string, isRollingUpdate: boolean, ): Promise { - let serverId: string | null = null; - let hasRolloutDeployments = false; - let handled = false; - - await db.transaction(async (tx) => { - const rollout = await tx - .select({ status: rollouts.status }) - .from(rollouts) - .where(eq(rollouts.id, rolloutId)) - .for("update") - .then((rows) => rows[0]); - - if ( - !rollout || - (rollout.status !== "queued" && rollout.status !== "in_progress") - ) { - return; - } - handled = true; - - const rolloutDeployments = await tx - .select() - .from(deployments) - .where(eq(deployments.rolloutId, rolloutId)); - - hasRolloutDeployments = rolloutDeployments.length > 0; - serverId = rolloutDeployments[0]?.serverId ?? null; - - await tx - .update(rollouts) - .set({ - status: hasRolloutDeployments ? "rolled_back" : "failed", - currentStage: reason, - completedAt: new Date(), - }) - .where(eq(rollouts.id, rolloutId)); - - if (!hasRolloutDeployments) return; - - if (isRollingUpdate) { - await tx - .update(deployments) - .set({ trafficState: "active" }) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "draining"), - ), - ); - } - - await tx - .update(deployments) - .set(markDeploymentFailedRemoved(reason)) - .where( - and( - eq(deployments.rolloutId, rolloutId), - ne(deployments.runtimeDesiredState, "removed"), - ), - ); - - const activeRevisionRows = await tx - .select({ - serviceRevisionId: deployments.serviceRevisionId, - revisionNumber: serviceRevisions.revisionNumber, - }) - .from(deployments) - .innerJoin( - serviceRevisions, - eq(deployments.serviceRevisionId, serviceRevisions.id), - ) - .where( - and( - eq(deployments.serviceId, serviceId), - eq(deployments.trafficState, "active"), - ne(deployments.runtimeDesiredState, "removed"), - ), - ); - const activeRevisionCount = new Set( - activeRevisionRows.map((row) => row.serviceRevisionId), - ).size; - if (activeRevisionCount > 1) { - console.error( - `[rollout:${rolloutId}] rollback found ${activeRevisionCount} active revisions for ${serviceId}; selecting the newest`, - ); - } - await tx - .update(services) - .set({ activeRevisionId: selectNewestRevisionId(activeRevisionRows) }) - .where(eq(services.id, serviceId)); - }); + const rolloutDeployments = await db + .select() + .from(deployments) + .where(eq(deployments.rolloutId, rolloutId)); - if (!handled) return; + await db + .update(rollouts) + .set({ + status: rolloutDeployments.length === 0 ? "failed" : "rolled_back", + currentStage: reason, + completedAt: new Date(), + }) + .where(eq(rollouts.id, rolloutId)); - if (!hasRolloutDeployments) { + if (rolloutDeployments.length === 0) { sendDeploymentFailureAlert({ serviceId, serverId: null, @@ -121,6 +50,22 @@ export async function handleRolloutFailure( return; } + const serverId = rolloutDeployments[0].serverId; + + if (isRollingUpdate) { + await restoreDrainingDeploymentsForRollback(serviceId); + } + + await db + .update(deployments) + .set(markDeploymentFailedRemoved(reason)) + .where( + and( + eq(deployments.rolloutId, rolloutId), + ne(deployments.runtimeDesiredState, "removed"), + ), + ); + sendDeploymentFailureAlert({ serviceId, serverId, diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index b056a54..ccafa19 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -1,7 +1,7 @@ import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; -import { deployments, rollouts, servers, services } from "@/db/schema"; +import { deployments, rollouts, servers } from "@/db/schema"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; import { getRolloutServiceRevision } from "@/lib/service-revisions"; import { ingestRolloutLog } from "@/lib/victoria-logs"; @@ -12,10 +12,9 @@ import { checkForRollingUpdate, cleanupExistingDeployments, cleanupTerminalDeployments, - completeRolloutWithRevision, + completeRollout, createDeploymentRecords, issueCertificatesForRevision, - prepareRollingUpdate, validateServers, } from "./rollout-helpers"; import { handleRolloutFailure } from "./rollout-utils"; @@ -120,28 +119,25 @@ export const rolloutWorkflow = inngest.createFunction( { event: inngestEvents.rolloutCancelled, match: "data.rolloutId" }, ], onFailure: async ({ event }) => { - const { rolloutId, serviceId } = event.data.event.data as { + const { rolloutId } = event.data.event.data as { rolloutId?: string; - serviceId?: string; }; if (!rolloutId) return; - const targetServiceId = - serviceId ?? - (await db - .select({ serviceId: rollouts.serviceId }) - .from(rollouts) - .where(eq(rollouts.id, rolloutId)) - .then((rows) => rows[0]?.serviceId)); - if (!targetServiceId) return; - - await handleRolloutFailure( - rolloutId, - targetServiceId, - "workflow_failed", - true, - ); + await db + .update(rollouts) + .set({ + status: "failed", + currentStage: "workflow_failed", + completedAt: new Date(), + }) + .where( + and( + eq(rollouts.id, rolloutId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ); }, }, async ({ event, step }) => { @@ -200,16 +196,12 @@ export const rolloutWorkflow = inngest.createFunction( const revision = await step.run("load-service-revision", async () => { const rolloutRevision = await getRolloutServiceRevision(rolloutId); - if (rolloutRevision.serviceId !== serviceId) { - throw new Error("Rollout revision does not belong to service"); - } const executionSpecification: ServiceRevisionSpec = { ...rolloutRevision.specification, secrets: [], }; return { id: rolloutRevision.id, - serviceId: rolloutRevision.serviceId, specification: executionSpecification, }; }); @@ -307,17 +299,7 @@ export const rolloutWorkflow = inngest.createFunction( return checkForRollingUpdate(serviceId, specification); }); - if (isRollingUpdate) { - await step.run("prepare-rolling-update", async () => { - await prepareRollingUpdate(serviceId); - await ingestRolloutLog( - rolloutId, - serviceId, - "preparing", - "Prepared rolling update", - ); - }); - } else { + if (!isRollingUpdate) { await step.run("cleanup-existing", async () => { const { deletedCount } = await cleanupExistingDeployments(serviceId); if (deletedCount > 0) { @@ -373,15 +355,6 @@ export const rolloutWorkflow = inngest.createFunction( } const { deploymentIds } = await step.run("create-deployments", async () => { - await db - .delete(deployments) - .where( - and( - eq(deployments.rolloutId, rolloutId), - eq(deployments.trafficState, "candidate"), - ), - ); - await db .update(rollouts) .set({ currentStage: "deploying" }) @@ -510,19 +483,8 @@ export const rolloutWorkflow = inngest.createFunction( }; } - const trafficPromoted = await step.run("start-dns-sync", async () => { - const promoted = await db.transaction(async (tx) => { - const activeRollout = await tx - .select({ status: rollouts.status }) - .from(rollouts) - .where(eq(rollouts.id, rolloutId)) - .for("update") - .then((rows) => rows[0]); - - if (activeRollout?.status !== "in_progress") { - return false; - } - + await step.run("start-dns-sync", async () => { + await db.transaction(async (tx) => { await tx .update(rollouts) .set({ currentStage: "dns_sync" }) @@ -552,15 +514,7 @@ export const rolloutWorkflow = inngest.createFunction( ), ), ); - - await tx - .update(services) - .set({ activeRevisionId: revision.id }) - .where(eq(services.id, serviceId)); - - return true; }); - if (!promoted) return false; await ingestRolloutLog( rolloutId, @@ -568,13 +522,8 @@ export const rolloutWorkflow = inngest.createFunction( "dns_sync", "Routing traffic to new deployments", ); - return true; }); - if (!trafficPromoted) { - return { status: "cancelled", rolloutId }; - } - const dnsResults = await Promise.all( serverIds.map((serverId) => step.waitForEvent(`wait-dns-${serverId}`, { @@ -630,8 +579,7 @@ export const rolloutWorkflow = inngest.createFunction( } const rolloutCompleted = await step.run("complete-rollout", async () => { - const result = await completeRolloutWithRevision(rolloutId, serviceId, { - revisionId: revision.id, + const result = await completeRollout(rolloutId, serviceId, { specification, placements, totalReplicas, diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index dd39d58..640e533 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -5,8 +5,8 @@ import { deleteBackup } from "@/actions/backups"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { + deploymentPorts, deployments, - rollouts, secrets, services, serviceVolumes, @@ -14,10 +14,7 @@ import { } from "@/db/schema"; import { addUtcDays, toDate } from "@/lib/date"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { - markDeploymentFailedRemoved, - markDeploymentRemoved, -} from "@/lib/deployment-status"; +import { markDeploymentRemoved } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -244,6 +241,10 @@ export const serviceDeletionWorkflow = inngest.createFunction( containerId: deployment.containerId, }); } + + await db + .delete(deploymentPorts) + .where(eq(deploymentPorts.deploymentId, deployment.id)); } await db @@ -420,9 +421,7 @@ export const serviceRestoreWorkflow = inngest.createFunction( .where(eq(services.id, serviceId)); try { - const result = await deployServiceInternal(serviceId, { - trigger: "restore", - }); + const result = await deployServiceInternal(serviceId); if (!("rolloutId" in result) || !result.rolloutId) { throw new Error("Restore could not start a deployment"); } @@ -484,33 +483,19 @@ export const serviceRestoreWorkflow = inngest.createFunction( if (!healthyDeployment || failedDeployment) { await step.run("mark-restore-deployment-failed", async () => { - await db.transaction(async (tx) => { - await tx - .update(deployments) - .set(markDeploymentFailedRemoved("restore_failed")) - .where(eq(deployments.rolloutId, deployResult.rolloutId)); - await tx - .update(rollouts) - .set({ - status: "failed", - currentStage: "restore_failed", - completedAt: new Date(), - }) - .where(eq(rollouts.id, deployResult.rolloutId)); - await tx - .update(services) - .set({ - deletedAt: toDate(setup.service.deletedAt), - purgeAfter: toDate(setup.service.purgeAfter), - hostname: null, - originalHostname: setup.service.originalHostname, - deletionStatus: "failed", - deletionError: - failedDeployment?.failedStage || - "Restore deployment did not become healthy", - }) - .where(eq(services.id, serviceId)); - }); + await db + .update(services) + .set({ + deletedAt: toDate(setup.service.deletedAt), + purgeAfter: toDate(setup.service.purgeAfter), + hostname: null, + originalHostname: setup.service.originalHostname, + deletionStatus: "failed", + deletionError: + failedDeployment?.failedStage || + "Restore deployment did not become healthy", + }) + .where(eq(services.id, serviceId)); }); return { status: "failed", reason: "deployment" }; } diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index aba878a..700f623 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -216,7 +216,7 @@ export async function checkAndRunScheduledDeployments(): Promise { if (service.sourceType === "github") { await triggerBuild(service.id, "scheduled"); } else { - await deployServiceInternal(service.id, { trigger: "scheduled" }); + await deployServiceInternal(service.id); } console.log( diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index cd52d80..ece73e8 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -1,7 +1,4 @@ -import { - getDefaultServiceHostname, - type ServiceRevisionSpec, -} from "@/lib/service-revision-spec"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; export type ReplicaConfig = { serverId: string; @@ -32,8 +29,7 @@ export type SourceConfig = { export type SecretConfig = { key: string; - updatedAt?: string; - fingerprint?: string; + updatedAt: string; }; export type VolumeConfig = { @@ -84,7 +80,6 @@ export type ConfigChange = { export function buildCurrentConfig( service: { - name: string; image: string; hostname: string | null; healthCheckCmd: string | null; @@ -109,11 +104,7 @@ export function buildCurrentConfig( protocol?: "http" | "tcp" | "udp" | null; tlsPassthrough?: boolean | null; }[], - secrets?: { - key: string; - updatedAt: Date | string; - fingerprint?: string; - }[], + secrets?: { key: string; updatedAt: Date | string }[], volumes?: { name: string; containerPath: string }[], ): DeployedConfig { const hasResourceLimits = @@ -124,7 +115,7 @@ export function buildCurrentConfig( type: "image", image: service.image, }, - hostname: service.hostname ?? getDefaultServiceHostname(service.name), + hostname: service.hostname ?? undefined, stateful: service.stateful ?? false, placement: { replicas: replicas.reduce((sum, r) => sum + r.count, 0), @@ -162,7 +153,6 @@ export function buildCurrentConfig( key: s.key, updatedAt: s.updatedAt instanceof Date ? s.updatedAt.toISOString() : s.updatedAt, - fingerprint: s.fingerprint, })), volumes: (volumes ?? []).map((v) => ({ name: v.name, @@ -493,11 +483,7 @@ export function diffConfigs( from: "(none)", to: key, }); - } else if ( - deployedSecret.fingerprint && currentSecret.fingerprint - ? deployedSecret.fingerprint !== currentSecret.fingerprint - : deployedSecret.updatedAt !== currentSecret.updatedAt - ) { + } else if (deployedSecret.updatedAt !== currentSecret.updatedAt) { changes.push({ field: "Secret", from: key, @@ -553,10 +539,41 @@ export function diffConfigs( return changes; } +export function normalizeServerlessConfig( + config: ServerlessConfig | undefined, +): ServerlessConfig { + return { + enabled: config?.enabled ?? false, + sleepAfterSeconds: Math.max( + config?.sleepAfterSeconds ?? DEFAULT_SERVERLESS_SLEEP_AFTER_SECONDS, + MIN_SERVERLESS_SLEEP_AFTER_SECONDS, + ), + wakeTimeoutSeconds: + config?.wakeTimeoutSeconds ?? DEFAULT_SERVERLESS_WAKE_TIMEOUT_SECONDS, + }; +} + +export function getCurrentServerlessConfig(service: { + serverlessEnabled?: boolean | null; + serverlessSleepAfterSeconds?: number | null; + serverlessWakeTimeoutSeconds?: number | null; +}): ServerlessConfig { + return { + enabled: service.serverlessEnabled ?? false, + sleepAfterSeconds: Math.max( + service.serverlessSleepAfterSeconds ?? + DEFAULT_SERVERLESS_SLEEP_AFTER_SECONDS, + MIN_SERVERLESS_SLEEP_AFTER_SECONDS, + ), + wakeTimeoutSeconds: + service.serverlessWakeTimeoutSeconds ?? + DEFAULT_SERVERLESS_WAKE_TIMEOUT_SECONDS, + }; +} + export function revisionSpecToDeployedConfig( specification: ServiceRevisionSpec, serverNames: Record, - secretFingerprints: Record, ): DeployedConfig { return { source: { type: "image", image: specification.image }, @@ -564,25 +581,21 @@ export function revisionSpecToDeployedConfig( stateful: specification.stateful, placement: { replicas: specification.placements.reduce( - (total, placement) => total + placement.count, + (sum, placement) => sum + placement.count, 0, ), }, replicas: specification.placements.map((placement) => ({ serverId: placement.serverId, - serverName: serverNames[placement.serverId] ?? placement.serverId, + serverName: serverNames[placement.serverId] ?? "Unknown", count: placement.count, })), healthCheck: specification.healthCheck, startCommand: specification.startCommand, - resourceLimits: - specification.resourceLimits.cpuCores != null || - specification.resourceLimits.memoryMb != null - ? { - cpuCores: specification.resourceLimits.cpuCores, - memoryMb: specification.resourceLimits.memoryMb, - } - : undefined, + resourceLimits: { + cpuCores: specification.resourceLimits.cpuCores, + memoryMb: specification.resourceLimits.memoryMb, + }, ports: specification.ports.map((port) => ({ port: port.containerPort, isPublic: port.isPublic, @@ -593,40 +606,8 @@ export function revisionSpecToDeployedConfig( serverless: specification.serverless, secrets: specification.secrets.map((secret) => ({ key: secret.key, - fingerprint: secretFingerprints[secret.key], + updatedAt: secret.updatedAt, })), volumes: specification.volumes, }; } - -export function normalizeServerlessConfig( - config: ServerlessConfig | undefined, -): ServerlessConfig { - return { - enabled: config?.enabled ?? false, - sleepAfterSeconds: Math.max( - config?.sleepAfterSeconds ?? DEFAULT_SERVERLESS_SLEEP_AFTER_SECONDS, - MIN_SERVERLESS_SLEEP_AFTER_SECONDS, - ), - wakeTimeoutSeconds: - config?.wakeTimeoutSeconds ?? DEFAULT_SERVERLESS_WAKE_TIMEOUT_SECONDS, - }; -} - -export function getCurrentServerlessConfig(service: { - serverlessEnabled?: boolean | null; - serverlessSleepAfterSeconds?: number | null; - serverlessWakeTimeoutSeconds?: number | null; -}): ServerlessConfig { - return { - enabled: service.serverlessEnabled ?? false, - sleepAfterSeconds: Math.max( - service.serverlessSleepAfterSeconds ?? - DEFAULT_SERVERLESS_SLEEP_AFTER_SECONDS, - MIN_SERVERLESS_SLEEP_AFTER_SECONDS, - ), - wakeTimeoutSeconds: - service.serverlessWakeTimeoutSeconds ?? - DEFAULT_SERVERLESS_WAKE_TIMEOUT_SECONDS, - }; -} diff --git a/web/lib/service-revision-cutover.ts b/web/lib/service-revision-cutover.ts index cdbeb42..f76d653 100644 --- a/web/lib/service-revision-cutover.ts +++ b/web/lib/service-revision-cutover.ts @@ -2,43 +2,9 @@ import { buildServiceRevisionSpec, type ServiceRevisionDraft, } from "./service-revision-spec"; +import type { DeployedConfig } from "./service-config"; -export type CutoverDeployedConfig = { - source?: { image?: string }; - hostname?: string | null; - stateful?: boolean; - replicas?: Array<{ serverId: string; count: number }>; - healthCheck?: { - cmd: string; - interval: number; - timeout: number; - retries: number; - startPeriod: number; - } | null; - startCommand?: string | null; - resourceLimits?: { - cpuCores?: number | null; - memoryMb?: number | null; - }; - ports?: Array<{ - port: number; - isPublic: boolean; - domain: string | null; - protocol?: "http" | "tcp" | "udp"; - tlsPassthrough?: boolean; - }>; - serverless?: { - enabled: boolean; - sleepAfterSeconds: number; - wakeTimeoutSeconds: number; - }; - secrets?: Array<{ - key: string; - updatedAt?: string; - }>; - secretKeys?: string[]; - volumes?: Array<{ name: string; containerPath: string }>; -}; +export type CutoverDeployedConfig = DeployedConfig; function assertSecretsMatchDeployedSnapshot( liveSecrets: ServiceRevisionDraft["secrets"], @@ -85,11 +51,9 @@ export function buildCutoverServiceRevisionSpec({ deployedConfig, }: { liveDraft: ServiceRevisionDraft; - deployedConfig: CutoverDeployedConfig | null; + deployedConfig: CutoverDeployedConfig; }) { - if (deployedConfig) { - assertSecretsMatchDeployedSnapshot(liveDraft.secrets, deployedConfig); - } + assertSecretsMatchDeployedSnapshot(liveDraft.secrets, deployedConfig); const currentPortsByIdentity = new Map( liveDraft.ports.map((port) => [ @@ -97,71 +61,49 @@ export function buildCutoverServiceRevisionSpec({ port, ]), ); - const deployedPorts = Array.isArray(deployedConfig?.ports) - ? deployedConfig.ports.map((port) => { - const currentPort = currentPortsByIdentity.get( - `${port.port}:${port.protocol ?? "http"}`, - ); - return { - port: port.port, - isPublic: port.isPublic, - domain: port.domain, - protocol: port.protocol ?? null, - externalPort: currentPort?.externalPort ?? null, - tlsPassthrough: port.tlsPassthrough ?? null, - }; - }) - : null; - const deployedHealthCheck = deployedConfig - ? (deployedConfig.healthCheck ?? null) - : undefined; + const deployedPorts = deployedConfig.ports.map((port) => { + const currentPort = currentPortsByIdentity.get( + `${port.port}:${port.protocol ?? "http"}`, + ); + return { + port: port.port, + isPublic: port.isPublic, + domain: port.domain, + protocol: port.protocol ?? null, + externalPort: currentPort?.externalPort ?? null, + tlsPassthrough: port.tlsPassthrough ?? null, + }; + }); + const deployedHealthCheck = deployedConfig.healthCheck ?? null; return buildServiceRevisionSpec({ service: { ...liveDraft.service, - image: deployedConfig?.source?.image ?? liveDraft.service.image, - hostname: deployedConfig?.hostname ?? liveDraft.service.hostname, - stateful: deployedConfig?.stateful ?? liveDraft.service.stateful, + image: deployedConfig.source.image, + hostname: deployedConfig.hostname ?? liveDraft.service.hostname, + stateful: deployedConfig.stateful ?? liveDraft.service.stateful, serverlessEnabled: - deployedConfig?.serverless?.enabled ?? + deployedConfig.serverless?.enabled ?? liveDraft.service.serverlessEnabled, serverlessSleepAfterSeconds: - deployedConfig?.serverless?.sleepAfterSeconds ?? + deployedConfig.serverless?.sleepAfterSeconds ?? liveDraft.service.serverlessSleepAfterSeconds, serverlessWakeTimeoutSeconds: - deployedConfig?.serverless?.wakeTimeoutSeconds ?? + deployedConfig.serverless?.wakeTimeoutSeconds ?? liveDraft.service.serverlessWakeTimeoutSeconds, - healthCheckCmd: deployedConfig - ? (deployedHealthCheck?.cmd ?? null) - : liveDraft.service.healthCheckCmd, - healthCheckInterval: deployedConfig - ? (deployedHealthCheck?.interval ?? null) - : liveDraft.service.healthCheckInterval, - healthCheckTimeout: deployedConfig - ? (deployedHealthCheck?.timeout ?? null) - : liveDraft.service.healthCheckTimeout, - healthCheckRetries: deployedConfig - ? (deployedHealthCheck?.retries ?? null) - : liveDraft.service.healthCheckRetries, - healthCheckStartPeriod: deployedConfig - ? (deployedHealthCheck?.startPeriod ?? null) - : liveDraft.service.healthCheckStartPeriod, - startCommand: deployedConfig - ? (deployedConfig.startCommand ?? null) - : liveDraft.service.startCommand, - resourceCpuLimit: deployedConfig - ? (deployedConfig.resourceLimits?.cpuCores ?? null) - : liveDraft.service.resourceCpuLimit, - resourceMemoryLimitMb: deployedConfig - ? (deployedConfig.resourceLimits?.memoryMb ?? null) - : liveDraft.service.resourceMemoryLimitMb, + healthCheckCmd: deployedHealthCheck?.cmd ?? null, + healthCheckInterval: deployedHealthCheck?.interval ?? null, + healthCheckTimeout: deployedHealthCheck?.timeout ?? null, + healthCheckRetries: deployedHealthCheck?.retries ?? null, + healthCheckStartPeriod: deployedHealthCheck?.startPeriod ?? null, + startCommand: deployedConfig.startCommand ?? null, + resourceCpuLimit: deployedConfig.resourceLimits?.cpuCores ?? null, + resourceMemoryLimitMb: deployedConfig.resourceLimits?.memoryMb ?? null, }, - placements: Array.isArray(deployedConfig?.replicas) - ? deployedConfig.replicas - : liveDraft.placements, - ports: deployedPorts ?? liveDraft.ports, + placements: deployedConfig.replicas, + ports: deployedPorts, secrets: liveDraft.secrets, - volumes: Array.isArray(deployedConfig?.volumes) + volumes: Array.isArray(deployedConfig.volumes) ? deployedConfig.volumes : liveDraft.volumes, }); diff --git a/web/lib/service-revision-spec.ts b/web/lib/service-revision-spec.ts index dd23ed6..a8f7aee 100644 --- a/web/lib/service-revision-spec.ts +++ b/web/lib/service-revision-spec.ts @@ -1,5 +1,3 @@ -import { createHash } from "node:crypto"; - export const SERVICE_REVISION_SCHEMA_VERSION = 1 as const; export function getDefaultServiceHostname(name: string): string { @@ -34,6 +32,7 @@ export type ServiceRevisionPort = { export type ServiceRevisionSecret = { key: string; encryptedValue: string; + updatedAt: string; }; export type ServiceRevisionVolume = { @@ -43,7 +42,6 @@ export type ServiceRevisionVolume = { export type ServiceRevisionSpec = { schemaVersion: typeof SERVICE_REVISION_SCHEMA_VERSION; - serviceId: string; image: string; hostname: string; stateful: boolean; @@ -66,7 +64,6 @@ export type ServiceRevisionSpec = { export type ServiceRevisionDraft = { service: { - id: string; name: string; image: string; hostname: string | null; @@ -95,11 +92,31 @@ export type ServiceRevisionDraft = { secrets: Array<{ key: string; encryptedValue: string; - updatedAt?: Date | string; + updatedAt: Date | string; }>; volumes: Array<{ name: string; containerPath: string }>; }; +function validateServiceRevisionSpec(specification: ServiceRevisionSpec) { + const totalReplicas = specification.placements.reduce( + (sum, placement) => sum + placement.count, + 0, + ); + + if (totalReplicas < 1) { + throw new Error("At least one replica is required"); + } + if (totalReplicas > 10) { + throw new Error("Maximum 10 replicas allowed"); + } + if (specification.stateful && totalReplicas !== 1) { + throw new Error("Stateful services can only have exactly 1 replica"); + } + if (specification.stateful && specification.placements.length !== 1) { + throw new Error("Stateful services must be deployed to exactly one server"); + } +} + function compareStrings(a: string, b: string) { return a.localeCompare(b, "en"); } @@ -109,9 +126,8 @@ export function buildServiceRevisionSpec( ): ServiceRevisionSpec { const { service } = draft; - return { + const specification: ServiceRevisionSpec = { schemaVersion: SERVICE_REVISION_SCHEMA_VERSION, - serviceId: service.id, image: service.image.trim(), hostname: service.hostname?.trim() || getDefaultServiceHostname(service.name), @@ -165,6 +181,10 @@ export function buildServiceRevisionSpec( .map((secret) => ({ key: secret.key, encryptedValue: secret.encryptedValue, + updatedAt: + secret.updatedAt instanceof Date + ? secret.updatedAt.toISOString() + : secret.updatedAt, })) .sort((a, b) => compareStrings(a.key, b.key)), volumes: draft.volumes @@ -178,8 +198,6 @@ export function buildServiceRevisionSpec( compareStrings(a.containerPath, b.containerPath), ), }; -} - -export function hashServiceRevisionSpec(spec: ServiceRevisionSpec): string { - return createHash("sha256").update(JSON.stringify(spec)).digest("hex"); + validateServiceRevisionSpec(specification); + return specification; } diff --git a/web/lib/service-revisions.ts b/web/lib/service-revisions.ts index 01374b4..eb43faf 100644 --- a/web/lib/service-revisions.ts +++ b/web/lib/service-revisions.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq, isNull, sql } from "drizzle-orm"; +import { and, eq, isNull } from "drizzle-orm"; import { db } from "@/db"; import { rollouts, @@ -12,152 +12,83 @@ import { } from "@/db/schema"; import { buildServiceRevisionSpec, - hashServiceRevisionSpec, SERVICE_REVISION_SCHEMA_VERSION, - type ServiceRevisionSpec, } from "@/lib/service-revision-spec"; -export type RevisionSourceMetadata = Record< - string, - string | number | boolean | null | undefined ->; - -function validateRevisionSpec(spec: ServiceRevisionSpec) { - const totalReplicas = spec.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); - - if (totalReplicas < 1) { - throw new Error("At least one replica is required"); - } - if (totalReplicas > 10) { - throw new Error("Maximum 10 replicas allowed"); - } - if (spec.stateful && totalReplicas !== 1) { - throw new Error("Stateful services can only have exactly 1 replica"); - } - if (spec.stateful && spec.placements.length !== 1) { - throw new Error("Stateful services must be deployed to exactly one server"); - } -} - -function compactSourceMetadata(metadata: RevisionSourceMetadata) { - return Object.fromEntries( - Object.entries(metadata).filter((entry) => entry[1] !== undefined), - ) as Record; -} - -export async function createRolloutWithServiceRevision( - serviceId: string, - sourceMetadata: RevisionSourceMetadata = {}, -) { - return db.transaction(async (tx) => { - await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); - - const service = await tx - .select() - .from(services) - .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) - .then((rows) => rows[0]); - - if (!service) { - throw new Error("Service not found"); - } - - const placements = await tx - .select({ - serverId: serviceReplicas.serverId, - count: serviceReplicas.count, - }) - .from(serviceReplicas) - .where(eq(serviceReplicas.serviceId, serviceId)); - const ports = await tx - .select() - .from(servicePorts) - .where(eq(servicePorts.serviceId, serviceId)); - const revisionSecrets = await tx - .select({ - key: secrets.key, - encryptedValue: secrets.encryptedValue, - }) - .from(secrets) - .where(eq(secrets.serviceId, serviceId)); - const volumes = await tx - .select({ - name: serviceVolumes.name, - containerPath: serviceVolumes.containerPath, - }) - .from(serviceVolumes) - .where(eq(serviceVolumes.serviceId, serviceId)); - - const specification = buildServiceRevisionSpec({ - service, - placements, - ports, - secrets: revisionSecrets, - volumes, - }); - validateRevisionSpec(specification); +export async function createRolloutWithServiceRevision(serviceId: string) { + return db.transaction( + async (tx) => { + const service = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .then((rows) => rows[0]); - const contentHash = hashServiceRevisionSpec(specification); - let revision = await tx - .select() - .from(serviceRevisions) - .where( - and( - eq(serviceRevisions.serviceId, serviceId), - eq(serviceRevisions.contentHash, contentHash), - ), - ) - .then((rows) => rows[0]); + if (!service) { + throw new Error("Service not found"); + } - if (!revision) { - const [{ nextRevisionNumber }] = await tx + const placements = await tx .select({ - nextRevisionNumber: sql`coalesce(max(${serviceRevisions.revisionNumber}), 0) + 1`, + serverId: serviceReplicas.serverId, + count: serviceReplicas.count, }) - .from(serviceRevisions) - .where(eq(serviceRevisions.serviceId, serviceId)); - - revision = await tx - .insert(serviceRevisions) - .values({ - id: randomUUID(), - serviceId, - revisionNumber: nextRevisionNumber, - schemaVersion: SERVICE_REVISION_SCHEMA_VERSION, - specification, - contentHash, - sourceMetadata: compactSourceMetadata({ - sourceType: service.sourceType, - ...sourceMetadata, - }), + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const ports = await tx + .select() + .from(servicePorts) + .where(eq(servicePorts.serviceId, serviceId)); + const revisionSecrets = await tx + .select({ + key: secrets.key, + encryptedValue: secrets.encryptedValue, + updatedAt: secrets.updatedAt, }) + .from(secrets) + .where(eq(secrets.serviceId, serviceId)); + const volumes = await tx + .select({ + name: serviceVolumes.name, + containerPath: serviceVolumes.containerPath, + }) + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)); + + const specification = buildServiceRevisionSpec({ + service, + placements, + ports, + secrets: revisionSecrets, + volumes, + }); + const revision = await tx + .insert(serviceRevisions) + .values({ id: randomUUID(), serviceId, specification }) .returning() .then((rows) => rows[0]); - } - if (!revision) { - throw new Error("Failed to create service revision"); - } - - const rolloutId = randomUUID(); - await tx.insert(rollouts).values({ - id: rolloutId, - serviceId, - serviceRevisionId: revision.id, - status: "queued", - currentStage: "queued", - }); - - return { rolloutId, revision }; - }); + if (!revision) { + throw new Error("Failed to create service revision"); + } + + const rolloutId = randomUUID(); + await tx.insert(rollouts).values({ + id: rolloutId, + serviceId, + serviceRevisionId: revision.id, + status: "queued", + currentStage: "queued", + }); + + return { rolloutId, revision }; + }, + { isolationLevel: "repeatable read" }, + ); } export async function getRolloutServiceRevision(rolloutId: string) { const result = await db .select({ - rolloutServiceId: rollouts.serviceId, revision: serviceRevisions, }) .from(rollouts) @@ -171,13 +102,9 @@ export async function getRolloutServiceRevision(rolloutId: string) { if (!result) { throw new Error("Rollout revision not found"); } - if (result.rolloutServiceId !== result.revision.serviceId) { - throw new Error("Rollout revision does not belong to service"); - } if ( - result.revision.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION || result.revision.specification.schemaVersion !== - SERVICE_REVISION_SCHEMA_VERSION + SERVICE_REVISION_SCHEMA_VERSION ) { throw new Error("Unsupported service revision schema version"); } diff --git a/web/lib/wireguard.ts b/web/lib/wireguard.ts index 5f47dd6..cb564a7 100644 --- a/web/lib/wireguard.ts +++ b/web/lib/wireguard.ts @@ -1,8 +1,8 @@ -import { and, isNotNull, ne } from "drizzle-orm"; -import { Address4 } from "ip-address"; import { db } from "@/db"; -import { servers } from "@/db/schema"; -import { CONTAINER_SUBNET_PREFIX, WIREGUARD_SUBNET_PREFIX } from "./constants"; +import { servers, deployments } from "@/db/schema"; +import { eq, isNotNull, and, ne } from "drizzle-orm"; +import { WIREGUARD_SUBNET_PREFIX, CONTAINER_SUBNET_PREFIX } from "./constants"; +import { Address4 } from "ip-address"; function sameSubnet(ip1: string, ip2: string, prefix: number = 16): boolean { if (!ip1 || !ip2) return false; @@ -36,14 +36,28 @@ export async function assignSubnet(): Promise<{ throw new Error("No available subnets"); } -export function findAvailableContainerIp( - subnetId: number, - existingIps: Iterable, -): string { - const usedIps = new Set(existingIps); +export async function assignContainerIp(serverId: string): Promise { + const server = await db + .select({ subnetId: servers.subnetId }) + .from(servers) + .where(eq(servers.id, serverId)) + .then((r) => r[0]); + + if (!server?.subnetId) { + throw new Error("Server does not have a subnet assigned"); + } + + const existingDeployments = await db + .select({ ipAddress: deployments.ipAddress }) + .from(deployments) + .where( + and(eq(deployments.serverId, serverId), isNotNull(deployments.ipAddress)), + ); + + const usedIps = new Set(existingDeployments.map((d) => d.ipAddress)); for (let hostPart = 2; hostPart <= 254; hostPart++) { - const ip = `${CONTAINER_SUBNET_PREFIX}.${subnetId}.${hostPart}`; + const ip = `${CONTAINER_SUBNET_PREFIX}.${server.subnetId}.${hostPart}`; if (!usedIps.has(ip)) { return ip; } diff --git a/web/scripts/cutover-service-revisions.ts b/web/scripts/cutover-service-revisions.ts index 931b060..acc5e1e 100644 --- a/web/scripts/cutover-service-revisions.ts +++ b/web/scripts/cutover-service-revisions.ts @@ -4,21 +4,15 @@ import { buildCutoverServiceRevisionSpec, type CutoverDeployedConfig, } from "../lib/service-revision-cutover"; -import { - hashServiceRevisionSpec, - SERVICE_REVISION_SCHEMA_VERSION, -} from "../lib/service-revision-spec"; const connectionString = process.env.DATABASE_URL; -if (!connectionString) { - throw new Error("DATABASE_URL is required"); -} +if (!connectionString) throw new Error("DATABASE_URL is required"); const pool = new Pool({ connectionString }); async function baseSchemaExists(client: PoolClient) { const result = await client.query<{ exists: boolean }>( - `SELECT to_regclass('services') IS NOT NULL AS exists`, + "SELECT to_regclass('services') IS NOT NULL AS exists", ); return result.rows[0]?.exists ?? false; } @@ -33,18 +27,6 @@ async function cutoverIsRequired(client: PoolClient) { AND table_name = 'deployments' AND column_name = 'service_revision_id' ) - OR NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'rollouts' - AND column_name = 'service_revision_id' - ) - OR NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'services' - AND column_name = 'active_revision_id' - ) OR NOT EXISTS ( SELECT 1 FROM information_schema.columns WHERE table_schema = current_schema() @@ -67,19 +49,12 @@ async function prepareSchema(client: PoolClient) { id text PRIMARY KEY, service_id text NOT NULL CONSTRAINT service_revisions_service_id_services_id_fk REFERENCES services(id) ON DELETE CASCADE, - revision_number integer NOT NULL, - schema_version integer NOT NULL, specification jsonb NOT NULL, - content_hash text NOT NULL, - source_metadata jsonb, - created_at timestamptz NOT NULL DEFAULT now() - ) - `); - await client.query(` - ALTER TABLE services ADD COLUMN IF NOT EXISTS active_revision_id text; + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT service_revisions_id_service_id_unique UNIQUE (id, service_id) + ); ALTER TABLE deployments ADD COLUMN IF NOT EXISTS service_revision_id text; - ALTER TABLE rollouts ADD COLUMN IF NOT EXISTS service_revision_id text; - ALTER TABLE deployment_ports ADD COLUMN IF NOT EXISTS container_port integer + ALTER TABLE deployment_ports ADD COLUMN IF NOT EXISTS container_port integer; `); await client.query(` DO $$ @@ -102,48 +77,46 @@ async function prepareSchema(client: PoolClient) { async function captureBootstrapRevision(client: PoolClient, serviceId: string) { const serviceResult = await client.query( - `SELECT * FROM services WHERE id = $1 FOR UPDATE`, + "SELECT * FROM services WHERE id = $1 FOR UPDATE", [serviceId], ); const service = serviceResult.rows[0]; - if (!service) throw new Error(`Service ${serviceId} not found`); - let deployedConfig: CutoverDeployedConfig | null = null; - if (service.deployed_config) { - try { - deployedConfig = JSON.parse( - service.deployed_config, - ) as CutoverDeployedConfig; - } catch { - throw new Error(`Service ${serviceId} has invalid deployed_config JSON`); - } + if (!service) throw new Error("service not found"); + + if (!service.deployed_config) { + throw new Error("missing deployed_config"); + } + let deployedConfig: CutoverDeployedConfig; + try { + deployedConfig = JSON.parse(service.deployed_config); + } catch { + throw new Error("invalid deployed_config JSON"); } - const [placementResult, portResult, secretResult, volumeResult] = - await Promise.all([ - client.query( - `SELECT server_id, count FROM service_replicas WHERE service_id = $1`, - [serviceId], - ), - client.query( - `SELECT port, is_public, domain, protocol, external_port, tls_passthrough - FROM service_ports WHERE service_id = $1`, - [serviceId], - ), - client.query( - `SELECT key, encrypted_value, updated_at FROM secrets WHERE service_id = $1`, - [serviceId], - ), - client.query( - `SELECT name, container_path FROM service_volumes WHERE service_id = $1`, - [serviceId], - ), - ]); + const [placements, ports, secrets, volumes] = await Promise.all([ + client.query( + "SELECT server_id, count FROM service_replicas WHERE service_id = $1", + [serviceId], + ), + client.query( + `SELECT port, is_public, domain, protocol, external_port, tls_passthrough + FROM service_ports WHERE service_id = $1`, + [serviceId], + ), + client.query( + "SELECT key, encrypted_value, updated_at FROM secrets WHERE service_id = $1", + [serviceId], + ), + client.query( + "SELECT name, container_path FROM service_volumes WHERE service_id = $1", + [serviceId], + ), + ]); const specification = buildCutoverServiceRevisionSpec({ deployedConfig, liveDraft: { service: { - id: service.id, name: service.name, image: service.image, hostname: service.hostname, @@ -160,134 +133,73 @@ async function captureBootstrapRevision(client: PoolClient, serviceId: string) { resourceCpuLimit: service.resource_cpu_limit, resourceMemoryLimitMb: service.resource_memory_limit_mb, }, - placements: placementResult.rows.map((placement) => ({ - serverId: placement.server_id, - count: placement.count, + placements: placements.rows.map((row) => ({ + serverId: row.server_id, + count: row.count, })), - ports: portResult.rows.map((port) => ({ - port: port.port, - isPublic: port.is_public, - domain: port.domain, - protocol: port.protocol, - externalPort: port.external_port, - tlsPassthrough: port.tls_passthrough, + ports: ports.rows.map((row) => ({ + port: row.port, + isPublic: row.is_public, + domain: row.domain, + protocol: row.protocol, + externalPort: row.external_port, + tlsPassthrough: row.tls_passthrough, })), - secrets: secretResult.rows.map((secret) => ({ - key: secret.key, - encryptedValue: secret.encrypted_value, - updatedAt: secret.updated_at, + secrets: secrets.rows.map((row) => ({ + key: row.key, + encryptedValue: row.encrypted_value, + updatedAt: row.updated_at, })), - volumes: volumeResult.rows.map((volume) => ({ - name: volume.name, - containerPath: volume.container_path, + volumes: volumes.rows.map((row) => ({ + name: row.name, + containerPath: row.container_path, })), }, }); - - const totalReplicas = specification.placements.reduce( - (total, placement) => total + placement.count, - 0, - ); - const activeDeploymentResult = await client.query<{ active: boolean }>( - `SELECT EXISTS ( - SELECT 1 FROM deployments - WHERE service_id = $1 - AND runtime_desired_state <> 'removed' - AND traffic_state <> 'inactive' - ) AS active`, + const expectedContainerPorts = specification.ports + .map((port) => port.containerPort) + .sort((a, b) => a - b); + const activeDeployments = await client.query<{ + id: string; + container_ports: number[]; + }>( + `SELECT deployment.id, + COALESCE( + array_agg(deployment_port.container_port ORDER BY deployment_port.container_port) + FILTER (WHERE deployment_port.container_port IS NOT NULL), + ARRAY[]::integer[] + ) AS container_ports + FROM deployments AS deployment + LEFT JOIN deployment_ports AS deployment_port + ON deployment_port.deployment_id = deployment.id + WHERE deployment.service_id = $1 + AND deployment.runtime_desired_state <> 'removed' + AND deployment.traffic_state = 'active' + GROUP BY deployment.id`, [serviceId], ); - const hasActiveDeployment = activeDeploymentResult.rows[0]?.active ?? false; - if (hasActiveDeployment && (totalReplicas < 1 || totalReplicas > 10)) { - throw new Error( - `Service ${serviceId} has invalid replica count ${totalReplicas}`, - ); - } - if ( - hasActiveDeployment && - specification.stateful && - (totalReplicas !== 1 || specification.placements.length !== 1) - ) { - throw new Error(`Stateful service ${serviceId} has invalid placement`); - } - if (hasActiveDeployment) { - const activeDeployments = await client.query<{ id: string }>( - `SELECT id FROM deployments - WHERE service_id = $1 - AND runtime_desired_state <> 'removed' - AND traffic_state <> 'inactive'`, - [serviceId], - ); - const expectedPorts = specification.ports - .map((port) => port.containerPort) - .sort((a, b) => a - b); - for (const deployment of activeDeployments.rows) { - const allocatedPorts = await client.query<{ container_port: number }>( - `SELECT container_port FROM deployment_ports - WHERE deployment_id = $1 ORDER BY container_port`, - [deployment.id], + for (const deployment of activeDeployments.rows) { + if ( + JSON.stringify(deployment.container_ports) !== + JSON.stringify(expectedContainerPorts) + ) { + throw new Error( + `active deployment ${deployment.id} ports differ from the deployed snapshot`, ); - if ( - JSON.stringify( - allocatedPorts.rows.map((port) => port.container_port), - ) !== JSON.stringify(expectedPorts) - ) { - throw new Error( - `Deployment ${deployment.id} has incomplete runtime port allocation`, - ); - } } } - const contentHash = hashServiceRevisionSpec(specification); - const existingRevision = await client.query( - `SELECT id FROM service_revisions WHERE service_id = $1 AND content_hash = $2`, - [serviceId, contentHash], - ); - let revisionId = existingRevision.rows[0]?.id as string | undefined; - if (!revisionId) { - revisionId = randomUUID(); - await client.query( - `INSERT INTO service_revisions ( - id, service_id, revision_number, schema_version, specification, - content_hash, source_metadata - ) VALUES ( - $1, $2, - (SELECT coalesce(max(revision_number), 0) + 1 FROM service_revisions WHERE service_id = $2), - $3, $4, $5, $6 - )`, - [ - revisionId, - serviceId, - SERVICE_REVISION_SCHEMA_VERSION, - JSON.stringify(specification), - contentHash, - JSON.stringify({ sourceType: "cutover_bootstrap" }), - ], - ); - } - + const revisionId = randomUUID(); await client.query( - `UPDATE deployments SET service_revision_id = $1 - WHERE service_id = $2 AND service_revision_id IS NULL`, - [revisionId, serviceId], + `INSERT INTO service_revisions (id, service_id, specification) + VALUES ($1, $2, $3)`, + [revisionId, serviceId, JSON.stringify(specification)], ); await client.query( - `UPDATE rollouts SET service_revision_id = $1 + `UPDATE deployments SET service_revision_id = $1 WHERE service_id = $2 AND service_revision_id IS NULL`, [revisionId, serviceId], ); - await client.query( - `UPDATE services SET active_revision_id = $1 - WHERE id = $2 AND active_revision_id IS NULL - AND EXISTS ( - SELECT 1 FROM deployments - WHERE deployments.service_id = services.id - AND deployments.runtime_desired_state <> 'removed' - AND deployments.traffic_state <> 'inactive' - )`, - [revisionId, serviceId], - ); } async function finalizeSchema(client: PoolClient) { @@ -297,82 +209,35 @@ async function finalizeSchema(client: PoolClient) { IF EXISTS (SELECT 1 FROM deployments WHERE service_revision_id IS NULL) THEN RAISE EXCEPTION 'deployments.service_revision_id backfill incomplete'; END IF; - IF EXISTS (SELECT 1 FROM rollouts WHERE service_revision_id IS NULL) THEN - RAISE EXCEPTION 'rollouts.service_revision_id backfill incomplete'; - END IF; IF EXISTS (SELECT 1 FROM deployment_ports WHERE container_port IS NULL) THEN RAISE EXCEPTION 'deployment_ports.container_port backfill incomplete'; END IF; END $$ `); - const duplicateIps = await client.query<{ - server_id: string; - ip_address: string; - deployment_ids: string[]; - }>(` - SELECT server_id, ip_address, array_agg(id ORDER BY id) AS deployment_ids - FROM deployments - WHERE ip_address IS NOT NULL - GROUP BY server_id, ip_address - HAVING count(*) > 1 - ORDER BY server_id, ip_address - LIMIT 20 - `); - if (duplicateIps.rows.length > 0) { - const conflicts = duplicateIps.rows - .map( - (row) => - `server ${row.server_id} IP ${row.ip_address}: ${row.deployment_ids.join(", ")}`, - ) - .join("; "); - throw new Error( - `Duplicate container IP allocations prevent cutover: ${conflicts}`, - ); - } await client.query(` - CREATE UNIQUE INDEX IF NOT EXISTS service_revisions_service_revision_number_idx - ON service_revisions(service_id, revision_number); - CREATE UNIQUE INDEX IF NOT EXISTS service_revisions_service_content_hash_idx - ON service_revisions(service_id, content_hash); CREATE INDEX IF NOT EXISTS service_revisions_service_id_idx ON service_revisions(service_id); CREATE INDEX IF NOT EXISTS deployments_service_revision_id_idx ON deployments(service_revision_id); - CREATE UNIQUE INDEX IF NOT EXISTS deployments_server_ip_address_idx - ON deployments(server_id, ip_address); - CREATE INDEX IF NOT EXISTS rollouts_service_revision_id_idx - ON rollouts(service_revision_id) - `); - await client.query(` ALTER TABLE deployments ALTER COLUMN service_revision_id SET NOT NULL; - ALTER TABLE rollouts ALTER COLUMN service_revision_id SET NOT NULL; - ALTER TABLE deployment_ports ALTER COLUMN container_port SET NOT NULL + ALTER TABLE deployment_ports ALTER COLUMN container_port SET NOT NULL; `); await client.query(` DO $$ BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'service_revisions_id_service_id_unique') THEN - ALTER TABLE service_revisions ADD CONSTRAINT service_revisions_id_service_id_unique - UNIQUE (id, service_id); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'services_active_revision_id_service_revisions_id_fk') THEN - ALTER TABLE services ADD CONSTRAINT services_active_revision_id_service_revisions_id_fk - FOREIGN KEY (active_revision_id) REFERENCES service_revisions(id) ON DELETE SET NULL; - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'deployments_service_revision_service_fk') THEN - ALTER TABLE deployments ADD CONSTRAINT deployments_service_revision_service_fk - FOREIGN KEY (service_revision_id, service_id) - REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'rollouts_service_revision_service_fk') THEN - ALTER TABLE rollouts ADD CONSTRAINT rollouts_service_revision_service_fk - FOREIGN KEY (service_revision_id, service_id) - REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'deployments_service_revision_service_fk' + ) THEN + ALTER TABLE deployments + ADD CONSTRAINT deployments_service_revision_service_fk + FOREIGN KEY (service_revision_id, service_id) + REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; END IF; END $$ `); await client.query( - `ALTER TABLE deployment_ports DROP COLUMN IF EXISTS service_port_id`, + "ALTER TABLE deployment_ports DROP COLUMN IF EXISTS service_port_id", ); } @@ -385,29 +250,27 @@ async function main() { ); if (!(await baseSchemaExists(client))) { await client.query("COMMIT"); - console.log( - "Fresh database detected; schema push will create service revisions.", - ); return; } if (!(await cutoverIsRequired(client))) { await client.query("COMMIT"); - console.log("Service revision cutover already complete; skipping."); return; } + const activeRollouts = await client.query<{ count: string }>( - `SELECT count(*)::text AS count FROM rollouts WHERE status IN ('queued', 'in_progress')`, + "SELECT count(*)::text AS count FROM rollouts WHERE status IN ('queued', 'in_progress')", ); if (Number(activeRollouts.rows[0]?.count ?? 0) > 0) { - throw new Error("Cutover requires zero queued or in-progress rollouts"); + throw new Error("cutover requires zero queued or in-progress rollouts"); } - await prepareSchema(client); - const serviceIds = await client.query<{ service_id: string }>(` - SELECT DISTINCT service_id FROM deployments - UNION - SELECT DISTINCT service_id FROM rollouts - `); + await client.query( + "DELETE FROM deployments WHERE traffic_state <> 'active' OR runtime_desired_state = 'removed'", + ); + await prepareSchema(client); + const serviceIds = await client.query<{ service_id: string }>( + "SELECT DISTINCT service_id FROM deployments", + ); for (const { service_id: serviceId } of serviceIds.rows) { try { await captureBootstrapRevision(client, serviceId); @@ -416,12 +279,9 @@ async function main() { throw new Error(`Service ${serviceId}: ${message}`); } } - await finalizeSchema(client); await client.query("COMMIT"); - console.log( - `Service revision cutover complete for ${serviceIds.rowCount ?? 0} services.`, - ); + console.log(`Backfilled ${serviceIds.rowCount ?? 0} service revisions.`); } catch (error) { await client.query("ROLLBACK"); throw error; diff --git a/web/tests/agent-capabilities.test.ts b/web/tests/agent-capabilities.test.ts deleted file mode 100644 index 388df2f..0000000 --- a/web/tests/agent-capabilities.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - getAgentCompatibilityStatus, - SERVICE_REVISION_CAPABILITY, -} from "@/lib/agent-capabilities"; - -describe("agent compatibility", () => { - it("requires an explicit service revision capability", () => { - expect(getAgentCompatibilityStatus(null)).toBe("upgrade_required"); - expect( - getAgentCompatibilityStatus({ - version: "1.0.0", - uptimeSecs: 10, - capabilities: ["serverless_gateway"], - }), - ).toBe("upgrade_required"); - }); - - it("accepts agents advertising the revision contract", () => { - expect( - getAgentCompatibilityStatus({ - version: "1.0.0", - uptimeSecs: 10, - capabilities: [SERVICE_REVISION_CAPABILITY], - }), - ).toBe("compatible"); - }); -}); diff --git a/web/tests/deployment-status.test.ts b/web/tests/deployment-status.test.ts index 6bb8226..06d958c 100644 --- a/web/tests/deployment-status.test.ts +++ b/web/tests/deployment-status.test.ts @@ -1,10 +1,9 @@ import { describe, expect, it } from "vitest"; import { isDeploymentRoutable, - isObservedReady, isRuntimeExpected, + isObservedReady, markDeploymentRemoved, - selectNewestRevisionId, } from "@/lib/deployment-status"; describe("deployment state helpers", () => { @@ -47,20 +46,3 @@ describe("deployment state helpers", () => { }); }); }); - -describe("selectNewestRevisionId", () => { - it("selects the highest revision deterministically", () => { - expect( - selectNewestRevisionId([ - { serviceRevisionId: "revision-1", revisionNumber: 1 }, - { serviceRevisionId: "revision-3", revisionNumber: 3 }, - { serviceRevisionId: "revision-2", revisionNumber: 2 }, - { serviceRevisionId: "revision-3", revisionNumber: 3 }, - ]), - ).toBe("revision-3"); - }); - - it("returns null when there are no active revisions", () => { - expect(selectNewestRevisionId([])).toBeNull(); - }); -}); diff --git a/web/tests/expected-state-route.test.ts b/web/tests/expected-state-route.test.ts deleted file mode 100644 index 8a39e51..0000000 --- a/web/tests/expected-state-route.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { NextRequest } from "next/server"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ - buildAgentExpectedState: vi.fn(), - getServer: vi.fn(), - verifyAgentRequest: vi.fn(), -})); - -vi.mock("@/lib/agent/expected-state", () => ({ - buildAgentExpectedState: mocks.buildAgentExpectedState, - getServer: mocks.getServer, -})); -vi.mock("@/lib/agent-auth", () => ({ - verifyAgentRequest: mocks.verifyAgentRequest, -})); - -import { GET } from "@/app/api/v1/agent/expected-state/route"; - -describe("agent expected-state route", () => { - beforeEach(() => { - vi.clearAllMocks(); - delete process.env.EXPECTED_STATE_MAINTENANCE_MODE; - mocks.verifyAgentRequest.mockResolvedValue({ - success: true, - serverId: "server-1", - }); - mocks.getServer.mockResolvedValue({ - id: "server-1", - agentHealth: { version: "old", uptimeSecs: 60, capabilities: [] }, - }); - }); - - it("returns a structured 426 for an incompatible agent", async () => { - const response = await GET( - new NextRequest("http://localhost/api/v1/agent/expected-state"), - ); - - expect(response.status).toBe(426); - expect(await response.json()).toEqual({ - error: "Agent upgrade required", - code: "AGENT_UPGRADE_REQUIRED", - requiredCapabilities: ["service_revision_v1"], - }); - expect(mocks.buildAgentExpectedState).not.toHaveBeenCalled(); - }); - - it("pauses expected state during the maintenance window", async () => { - process.env.EXPECTED_STATE_MAINTENANCE_MODE = "true"; - mocks.getServer.mockResolvedValue({ - id: "server-1", - agentHealth: { - version: "new", - uptimeSecs: 60, - capabilities: ["service_revision_v1"], - }, - }); - - const response = await GET( - new NextRequest("http://localhost/api/v1/agent/expected-state"), - ); - - expect(response.status).toBe(503); - expect(response.headers.get("Retry-After")).toBe("30"); - expect(await response.json()).toMatchObject({ - code: "EXPECTED_STATE_MAINTENANCE", - }); - }); - - it("returns a structured retryable error when state construction fails", async () => { - mocks.getServer.mockResolvedValue({ - id: "server-1", - agentHealth: { - version: "new", - uptimeSecs: 60, - capabilities: ["service_revision_v1"], - }, - }); - mocks.buildAgentExpectedState.mockRejectedValue( - new Error("incomplete port allocation"), - ); - - const response = await GET( - new NextRequest("http://localhost/api/v1/agent/expected-state"), - ); - - expect(response.status).toBe(503); - expect(response.headers.get("Retry-After")).toBe("15"); - expect(await response.json()).toMatchObject({ - code: "EXPECTED_STATE_BUILD_FAILED", - }); - }); -}); diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index eaf13dc..fcc1542 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -13,7 +13,6 @@ import { buildServerlessTraefikRouteSets, buildTraefikCertificateDomains, buildTraefikRoutes, - selectRuntimeServiceRevisions, } from "@/lib/agent/expected-state"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; @@ -26,11 +25,9 @@ describe("expected-state pure builders", () => { id: `rev_${serviceId}`, name: serviceId, serviceId, - schemaVersion: 1, revisionId: `rev_${serviceId}`, specification: { schemaVersion: 1, - serviceId, image: "nginx", hostname: serviceId, stateful: false, @@ -121,8 +118,16 @@ describe("expected-state pure builders", () => { }, ], secrets: [ - { key: "ZED", encryptedValue: "last" }, - { key: "ALPHA", encryptedValue: "first" }, + { + key: "ZED", + encryptedValue: "last", + updatedAt: "2026-07-01T00:00:00.000Z", + }, + { + key: "ALPHA", + encryptedValue: "first", + updatedAt: "2026-07-01T00:00:00.000Z", + }, ], volumes: [ { name: "cache", containerPath: "/var/cache" }, @@ -186,57 +191,6 @@ describe("expected-state pure builders", () => { }); }); - it("keeps the spec hash stable for duplicate container ports", () => { - const build = (hostPorts: number[]) => - buildExpectedContainersFromRows({ - deployments: [ - { - id: "dep_dns", - serviceId: "svc_dns", - serviceRevisionId: "rev_svc_dns", - runtimeDesiredState: "running", - }, - ] as any, - services: [{ id: "svc_dns", name: "dns" }] as any, - revisions: [ - revision("svc_dns", { - ports: [ - { - containerPort: 53, - isPublic: true, - domain: null, - protocol: "tcp", - externalPort: null, - tlsPassthrough: false, - }, - { - containerPort: 53, - isPublic: true, - domain: null, - protocol: "udp", - externalPort: null, - tlsPassthrough: false, - }, - ], - }), - ], - deploymentPorts: hostPorts.map((hostPort) => ({ - deploymentId: "dep_dns", - containerPort: 53, - hostPort, - })) as any, - })[0]; - - const first = build([30002, 30001]); - const second = build([30001, 30002]); - - expect(first.ports).toEqual([ - { containerPort: 53, hostPort: 30001 }, - { containerPort: 53, hostPort: 30002 }, - ]); - expect(first.containerSpecHash).toBe(second.containerSpecHash); - }); - it("rejects partial expected state when a deployment revision is missing", () => { expect(() => buildExpectedContainersFromRows({ @@ -304,58 +258,6 @@ describe("expected-state pure builders", () => { ).toThrow("Deployment dep_incomplete_ports has incomplete port allocation"); }); - it("contains multiple active revisions to the authoritative revision", () => { - const specification = runtimeRevision("svc_1").specification; - const result = selectRuntimeServiceRevisions([ - { - deploymentId: "dep_old", - serviceId: "svc_1", - serviceName: "api", - serviceActiveRevisionId: "rev_old", - revisionId: "rev_old", - revisionServiceId: "svc_1", - revisionSchemaVersion: 1, - specification, - }, - { - deploymentId: "dep_new", - serviceId: "svc_1", - serviceName: "api", - serviceActiveRevisionId: "rev_old", - revisionId: "rev_new", - revisionServiceId: "svc_1", - revisionSchemaVersion: 1, - specification, - }, - ]); - - expect(result.services).toHaveLength(1); - expect(result.services[0]?.revisionId).toBe("rev_old"); - expect(result.errors[0]).toContain("multiple active revisions"); - }); - - it("excludes desired running state from the container creation hash", () => { - const build = (runtimeDesiredState: "running" | "stopped") => - buildExpectedContainersFromRows({ - deployments: [ - { - id: "dep_1", - serviceId: "svc_1", - serviceRevisionId: "rev_svc_1", - ipAddress: "10.0.0.1", - runtimeDesiredState, - }, - ] as any, - services: [{ id: "svc_1", name: "api" }] as any, - revisions: [revision("svc_1")], - deploymentPorts: [], - })[0]; - - expect(build("running").containerSpecHash).toBe( - build("stopped").containerSpecHash, - ); - }); - it("keeps HTTP local upstreams before remote upstreams", () => { const routes = buildTraefikRoutes({ serverId: "server_local", diff --git a/web/tests/rollout-helpers.test.ts b/web/tests/rollout-helpers.test.ts deleted file mode 100644 index f894642..0000000 --- a/web/tests/rollout-helpers.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -vi.mock("@/db", () => ({ db: {} })); -vi.mock("@/db/queries", () => ({ getService: vi.fn() })); -vi.mock("@/lib/acme-manager", () => ({ - getCertificate: vi.fn(), - issueCertificate: vi.fn(), -})); -vi.mock("@/lib/wireguard", () => ({ - findAvailableContainerIp: vi.fn(), -})); -vi.mock("@/lib/work-queue", () => ({ - enqueueWork: vi.fn(), -})); - -import { - findAvailableHostPorts, - isActiveDeploymentForRollout, -} from "@/lib/inngest/functions/rollout-helpers"; - -describe("rollout helpers", () => { - it("treats active traffic deployments as the live rollout version", () => { - expect(isActiveDeploymentForRollout({ trafficState: "active" })).toBe(true); - expect(isActiveDeploymentForRollout({ trafficState: "candidate" })).toBe( - false, - ); - expect(isActiveDeploymentForRollout({ trafficState: "draining" })).toBe( - false, - ); - }); - - it("allocates host ports around ports already in use", () => { - expect(findAvailableHostPorts([30000, 30002], 3)).toEqual([ - 30001, 30003, 30004, - ]); - }); -}); diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index 332744e..8609fed 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -6,7 +6,6 @@ import { MIN_SERVERLESS_SLEEP_AFTER_SECONDS, revisionSpecToDeployedConfig, } from "@/lib/service-config"; -import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; function deployedConfig( overrides: Partial = {}, @@ -27,7 +26,7 @@ function deployedConfig( } describe("service config", () => { - it("enforces the minimum serverless sleep timeout for draft config", () => { + it("enforces the minimum serverless sleep timeout", () => { expect( getCurrentServerlessConfig({ serverlessEnabled: true, @@ -39,39 +38,32 @@ describe("service config", () => { }); }); - it("converts an immutable revision into the pending-change baseline", () => { - const specification: ServiceRevisionSpec = { - schemaVersion: 1, - serviceId: "service-1", - image: "nginx", - hostname: "api", - stateful: true, - serverless: { - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, + it("converts an immutable revision for pending-change comparisons", () => { + const config = revisionSpecToDeployedConfig( + { + schemaVersion: 1, + image: "nginx", + hostname: "api", + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "server-1", count: 1 }], + ports: [], + secrets: [], + volumes: [], }, - healthCheck: null, - startCommand: null, - resourceLimits: { cpuCores: null, memoryMb: null }, - placements: [{ serverId: "server-1", count: 1 }], - ports: [], - secrets: [{ key: "TOKEN", encryptedValue: "ciphertext" }], - volumes: [], - }; + { "server-1": "Sydney" }, + ); - expect( - revisionSpecToDeployedConfig( - specification, - { "server-1": "Sydney" }, - { TOKEN: "fingerprint" }, - ), - ).toMatchObject({ - stateful: true, - serverless: { enabled: true }, - replicas: [{ serverId: "server-1", serverName: "Sydney", count: 1 }], - secrets: [{ key: "TOKEN", fingerprint: "fingerprint" }], - }); + expect(config.replicas).toEqual([ + { serverId: "server-1", serverName: "Sydney", count: 1 }, + ]); }); it("reports serverless changes as pending config", () => { diff --git a/web/tests/service-revision-cutover.test.ts b/web/tests/service-revision-cutover.test.ts index 0927c4b..592ea99 100644 --- a/web/tests/service-revision-cutover.test.ts +++ b/web/tests/service-revision-cutover.test.ts @@ -5,7 +5,6 @@ import type { ServiceRevisionDraft } from "@/lib/service-revision-spec"; function liveDraft(): ServiceRevisionDraft { return { service: { - id: "service-1", name: "API", image: "current:image", hostname: "current-host", @@ -94,6 +93,10 @@ describe("service revision cutover", () => { const specification = buildCutoverServiceRevisionSpec({ liveDraft: liveDraft(), deployedConfig: { + source: { type: "image", image: "deployed:image" }, + replicas: [{ serverId: "deployed-server", count: 1 }], + healthCheck: null, + ports: [], secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], }, }); @@ -112,27 +115,13 @@ describe("service revision cutover", () => { buildCutoverServiceRevisionSpec({ liveDraft: draft, deployedConfig: { + source: { type: "image", image: "deployed:image" }, + replicas: [{ serverId: "deployed-server", count: 1 }], + healthCheck: null, + ports: [], secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], }, }), ).toThrow("Secret TOKEN differs from the deployed snapshot"); }); - - it("uses live configuration for a service that has never been deployed", () => { - const draft = liveDraft(); - const specification = buildCutoverServiceRevisionSpec({ - liveDraft: draft, - deployedConfig: null, - }); - - expect(specification).toMatchObject({ - image: draft.service.image, - hostname: draft.service.hostname, - placements: draft.placements, - healthCheck: { cmd: "current-health" }, - startCommand: "current-command", - ports: [{ containerPort: 8080, externalPort: 443 }], - volumes: draft.volumes, - }); - }); }); diff --git a/web/tests/service-revision-spec.test.ts b/web/tests/service-revision-spec.test.ts index fed67ad..24bc362 100644 --- a/web/tests/service-revision-spec.test.ts +++ b/web/tests/service-revision-spec.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from "vitest"; import { buildServiceRevisionSpec, - hashServiceRevisionSpec, type ServiceRevisionDraft, } from "@/lib/service-revision-spec"; @@ -10,7 +9,6 @@ function draft( ): ServiceRevisionDraft { return { service: { - id: "service-1", name: "API Service", image: "nginx:latest", hostname: "api.internal", @@ -50,8 +48,16 @@ function draft( }, ], secrets: [ - { key: "TOKEN", encryptedValue: "ciphertext-2" }, - { key: "API_KEY", encryptedValue: "ciphertext-1" }, + { + key: "TOKEN", + encryptedValue: "ciphertext-2", + updatedAt: "2026-07-01T00:00:00.000Z", + }, + { + key: "API_KEY", + encryptedValue: "ciphertext-1", + updatedAt: "2026-07-01T00:00:00.000Z", + }, ], volumes: [ { name: "logs", containerPath: "/logs" }, @@ -62,7 +68,7 @@ function draft( } describe("service revision specification", () => { - it("produces the same hash regardless of draft row ordering", () => { + it("normalizes draft row ordering", () => { const first = buildServiceRevisionSpec(draft()); const reorderedDraft = draft(); reorderedDraft.placements.reverse(); @@ -72,9 +78,6 @@ describe("service revision specification", () => { const second = buildServiceRevisionSpec(reorderedDraft); expect(second).toEqual(first); - expect(hashServiceRevisionSpec(second)).toBe( - hashServiceRevisionSpec(first), - ); }); it("normalizes defaults once", () => { @@ -94,14 +97,11 @@ describe("service revision specification", () => { }); }); - it("changes identity when encrypted secret content changes", () => { - const first = buildServiceRevisionSpec(draft()); - const changedDraft = draft(); - changedDraft.secrets[0].encryptedValue = "new-ciphertext"; - const changed = buildServiceRevisionSpec(changedDraft); + it("rejects an invalid replica layout before it can be persisted", () => { + const input = draft({ placements: [] }); - expect(hashServiceRevisionSpec(changed)).not.toBe( - hashServiceRevisionSpec(first), + expect(() => buildServiceRevisionSpec(input)).toThrow( + "At least one replica is required", ); }); }); From ac6f9decc3b44ed7ec191cd8eeb4f86214a41138 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:42:45 +1000 Subject: [PATCH 07/10] Remove service revision cutover script --- deployment/README.md | 19 +- deployment/compose.postgres.yml | 2 +- deployment/compose.production.yml | 2 +- web/Dockerfile | 4 +- web/lib/service-revision-cutover.ts | 110 -------- web/package.json | 1 - web/scripts/cutover-service-revisions.ts | 294 --------------------- web/tests/service-revision-cutover.test.ts | 127 --------- 8 files changed, 15 insertions(+), 544 deletions(-) delete mode 100644 web/lib/service-revision-cutover.ts delete mode 100644 web/scripts/cutover-service-revisions.ts delete mode 100644 web/tests/service-revision-cutover.test.ts diff --git a/deployment/README.md b/deployment/README.md index 5d9ef44..e02aaf8 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -93,13 +93,18 @@ Schema is synced automatically by the one-shot `migrate` service via `drizzle-ki ### Immutable service revision cutover Before deploying this release, stop rollout producers, verify no rollout is -queued or in progress, and take a PostgreSQL backup. The migrate service creates -one baseline revision for each existing deployment before applying the new -schema. Existing containers keep running unchanged; subsequent deployments use -immutable revision snapshots. Stale non-active deployment records are removed -before the baseline is captured. The cutover aborts if an active deployment -cannot be reconstructed safely from its stored deployed configuration; that -legacy column is dropped after the backfill succeeds. +queued or in progress, and take a PostgreSQL backup. Then run this once against +the database: + +```sql +DELETE FROM deployments; +``` + +Port allocations are removed by cascade. The schema sync can then add the +required revision reference without migrating legacy deployment state. After +the new version starts, redeploy previously active services from their current +configuration. Agents remove the orphaned containers; service volumes and +backups are not deleted. **Future plan:** Once the schema stabilizes, switch to `drizzle-kit generate` + `drizzle-orm migrate()` with pre-generated SQL migration files. This will eliminate the esbuild/drizzle-kit dependency from the production image. diff --git a/deployment/compose.postgres.yml b/deployment/compose.postgres.yml index 3a196a0..2ca6c88 100644 --- a/deployment/compose.postgres.yml +++ b/deployment/compose.postgres.yml @@ -93,7 +93,7 @@ services: depends_on: postgres: condition: service_healthy - command: ["sh", "-c", "npx tsx scripts/cutover-service-revisions.ts && npx drizzle-kit push --force"] + command: ["npx", "drizzle-kit", "push", "--force"] restart: on-failure web: diff --git a/deployment/compose.production.yml b/deployment/compose.production.yml index 94dca63..9060030 100644 --- a/deployment/compose.production.yml +++ b/deployment/compose.production.yml @@ -72,7 +72,7 @@ services: - CONTROL_PLANE_UPDATER_URL=http://control-plane-updater:8080 - CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN} - ALLOW_SIGNUP=${ALLOW_SIGNUP:-false} - command: ["sh", "-c", "npx tsx scripts/cutover-service-revisions.ts && npx drizzle-kit push --force"] + command: ["npx", "drizzle-kit", "push", "--force"] restart: on-failure web: diff --git a/web/Dockerfile b/web/Dockerfile index 4e7fb73..6206f45 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -17,7 +17,7 @@ RUN npx next build FROM node:24-slim AS drizzle WORKDIR /drizzle -RUN npm install drizzle-kit drizzle-orm pg tsx +RUN npm install drizzle-kit drizzle-orm FROM node:24-slim AS runner WORKDIR /app @@ -27,8 +27,6 @@ COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static COPY --from=builder /app/public ./public COPY --from=builder /app/db ./db -COPY --from=builder /app/lib/service-revision-spec.ts ./lib/service-revision-spec.ts -COPY --from=builder /app/lib/service-revision-cutover.ts ./lib/service-revision-cutover.ts COPY --from=builder /app/scripts ./scripts COPY --from=builder /app/drizzle.config.ts ./drizzle.config.ts COPY --from=drizzle /drizzle/node_modules ./node_modules diff --git a/web/lib/service-revision-cutover.ts b/web/lib/service-revision-cutover.ts deleted file mode 100644 index f76d653..0000000 --- a/web/lib/service-revision-cutover.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { - buildServiceRevisionSpec, - type ServiceRevisionDraft, -} from "./service-revision-spec"; -import type { DeployedConfig } from "./service-config"; - -export type CutoverDeployedConfig = DeployedConfig; - -function assertSecretsMatchDeployedSnapshot( - liveSecrets: ServiceRevisionDraft["secrets"], - deployedConfig: CutoverDeployedConfig, -) { - const deployedSecrets = deployedConfig.secrets; - if (!deployedSecrets) { - if ( - liveSecrets.length > 0 || - (deployedConfig.secretKeys?.length ?? 0) > 0 - ) { - throw new Error( - "Cannot verify deployed secrets because the deployed snapshot has no secret timestamps", - ); - } - return; - } - - const liveSecretsByKey = new Map( - liveSecrets.map((secret) => [secret.key, secret]), - ); - if (liveSecretsByKey.size !== deployedSecrets.length) { - throw new Error("Live secrets differ from the deployed snapshot"); - } - - for (const deployedSecret of deployedSecrets) { - const liveSecret = liveSecretsByKey.get(deployedSecret.key); - const liveUpdatedAt = liveSecret?.updatedAt - ? new Date(liveSecret.updatedAt).toISOString() - : null; - const deployedUpdatedAt = deployedSecret.updatedAt - ? new Date(deployedSecret.updatedAt).toISOString() - : null; - if (!liveSecret || !liveUpdatedAt || liveUpdatedAt !== deployedUpdatedAt) { - throw new Error( - `Secret ${deployedSecret.key} differs from the deployed snapshot`, - ); - } - } -} - -export function buildCutoverServiceRevisionSpec({ - liveDraft, - deployedConfig, -}: { - liveDraft: ServiceRevisionDraft; - deployedConfig: CutoverDeployedConfig; -}) { - assertSecretsMatchDeployedSnapshot(liveDraft.secrets, deployedConfig); - - const currentPortsByIdentity = new Map( - liveDraft.ports.map((port) => [ - `${port.port}:${port.protocol ?? "http"}`, - port, - ]), - ); - const deployedPorts = deployedConfig.ports.map((port) => { - const currentPort = currentPortsByIdentity.get( - `${port.port}:${port.protocol ?? "http"}`, - ); - return { - port: port.port, - isPublic: port.isPublic, - domain: port.domain, - protocol: port.protocol ?? null, - externalPort: currentPort?.externalPort ?? null, - tlsPassthrough: port.tlsPassthrough ?? null, - }; - }); - const deployedHealthCheck = deployedConfig.healthCheck ?? null; - - return buildServiceRevisionSpec({ - service: { - ...liveDraft.service, - image: deployedConfig.source.image, - hostname: deployedConfig.hostname ?? liveDraft.service.hostname, - stateful: deployedConfig.stateful ?? liveDraft.service.stateful, - serverlessEnabled: - deployedConfig.serverless?.enabled ?? - liveDraft.service.serverlessEnabled, - serverlessSleepAfterSeconds: - deployedConfig.serverless?.sleepAfterSeconds ?? - liveDraft.service.serverlessSleepAfterSeconds, - serverlessWakeTimeoutSeconds: - deployedConfig.serverless?.wakeTimeoutSeconds ?? - liveDraft.service.serverlessWakeTimeoutSeconds, - healthCheckCmd: deployedHealthCheck?.cmd ?? null, - healthCheckInterval: deployedHealthCheck?.interval ?? null, - healthCheckTimeout: deployedHealthCheck?.timeout ?? null, - healthCheckRetries: deployedHealthCheck?.retries ?? null, - healthCheckStartPeriod: deployedHealthCheck?.startPeriod ?? null, - startCommand: deployedConfig.startCommand ?? null, - resourceCpuLimit: deployedConfig.resourceLimits?.cpuCores ?? null, - resourceMemoryLimitMb: deployedConfig.resourceLimits?.memoryMb ?? null, - }, - placements: deployedConfig.replicas, - ports: deployedPorts, - secrets: liveDraft.secrets, - volumes: Array.isArray(deployedConfig.volumes) - ? deployedConfig.volumes - : liveDraft.volumes, - }); -} diff --git a/web/package.json b/web/package.json index 9e2e355..8bae61e 100644 --- a/web/package.json +++ b/web/package.json @@ -12,7 +12,6 @@ "admin:create": "node scripts/admin.mjs --create", "admin:reset-password": "node scripts/admin.mjs --reset-password", "db:push": "drizzle-kit push", - "db:cutover-service-revisions": "tsx scripts/cutover-service-revisions.ts", "db:studio": "drizzle-kit studio" }, "dependencies": { diff --git a/web/scripts/cutover-service-revisions.ts b/web/scripts/cutover-service-revisions.ts deleted file mode 100644 index acc5e1e..0000000 --- a/web/scripts/cutover-service-revisions.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { Pool, type PoolClient } from "pg"; -import { - buildCutoverServiceRevisionSpec, - type CutoverDeployedConfig, -} from "../lib/service-revision-cutover"; - -const connectionString = process.env.DATABASE_URL; -if (!connectionString) throw new Error("DATABASE_URL is required"); - -const pool = new Pool({ connectionString }); - -async function baseSchemaExists(client: PoolClient) { - const result = await client.query<{ exists: boolean }>( - "SELECT to_regclass('services') IS NOT NULL AS exists", - ); - return result.rows[0]?.exists ?? false; -} - -async function cutoverIsRequired(client: PoolClient) { - const result = await client.query<{ required: boolean }>(` - SELECT - to_regclass('service_revisions') IS NULL - OR NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'deployments' - AND column_name = 'service_revision_id' - ) - OR NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'deployment_ports' - AND column_name = 'container_port' - ) - OR EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'deployment_ports' - AND column_name = 'service_port_id' - ) AS required - `); - return result.rows[0]?.required ?? true; -} - -async function prepareSchema(client: PoolClient) { - await client.query(` - CREATE TABLE IF NOT EXISTS service_revisions ( - id text PRIMARY KEY, - service_id text NOT NULL CONSTRAINT service_revisions_service_id_services_id_fk - REFERENCES services(id) ON DELETE CASCADE, - specification jsonb NOT NULL, - created_at timestamptz NOT NULL DEFAULT now(), - CONSTRAINT service_revisions_id_service_id_unique UNIQUE (id, service_id) - ); - ALTER TABLE deployments ADD COLUMN IF NOT EXISTS service_revision_id text; - ALTER TABLE deployment_ports ADD COLUMN IF NOT EXISTS container_port integer; - `); - await client.query(` - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = current_schema() - AND table_name = 'deployment_ports' - AND column_name = 'service_port_id' - ) THEN - UPDATE deployment_ports AS deployment_port - SET container_port = service_port.port - FROM service_ports AS service_port - WHERE deployment_port.service_port_id = service_port.id - AND deployment_port.container_port IS NULL; - END IF; - END $$ - `); -} - -async function captureBootstrapRevision(client: PoolClient, serviceId: string) { - const serviceResult = await client.query( - "SELECT * FROM services WHERE id = $1 FOR UPDATE", - [serviceId], - ); - const service = serviceResult.rows[0]; - if (!service) throw new Error("service not found"); - - if (!service.deployed_config) { - throw new Error("missing deployed_config"); - } - let deployedConfig: CutoverDeployedConfig; - try { - deployedConfig = JSON.parse(service.deployed_config); - } catch { - throw new Error("invalid deployed_config JSON"); - } - - const [placements, ports, secrets, volumes] = await Promise.all([ - client.query( - "SELECT server_id, count FROM service_replicas WHERE service_id = $1", - [serviceId], - ), - client.query( - `SELECT port, is_public, domain, protocol, external_port, tls_passthrough - FROM service_ports WHERE service_id = $1`, - [serviceId], - ), - client.query( - "SELECT key, encrypted_value, updated_at FROM secrets WHERE service_id = $1", - [serviceId], - ), - client.query( - "SELECT name, container_path FROM service_volumes WHERE service_id = $1", - [serviceId], - ), - ]); - - const specification = buildCutoverServiceRevisionSpec({ - deployedConfig, - liveDraft: { - service: { - name: service.name, - image: service.image, - hostname: service.hostname, - stateful: service.stateful, - serverlessEnabled: service.serverless_enabled, - serverlessSleepAfterSeconds: service.serverless_sleep_after_seconds, - serverlessWakeTimeoutSeconds: service.serverless_wake_timeout_seconds, - healthCheckCmd: service.health_check_cmd, - healthCheckInterval: service.health_check_interval, - healthCheckTimeout: service.health_check_timeout, - healthCheckRetries: service.health_check_retries, - healthCheckStartPeriod: service.health_check_start_period, - startCommand: service.start_command, - resourceCpuLimit: service.resource_cpu_limit, - resourceMemoryLimitMb: service.resource_memory_limit_mb, - }, - placements: placements.rows.map((row) => ({ - serverId: row.server_id, - count: row.count, - })), - ports: ports.rows.map((row) => ({ - port: row.port, - isPublic: row.is_public, - domain: row.domain, - protocol: row.protocol, - externalPort: row.external_port, - tlsPassthrough: row.tls_passthrough, - })), - secrets: secrets.rows.map((row) => ({ - key: row.key, - encryptedValue: row.encrypted_value, - updatedAt: row.updated_at, - })), - volumes: volumes.rows.map((row) => ({ - name: row.name, - containerPath: row.container_path, - })), - }, - }); - const expectedContainerPorts = specification.ports - .map((port) => port.containerPort) - .sort((a, b) => a - b); - const activeDeployments = await client.query<{ - id: string; - container_ports: number[]; - }>( - `SELECT deployment.id, - COALESCE( - array_agg(deployment_port.container_port ORDER BY deployment_port.container_port) - FILTER (WHERE deployment_port.container_port IS NOT NULL), - ARRAY[]::integer[] - ) AS container_ports - FROM deployments AS deployment - LEFT JOIN deployment_ports AS deployment_port - ON deployment_port.deployment_id = deployment.id - WHERE deployment.service_id = $1 - AND deployment.runtime_desired_state <> 'removed' - AND deployment.traffic_state = 'active' - GROUP BY deployment.id`, - [serviceId], - ); - for (const deployment of activeDeployments.rows) { - if ( - JSON.stringify(deployment.container_ports) !== - JSON.stringify(expectedContainerPorts) - ) { - throw new Error( - `active deployment ${deployment.id} ports differ from the deployed snapshot`, - ); - } - } - - const revisionId = randomUUID(); - await client.query( - `INSERT INTO service_revisions (id, service_id, specification) - VALUES ($1, $2, $3)`, - [revisionId, serviceId, JSON.stringify(specification)], - ); - await client.query( - `UPDATE deployments SET service_revision_id = $1 - WHERE service_id = $2 AND service_revision_id IS NULL`, - [revisionId, serviceId], - ); -} - -async function finalizeSchema(client: PoolClient) { - await client.query(` - DO $$ - BEGIN - IF EXISTS (SELECT 1 FROM deployments WHERE service_revision_id IS NULL) THEN - RAISE EXCEPTION 'deployments.service_revision_id backfill incomplete'; - END IF; - IF EXISTS (SELECT 1 FROM deployment_ports WHERE container_port IS NULL) THEN - RAISE EXCEPTION 'deployment_ports.container_port backfill incomplete'; - END IF; - END $$ - `); - await client.query(` - CREATE INDEX IF NOT EXISTS service_revisions_service_id_idx - ON service_revisions(service_id); - CREATE INDEX IF NOT EXISTS deployments_service_revision_id_idx - ON deployments(service_revision_id); - ALTER TABLE deployments ALTER COLUMN service_revision_id SET NOT NULL; - ALTER TABLE deployment_ports ALTER COLUMN container_port SET NOT NULL; - `); - await client.query(` - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conname = 'deployments_service_revision_service_fk' - ) THEN - ALTER TABLE deployments - ADD CONSTRAINT deployments_service_revision_service_fk - FOREIGN KEY (service_revision_id, service_id) - REFERENCES service_revisions(id, service_id) ON DELETE NO ACTION; - END IF; - END $$ - `); - await client.query( - "ALTER TABLE deployment_ports DROP COLUMN IF EXISTS service_port_id", - ); -} - -async function main() { - const client = await pool.connect(); - try { - await client.query("BEGIN"); - await client.query( - "SELECT pg_advisory_xact_lock(hashtext('service-revisions-cutover'))", - ); - if (!(await baseSchemaExists(client))) { - await client.query("COMMIT"); - return; - } - if (!(await cutoverIsRequired(client))) { - await client.query("COMMIT"); - return; - } - - const activeRollouts = await client.query<{ count: string }>( - "SELECT count(*)::text AS count FROM rollouts WHERE status IN ('queued', 'in_progress')", - ); - if (Number(activeRollouts.rows[0]?.count ?? 0) > 0) { - throw new Error("cutover requires zero queued or in-progress rollouts"); - } - - await client.query( - "DELETE FROM deployments WHERE traffic_state <> 'active' OR runtime_desired_state = 'removed'", - ); - await prepareSchema(client); - const serviceIds = await client.query<{ service_id: string }>( - "SELECT DISTINCT service_id FROM deployments", - ); - for (const { service_id: serviceId } of serviceIds.rows) { - try { - await captureBootstrapRevision(client, serviceId); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`Service ${serviceId}: ${message}`); - } - } - await finalizeSchema(client); - await client.query("COMMIT"); - console.log(`Backfilled ${serviceIds.rowCount ?? 0} service revisions.`); - } catch (error) { - await client.query("ROLLBACK"); - throw error; - } finally { - client.release(); - await pool.end(); - } -} - -await main(); diff --git a/web/tests/service-revision-cutover.test.ts b/web/tests/service-revision-cutover.test.ts deleted file mode 100644 index 592ea99..0000000 --- a/web/tests/service-revision-cutover.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { buildCutoverServiceRevisionSpec } from "@/lib/service-revision-cutover"; -import type { ServiceRevisionDraft } from "@/lib/service-revision-spec"; - -function liveDraft(): ServiceRevisionDraft { - return { - service: { - name: "API", - image: "current:image", - hostname: "current-host", - stateful: false, - serverlessEnabled: false, - serverlessSleepAfterSeconds: 600, - serverlessWakeTimeoutSeconds: 600, - healthCheckCmd: "current-health", - healthCheckInterval: 10, - healthCheckTimeout: 5, - healthCheckRetries: 3, - healthCheckStartPeriod: 30, - startCommand: "current-command", - resourceCpuLimit: 2, - resourceMemoryLimitMb: 1024, - }, - placements: [{ serverId: "current-server", count: 2 }], - ports: [ - { - port: 8080, - isPublic: true, - domain: "current.example.com", - protocol: "tcp", - externalPort: 443, - tlsPassthrough: true, - }, - ], - secrets: [ - { - key: "TOKEN", - encryptedValue: "ciphertext", - updatedAt: "2026-07-01T00:00:00.000Z", - }, - ], - volumes: [{ name: "current", containerPath: "/current" }], - }; -} - -describe("service revision cutover", () => { - it("uses the legacy deployed snapshot while retaining runtime-only values", () => { - const specification = buildCutoverServiceRevisionSpec({ - liveDraft: liveDraft(), - deployedConfig: { - source: { image: "deployed:image" }, - hostname: "deployed-host", - stateful: true, - replicas: [{ serverId: "deployed-server", count: 1 }], - healthCheck: null, - startCommand: null, - resourceLimits: { cpuCores: 1, memoryMb: 512 }, - ports: [ - { - port: 8080, - isPublic: false, - domain: null, - protocol: "tcp", - tlsPassthrough: false, - }, - ], - serverless: { - enabled: true, - sleepAfterSeconds: 300, - wakeTimeoutSeconds: 120, - }, - secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], - volumes: [{ name: "deployed", containerPath: "/data" }], - }, - }); - - expect(specification).toMatchObject({ - image: "deployed:image", - hostname: "deployed-host", - stateful: true, - placements: [{ serverId: "deployed-server", count: 1 }], - healthCheck: null, - startCommand: null, - resourceLimits: { cpuCores: 1, memoryMb: 512 }, - serverless: { enabled: true }, - ports: [{ containerPort: 8080, externalPort: 443 }], - secrets: [{ key: "TOKEN", encryptedValue: "ciphertext" }], - volumes: [{ name: "deployed", containerPath: "/data" }], - }); - }); - - it("keeps deployed resource limits unset instead of using live draft limits", () => { - const specification = buildCutoverServiceRevisionSpec({ - liveDraft: liveDraft(), - deployedConfig: { - source: { type: "image", image: "deployed:image" }, - replicas: [{ serverId: "deployed-server", count: 1 }], - healthCheck: null, - ports: [], - secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], - }, - }); - - expect(specification.resourceLimits).toEqual({ - cpuCores: null, - memoryMb: null, - }); - }); - - it("rejects undeployed secret changes", () => { - const draft = liveDraft(); - draft.secrets[0].updatedAt = "2026-07-02T00:00:00.000Z"; - - expect(() => - buildCutoverServiceRevisionSpec({ - liveDraft: draft, - deployedConfig: { - source: { type: "image", image: "deployed:image" }, - replicas: [{ serverId: "deployed-server", count: 1 }], - healthCheck: null, - ports: [], - secrets: [{ key: "TOKEN", updatedAt: "2026-07-01T00:00:00.000Z" }], - }, - }), - ).toThrow("Secret TOKEN differs from the deployed snapshot"); - }); -}); From 6f52309838cf437eb207a144d43b91b50b9f2b21 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Tue, 14 Jul 2026 08:02:46 +1000 Subject: [PATCH 08/10] Remove revision cutover instructions --- deployment/README.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/deployment/README.md b/deployment/README.md index e02aaf8..1d60552 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -90,22 +90,6 @@ start, so scaling `WEB_REPLICAS` does not run migrations from every replica. Schema is synced automatically by the one-shot `migrate` service via `drizzle-kit push --force`. This keeps deployment non-interactive, including schema changes Drizzle classifies as data-loss operations such as dropping columns. If schema sync fails, `web` startup is blocked; inspect the failure with `docker compose -f compose.production.yml logs migrate`. -### Immutable service revision cutover - -Before deploying this release, stop rollout producers, verify no rollout is -queued or in progress, and take a PostgreSQL backup. Then run this once against -the database: - -```sql -DELETE FROM deployments; -``` - -Port allocations are removed by cascade. The schema sync can then add the -required revision reference without migrating legacy deployment state. After -the new version starts, redeploy previously active services from their current -configuration. Agents remove the orphaned containers; service volumes and -backups are not deleted. - **Future plan:** Once the schema stabilizes, switch to `drizzle-kit generate` + `drizzle-orm migrate()` with pre-generated SQL migration files. This will eliminate the esbuild/drizzle-kit dependency from the production image. ## Commands From e8496a8ffff95af0448d4e97c8b444fa7d07d449 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Tue, 14 Jul 2026 18:42:31 +1000 Subject: [PATCH 09/10] feat(web): add service revision changes history --- .../services/[serviceId]/changelog/page.tsx | 16 + web/app/(dashboard)/layout-client.tsx | 26 +- web/app/api/services/[id]/revisions/route.ts | 216 +++++++++++++ web/components/builds/build-details.tsx | 1 - .../service/details/changelog-history.tsx | 275 ++++++++++++++++ .../details/service-details-overview.tsx | 10 +- web/components/service/service-canvas.tsx | 2 +- .../service/service-layout-client.tsx | 2 + web/db/schema.ts | 5 + web/lib/service-config.ts | 8 +- web/lib/service-revision-changes.ts | 301 ++++++++++++++++++ web/tests/service-config.test.ts | 9 + web/tests/service-revision-changes.test.ts | 192 +++++++++++ web/tests/service-revisions-route.test.ts | 219 +++++++++++++ 14 files changed, 1248 insertions(+), 34 deletions(-) create mode 100644 web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/changelog/page.tsx create mode 100644 web/app/api/services/[id]/revisions/route.ts create mode 100644 web/components/service/details/changelog-history.tsx create mode 100644 web/lib/service-revision-changes.ts create mode 100644 web/tests/service-revision-changes.test.ts create mode 100644 web/tests/service-revisions-route.test.ts diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/changelog/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/changelog/page.tsx new file mode 100644 index 0000000..521e659 --- /dev/null +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/changelog/page.tsx @@ -0,0 +1,16 @@ +"use client"; + +import { ChangelogHistory } from "@/components/service/details/changelog-history"; +import { useService } from "@/components/service/service-layout-client"; + +export default function ChangelogPage() { + const { service, projectSlug, envName } = useService(); + + return ( + + ); +} diff --git a/web/app/(dashboard)/layout-client.tsx b/web/app/(dashboard)/layout-client.tsx index 1bbc205..9de03ef 100644 --- a/web/app/(dashboard)/layout-client.tsx +++ b/web/app/(dashboard)/layout-client.tsx @@ -1,17 +1,16 @@ "use client"; +import { LogOut, Settings, User } from "lucide-react"; import Image from "next/image"; import Link from "next/link"; import { useRouter } from "next/navigation"; import { useEffect } from "react"; -import { LogOut, Settings, User } from "lucide-react"; import { BreadcrumbDataProvider, useBreadcrumbs, } from "@/components/core/breadcrumb-data"; import { DashboardPageSkeleton } from "@/components/dashboard/dashboard-page-skeleton"; import { OfflineServersBanner } from "@/components/server/offline-servers-banner"; -import { Button } from "@/components/ui/button"; import { DropdownMenu, DropdownMenuContent, @@ -24,13 +23,7 @@ import { import { Toaster } from "@/components/ui/sonner"; import { signOut, useSession } from "@/lib/auth-client"; -function DashboardHeader({ - email, - name, -}: { - email: string; - name: string; -}) { +function DashboardHeader({ email, name }: { email: string; name: string }) { const router = useRouter(); const breadcrumbs = useBreadcrumbs(); const getBreadcrumbKey = ( @@ -43,8 +36,8 @@ function DashboardHeader({ const showEllipsis = breadcrumbs.length > 2; return ( -
-
+
+
- + @@ -183,10 +172,7 @@ export function DashboardLayoutClient({ return (
- +
{children}
diff --git a/web/app/api/services/[id]/revisions/route.ts b/web/app/api/services/[id]/revisions/route.ts new file mode 100644 index 0000000..4a2cfe9 --- /dev/null +++ b/web/app/api/services/[id]/revisions/route.ts @@ -0,0 +1,216 @@ +export const dynamic = "force-dynamic"; + +import { and, desc, eq, inArray, isNull, lt, or, sql } from "drizzle-orm"; +import { db } from "@/db"; +import { rollouts, servers, serviceRevisions, services } from "@/db/schema"; +import { requireRequestSession } from "@/lib/api-auth"; +import { + diffServiceRevisionSpecs, + parseServiceRevisionSpec, + type ServiceRevisionChangelogItem, + type ServiceRevisionChangelogResponse, +} from "@/lib/service-revision-changes"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; + +const PAGE_SIZE = 25; +const REVISION_CURSOR_TIMESTAMP = + /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}(?::?\d{2})?)$/; + +type RevisionCursor = { + createdAt: string; + id: string; +}; + +function encodeCursor(cursor: RevisionCursor): string { + return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); +} + +function decodeCursor(value: string): RevisionCursor | null { + try { + const decoded = JSON.parse( + Buffer.from(value, "base64url").toString("utf8"), + ) as unknown; + if (!decoded || typeof decoded !== "object") return null; + + const cursor = decoded as Partial; + if ( + typeof cursor.createdAt !== "string" || + !REVISION_CURSOR_TIMESTAMP.test(cursor.createdAt) || + Number.isNaN(Date.parse(cursor.createdAt)) || + typeof cursor.id !== "string" || + cursor.id.length === 0 || + cursor.id.length > 200 + ) { + return null; + } + + return { createdAt: cursor.createdAt, id: cursor.id }; + } catch { + return null; + } +} + +export async function GET( + request: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const sessionResult = await requireRequestSession(request); + if (!sessionResult.ok) return sessionResult.response; + + const { id: serviceId } = await params; + const cursorValue = new URL(request.url).searchParams.get("cursor"); + const cursor = cursorValue ? decodeCursor(cursorValue) : null; + if (cursorValue && !cursor) { + return Response.json( + { message: "Invalid revision cursor" }, + { status: 400 }, + ); + } + + const service = await db + .select({ id: services.id }) + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .then((rows) => rows[0]); + if (!service) { + return Response.json({ message: "Service not found" }, { status: 404 }); + } + + const revisions = await db + .select({ + id: serviceRevisions.id, + createdAt: serviceRevisions.createdAt, + cursorCreatedAt: sql`${serviceRevisions.createdAt}::text`, + specification: serviceRevisions.specification, + }) + .from(serviceRevisions) + .where( + and( + eq(serviceRevisions.serviceId, serviceId), + cursor + ? or( + lt( + serviceRevisions.createdAt, + sql`${cursor.createdAt}::timestamptz`, + ), + and( + eq( + serviceRevisions.createdAt, + sql`${cursor.createdAt}::timestamptz`, + ), + lt(serviceRevisions.id, cursor.id), + ), + ) + : undefined, + ), + ) + .orderBy(desc(serviceRevisions.createdAt), desc(serviceRevisions.id)) + .limit(PAGE_SIZE + 1); + + const pageRevisions = revisions.slice(0, PAGE_SIZE); + const revisionIds = pageRevisions.map((revision) => revision.id); + const parsedSpecifications = new Map(); + for (const revision of revisions) { + try { + parsedSpecifications.set( + revision.id, + parseServiceRevisionSpec(revision.specification), + ); + } catch { + // Unsupported or malformed historical revisions remain visible. + } + } + const placementServerIds = [ + ...new Set( + [...parsedSpecifications.values()].flatMap((specification) => + specification.placements.map((placement) => placement.serverId), + ), + ), + ]; + const [serverRows, revisionRollouts] = await Promise.all([ + placementServerIds.length > 0 + ? db + .select({ id: servers.id, name: servers.name }) + .from(servers) + .where(inArray(servers.id, placementServerIds)) + : Promise.resolve([]), + revisionIds.length > 0 + ? db + .select({ + id: rollouts.id, + serviceRevisionId: rollouts.serviceRevisionId, + status: rollouts.status, + createdAt: rollouts.createdAt, + }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.serviceRevisionId, revisionIds), + ), + ) + .orderBy(desc(rollouts.createdAt), desc(rollouts.id)) + : Promise.resolve([]), + ]); + const serverNames = new Map( + serverRows.map((server) => [server.id, server.name] as const), + ); + const rolloutByRevisionId = new Map< + string, + (typeof revisionRollouts)[number] + >(); + for (const rollout of revisionRollouts) { + if ( + rollout.serviceRevisionId && + !rolloutByRevisionId.has(rollout.serviceRevisionId) + ) { + rolloutByRevisionId.set(rollout.serviceRevisionId, rollout); + } + } + + const items: ServiceRevisionChangelogItem[] = pageRevisions.map( + (revision, index) => { + const previous = revisions[index + 1]; + let comparison: ServiceRevisionChangelogItem["comparison"]; + if (!previous) { + comparison = { kind: "initial" }; + } else { + const currentSpecification = parsedSpecifications.get(revision.id); + const previousSpecification = parsedSpecifications.get(previous.id); + if (currentSpecification && previousSpecification) { + comparison = { + kind: "changes", + changes: diffServiceRevisionSpecs( + previousSpecification, + currentSpecification, + serverNames, + ), + }; + } else { + comparison = { kind: "unavailable" }; + } + } + + const rollout = rolloutByRevisionId.get(revision.id); + return { + id: revision.id, + createdAt: revision.createdAt.toISOString(), + comparison, + rollout: rollout ? { id: rollout.id, status: rollout.status } : null, + }; + }, + ); + + const response: ServiceRevisionChangelogResponse = { + revisions: items, + nextCursor: + revisions.length > PAGE_SIZE && pageRevisions.length > 0 + ? encodeCursor({ + createdAt: pageRevisions[pageRevisions.length - 1].cursorCreatedAt, + id: pageRevisions[pageRevisions.length - 1].id, + }) + : null, + }; + + return Response.json(response); +} diff --git a/web/components/builds/build-details.tsx b/web/components/builds/build-details.tsx index dfeef85..6b7aa5b 100644 --- a/web/components/builds/build-details.tsx +++ b/web/components/builds/build-details.tsx @@ -265,7 +265,6 @@ export function BuildDetails({ - Timing
Created diff --git a/web/components/service/details/changelog-history.tsx b/web/components/service/details/changelog-history.tsx new file mode 100644 index 0000000..364db40 --- /dev/null +++ b/web/components/service/details/changelog-history.tsx @@ -0,0 +1,275 @@ +"use client"; + +import { + ArrowRight, + CheckCircle2, + Clock, + GitCommitHorizontal, + Loader2, + RotateCcw, + XCircle, +} from "lucide-react"; +import Link from "next/link"; +import { useMemo } from "react"; +import useSWRInfinite from "swr/infinite"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Empty, + EmptyDescription, + EmptyMedia, + EmptyTitle, +} from "@/components/ui/empty"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { RolloutStatus } from "@/db/types"; +import { formatDateTime, formatRelativeTime } from "@/lib/date"; +import { fetcher } from "@/lib/fetcher"; +import type { + ServiceRevisionChangelogItem, + ServiceRevisionChangelogResponse, +} from "@/lib/service-revision-changes"; + +const STATUS_CONFIG: Record< + RolloutStatus, + { label: string; icon: typeof Clock; className: string } +> = { + queued: { label: "Queued", icon: Clock, className: "text-slate-600" }, + in_progress: { + label: "In progress", + icon: Loader2, + className: "text-blue-600", + }, + completed: { + label: "Completed", + icon: CheckCircle2, + className: "text-emerald-600", + }, + failed: { label: "Failed", icon: XCircle, className: "text-red-600" }, + rolled_back: { + label: "Rolled back", + icon: RotateCcw, + className: "text-orange-600", + }, +}; + +function RolloutBadge({ + rollout, + serviceId, + projectSlug, + envName, +}: { + rollout: NonNullable; + serviceId: string; + projectSlug: string; + envName: string; +}) { + const config = STATUS_CONFIG[rollout.status] ?? { + label: rollout.status, + icon: Clock, + className: "text-muted-foreground", + }; + const Icon = config.icon; + + return ( + + } + > + + {config.label} + + ); +} + +function ChangelogSkeleton() { + return ( +
+ {[1, 2, 3].map((item) => ( +
+ + +
+ ))} +
+ ); +} + +function RevisionChanges({ item }: { item: ServiceRevisionChangelogItem }) { + if (item.comparison.kind === "initial") { + return ( +

+ Initial service configuration captured. +

+ ); + } + + if (item.comparison.kind === "unavailable") { + return ( +

+ Changes are unavailable for this revision format. +

+ ); + } + + if (item.comparison.changes.length === 0) { + return ( +

+ No configuration changes — redeploy. +

+ ); + } + + return ( +
+ {item.comparison.changes.map((change) => ( +
+
{change.field}
+
+ {change.from} + + {change.to} +
+
+ ))} +
+ ); +} + +export function ChangelogHistory({ + serviceId, + projectSlug, + envName, +}: { + serviceId: string; + projectSlug: string; + envName: string; +}) { + const { data, error, isLoading, isValidating, mutate, size, setSize } = + useSWRInfinite( + (pageIndex, previousPage) => { + if (previousPage && !previousPage.nextCursor) return null; + const cursor = pageIndex === 0 ? null : previousPage?.nextCursor; + return `/api/services/${serviceId}/revisions${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ""}`; + }, + fetcher, + { + refreshInterval: (pages) => + pages?.some((page) => + page.revisions.some( + (revision) => + revision.rollout?.status === "queued" || + revision.rollout?.status === "in_progress", + ), + ) + ? 3000 + : 0, + revalidateOnFocus: true, + }, + ); + + const revisions = useMemo(() => { + const byId = new Map(); + for (const page of data ?? []) { + for (const revision of page.revisions) byId.set(revision.id, revision); + } + return [...byId.values()]; + }, [data]); + const hasMore = data?.[data.length - 1]?.nextCursor != null; + const isLoadingMore = isValidating && Boolean(data?.[size - 1] === undefined); + + if (isLoading) return ; + + if (error) { + return ( + + + + + Unable to load changelog + + Revision history could not be loaded. Try again. + + + + ); + } + + return ( +
+ {revisions.length === 0 ? ( + + + + + No revisions yet + + Deploy this service to create its first revision. + + + ) : ( +
+ {revisions.map((revision) => ( +
+
+
+
+ {revision.comparison.kind === "initial" + ? "Initial revision" + : revision.comparison.kind === "changes" && + revision.comparison.changes.length === 0 + ? "Redeployed" + : "Configuration updated"} +
+
+ {formatRelativeTime(revision.createdAt)} ·{" "} + {revision.id.slice(0, 8)} +
+
+ {revision.rollout ? ( + + ) : null} +
+ +
+ ))} +
+ )} + + {hasMore ? ( +
+ +
+ ) : null} +
+ ); +} diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index b9cd6f4..a5d584b 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -316,7 +316,7 @@ export function ServiceMetricsPanel({ margin={{ top: 8, right: fixedMode ? 48 : 4, - left: fixedMode ? 0 : getYAxisMargin(chartMode), + left: 0, bottom: 0, }} > @@ -338,6 +338,7 @@ export function ServiceMetricsPanel({ className="text-xs" /> @@ -1257,13 +1258,6 @@ function formatRateTick(value: number): string { return value.toFixed(1); } -function getYAxisMargin(mode: ServiceChartMode): number { - if (mode === "traffic") return -4; - if (mode === "latency") return -12; - if (mode === "resources") return -24; - return -20; -} - function formatRequestCount(value: number): string { if (!Number.isFinite(value)) return "-"; diff --git a/web/components/service/service-canvas.tsx b/web/components/service/service-canvas.tsx index 1e82549..c5aa536 100644 --- a/web/components/service/service-canvas.tsx +++ b/web/components/service/service-canvas.tsx @@ -856,7 +856,7 @@ export function ServiceCanvas({
-
+
{ diff --git a/web/db/schema.ts b/web/db/schema.ts index 5727e28..4e158cf 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -569,6 +569,11 @@ export const serviceRevisions = pgTable( table.serviceId, ), index("service_revisions_service_id_idx").on(table.serviceId), + index("service_revisions_service_created_id_idx").on( + table.serviceId, + table.createdAt, + table.id, + ), ], ); diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index ece73e8..22d5618 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -401,8 +401,8 @@ export function diffConfigs( }); } - const deployedCpu = deployed.resourceLimits?.cpuCores; - const currentCpu = current.resourceLimits?.cpuCores; + const deployedCpu = deployed.resourceLimits?.cpuCores ?? null; + const currentCpu = current.resourceLimits?.cpuCores ?? null; if (deployedCpu !== currentCpu) { changes.push({ field: "CPU limit", @@ -411,8 +411,8 @@ export function diffConfigs( }); } - const deployedMemory = deployed.resourceLimits?.memoryMb; - const currentMemory = current.resourceLimits?.memoryMb; + const deployedMemory = deployed.resourceLimits?.memoryMb ?? null; + const currentMemory = current.resourceLimits?.memoryMb ?? null; if (deployedMemory !== currentMemory) { changes.push({ field: "Memory limit", diff --git a/web/lib/service-revision-changes.ts b/web/lib/service-revision-changes.ts new file mode 100644 index 0000000..45516ed --- /dev/null +++ b/web/lib/service-revision-changes.ts @@ -0,0 +1,301 @@ +import { z } from "zod"; +import type { RolloutStatus } from "@/db/types"; +import type { + ServiceRevisionHealthCheck, + ServiceRevisionPort, + ServiceRevisionSpec, +} from "@/lib/service-revision-spec"; + +const serviceRevisionSpecSchema = z.strictObject({ + schemaVersion: z.literal(1), + image: z.string(), + hostname: z.string(), + stateful: z.boolean(), + serverless: z.strictObject({ + enabled: z.boolean(), + sleepAfterSeconds: z.number(), + wakeTimeoutSeconds: z.number(), + }), + healthCheck: z + .strictObject({ + cmd: z.string(), + interval: z.number(), + timeout: z.number(), + retries: z.number(), + startPeriod: z.number(), + }) + .nullable(), + startCommand: z.string().nullable(), + resourceLimits: z.strictObject({ + cpuCores: z.number().nullable(), + memoryMb: z.number().nullable(), + }), + placements: z.array( + z.strictObject({ serverId: z.string(), count: z.number() }), + ), + ports: z.array( + z.strictObject({ + containerPort: z.number(), + isPublic: z.boolean(), + domain: z.string().nullable(), + protocol: z.enum(["http", "tcp", "udp"]), + externalPort: z.number().nullable(), + tlsPassthrough: z.boolean(), + }), + ), + secrets: z.array( + z.strictObject({ + key: z.string(), + encryptedValue: z.string(), + updatedAt: z.string(), + }), + ), + volumes: z.array( + z.strictObject({ name: z.string(), containerPath: z.string() }), + ), +}); + +export type ServiceRevisionChange = { + field: string; + from: string; + to: string; +}; + +export type ServiceRevisionComparison = + | { kind: "initial" } + | { kind: "changes"; changes: ServiceRevisionChange[] } + | { kind: "unavailable" }; + +export type ServiceRevisionChangelogItem = { + id: string; + createdAt: string; + comparison: ServiceRevisionComparison; + rollout: { + id: string; + status: RolloutStatus; + } | null; +}; + +export type ServiceRevisionChangelogResponse = { + revisions: ServiceRevisionChangelogItem[]; + nextCursor: string | null; +}; + +export function parseServiceRevisionSpec(value: unknown): ServiceRevisionSpec { + return serviceRevisionSpecSchema.parse(value); +} + +function compareStrings(a: string, b: string): number { + return a.localeCompare(b, "en"); +} + +function enabled(value: boolean): string { + return value ? "Enabled" : "Disabled"; +} + +function healthCheckDescription( + healthCheck: ServiceRevisionHealthCheck, +): string { + return `${healthCheck.cmd} (interval ${healthCheck.interval}s, timeout ${healthCheck.timeout}s, retries ${healthCheck.retries}, start period ${healthCheck.startPeriod}s)`; +} + +function portIdentity(port: ServiceRevisionPort): string { + return `${port.containerPort}/${port.protocol}/${port.domain ?? "(none)"}`; +} + +function portLabel(port: ServiceRevisionPort): string { + return `${port.containerPort}/${port.protocol}`; +} + +function portDescription(port: ServiceRevisionPort): string { + return [ + `container ${port.containerPort}`, + `protocol ${port.protocol}`, + port.isPublic ? "public" : "internal", + `domain ${port.domain ?? "(none)"}`, + `external ${port.externalPort ?? "(default)"}`, + `TLS passthrough ${enabled(port.tlsPassthrough).toLowerCase()}`, + ].join(", "); +} + +/** Compare two immutable v1 specifications without requiring browser APIs. */ +export function diffServiceRevisionSpecs( + previous: ServiceRevisionSpec, + current: ServiceRevisionSpec, + serverNames: ReadonlyMap = new Map(), +): ServiceRevisionChange[] { + const changes: ServiceRevisionChange[] = []; + const add = (field: string, from: string, to: string) => { + if (from !== to) changes.push({ field, from, to }); + }; + + add("Image", previous.image, current.image); + add("Hostname", previous.hostname, current.hostname); + add( + "Service type", + previous.stateful ? "Stateful" : "Stateless", + current.stateful ? "Stateful" : "Stateless", + ); + add( + "Serverless", + enabled(previous.serverless.enabled), + enabled(current.serverless.enabled), + ); + add( + "Serverless sleep timeout", + `${previous.serverless.sleepAfterSeconds}s`, + `${current.serverless.sleepAfterSeconds}s`, + ); + add( + "Serverless wake timeout", + `${previous.serverless.wakeTimeoutSeconds}s`, + `${current.serverless.wakeTimeoutSeconds}s`, + ); + + if (previous.healthCheck === null || current.healthCheck === null) { + add( + "Health check", + previous.healthCheck + ? healthCheckDescription(previous.healthCheck) + : "(none)", + current.healthCheck + ? healthCheckDescription(current.healthCheck) + : "(none)", + ); + } else { + add( + "Health check command", + previous.healthCheck.cmd, + current.healthCheck.cmd, + ); + add( + "Health check interval", + `${previous.healthCheck.interval}s`, + `${current.healthCheck.interval}s`, + ); + add( + "Health check timeout", + `${previous.healthCheck.timeout}s`, + `${current.healthCheck.timeout}s`, + ); + add( + "Health check retries", + String(previous.healthCheck.retries), + String(current.healthCheck.retries), + ); + add( + "Health check start period", + `${previous.healthCheck.startPeriod}s`, + `${current.healthCheck.startPeriod}s`, + ); + } + + add( + "Start command", + previous.startCommand ?? "(default)", + current.startCommand ?? "(default)", + ); + add( + "CPU limit", + previous.resourceLimits.cpuCores === null + ? "(no limit)" + : `${previous.resourceLimits.cpuCores} cores`, + current.resourceLimits.cpuCores === null + ? "(no limit)" + : `${current.resourceLimits.cpuCores} cores`, + ); + add( + "Memory limit", + previous.resourceLimits.memoryMb === null + ? "(no limit)" + : `${previous.resourceLimits.memoryMb} MB`, + current.resourceLimits.memoryMb === null + ? "(no limit)" + : `${current.resourceLimits.memoryMb} MB`, + ); + + const previousPlacements = new Map( + previous.placements.map((placement) => [placement.serverId, placement]), + ); + const currentPlacements = new Map( + current.placements.map((placement) => [placement.serverId, placement]), + ); + for (const serverId of [ + ...new Set([...previousPlacements.keys(), ...currentPlacements.keys()]), + ].sort(compareStrings)) { + const before = previousPlacements.get(serverId); + const after = currentPlacements.get(serverId); + const serverName = serverNames.get(serverId)?.trim(); + add( + serverName + ? `${serverName} replicas` + : `Deleted server (${serverId.slice(0, 8)}) replicas`, + before ? `${before.count} replicas` : "(none)", + after ? `${after.count} replicas` : "(removed)", + ); + } + + const previousPorts = new Map( + previous.ports.map((port) => [portIdentity(port), port]), + ); + const currentPorts = new Map( + current.ports.map((port) => [portIdentity(port), port]), + ); + for (const identity of [ + ...new Set([...previousPorts.keys(), ...currentPorts.keys()]), + ].sort(compareStrings)) { + const before = previousPorts.get(identity); + const after = currentPorts.get(identity); + const port = after ?? before; + add( + port ? `Port ${portLabel(port)}` : "Port", + before ? portDescription(before) : "(none)", + after ? portDescription(after) : "(removed)", + ); + } + + const previousSecrets = new Map( + previous.secrets.map((secret) => [secret.key, secret]), + ); + const currentSecrets = new Map( + current.secrets.map((secret) => [secret.key, secret]), + ); + for (const key of [ + ...new Set([...previousSecrets.keys(), ...currentSecrets.keys()]), + ].sort(compareStrings)) { + const before = previousSecrets.get(key); + const after = currentSecrets.get(key); + if (!before && after) + changes.push({ field: "Secret", from: "(none)", to: `${key} (added)` }); + else if (before && !after) + changes.push({ field: "Secret", from: key, to: "(removed)" }); + else if ( + before && + after && + (before.encryptedValue !== after.encryptedValue || + before.updatedAt !== after.updatedAt) + ) { + changes.push({ field: "Secret", from: key, to: `${key} (updated)` }); + } + } + + const previousVolumes = new Map( + previous.volumes.map((volume) => [volume.name, volume]), + ); + const currentVolumes = new Map( + current.volumes.map((volume) => [volume.name, volume]), + ); + for (const name of [ + ...new Set([...previousVolumes.keys(), ...currentVolumes.keys()]), + ].sort(compareStrings)) { + const before = previousVolumes.get(name); + const after = currentVolumes.get(name); + add( + `Volume ${name}`, + before?.containerPath ?? "(none)", + after?.containerPath ?? "(removed)", + ); + } + + return changes; +} diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index 8609fed..10a5434 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -66,6 +66,15 @@ describe("service config", () => { ]); }); + it("does not report null and omitted resource limits as pending", () => { + const deployed = deployedConfig({ + resourceLimits: { cpuCores: null, memoryMb: null }, + }); + const current = deployedConfig(); + + expect(diffConfigs(deployed, current)).toEqual([]); + }); + it("reports serverless changes as pending config", () => { const changes = diffConfigs(deployedConfig(), { source: { type: "image", image: "nginx" }, diff --git a/web/tests/service-revision-changes.test.ts b/web/tests/service-revision-changes.test.ts new file mode 100644 index 0000000..ba956c5 --- /dev/null +++ b/web/tests/service-revision-changes.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vitest"; +import { diffServiceRevisionSpecs } from "@/lib/service-revision-changes"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; + +function spec(): ServiceRevisionSpec { + return { + schemaVersion: 1, + image: "app:v1", + hostname: "app", + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 60, + }, + healthCheck: { + cmd: "curl /health", + interval: 10, + timeout: 5, + retries: 3, + startPeriod: 30, + }, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "server-a", count: 1 }], + ports: [ + { + containerPort: 80, + protocol: "http", + isPublic: false, + domain: null, + externalPort: null, + tlsPassthrough: false, + }, + ], + secrets: [ + { key: "TOKEN", encryptedValue: "cipher-one", updatedAt: "2026-01-01" }, + ], + volumes: [{ name: "data", containerPath: "/data" }], + }; +} + +describe("diffServiceRevisionSpecs", () => { + it("reports representative scalar and health check changes", () => { + const previous = spec(); + const current = structuredClone(previous); + current.image = "app:v2"; + current.hostname = "new-app"; + current.stateful = true; + current.serverless = { + enabled: true, + sleepAfterSeconds: 600, + wakeTimeoutSeconds: 90, + }; + current.healthCheck = { + cmd: "wget /ready", + interval: 20, + timeout: 8, + retries: 5, + startPeriod: 40, + }; + current.startCommand = "npm start"; + current.resourceLimits = { cpuCores: 2, memoryMb: 512 }; + + expect(diffServiceRevisionSpecs(previous, current)).toEqual( + expect.arrayContaining([ + { field: "Image", from: "app:v1", to: "app:v2" }, + { field: "Health check start period", from: "30s", to: "40s" }, + { field: "Start command", from: "(default)", to: "npm start" }, + { field: "CPU limit", from: "(no limit)", to: "2 cores" }, + { field: "Memory limit", from: "(no limit)", to: "512 MB" }, + ]), + ); + }); + + it("compares collections and includes every port property", () => { + const previous = spec(); + const current = structuredClone(previous); + current.placements[0].count = 2; + current.volumes[0].containerPath = "/mnt/data"; + current.ports[0] = { + containerPort: 80, + protocol: "http", + isPublic: true, + domain: "app.test", + externalPort: 443, + tlsPassthrough: true, + }; + current.ports.push({ + containerPort: 80, + protocol: "tcp", + isPublic: false, + domain: null, + externalPort: 8080, + tlsPassthrough: false, + }); + + const changes = diffServiceRevisionSpecs( + previous, + current, + new Map([["server-a", "Sydney"]]), + ); + expect(changes).toContainEqual({ + field: "Sydney replicas", + from: "1 replicas", + to: "2 replicas", + }); + expect(changes).toContainEqual({ + field: "Volume data", + from: "/data", + to: "/mnt/data", + }); + expect(changes).toContainEqual({ + field: "Port 80/http", + from: "container 80, protocol http, internal, domain (none), external (default), TLS passthrough disabled", + to: "(removed)", + }); + expect(changes).toContainEqual({ + field: "Port 80/http", + from: "(none)", + to: "container 80, protocol http, public, domain app.test, external 443, TLS passthrough enabled", + }); + expect(changes).toContainEqual( + expect.objectContaining({ field: "Port 80/tcp" }), + ); + }); + + it("identifies deleted placement servers without exposing full IDs", () => { + const previous = spec(); + const current = structuredClone(previous); + current.placements[0].count = 2; + + expect(diffServiceRevisionSpecs(previous, current)).toContainEqual({ + field: "Deleted server (server-a) replicas", + from: "1 replicas", + to: "2 replicas", + }); + }); + + it("never exposes secret ciphertext while detecting additions, updates, and removals", () => { + const previous = spec(); + previous.secrets.push({ + key: "OLD", + encryptedValue: "do-not-leak-old", + updatedAt: "1", + }); + const current = structuredClone(previous); + current.secrets = [ + { + key: "TOKEN", + encryptedValue: "do-not-leak-new", + updatedAt: "2026-01-01", + }, + { key: "NEW", encryptedValue: "do-not-leak-added", updatedAt: "1" }, + ]; + + const changes = diffServiceRevisionSpecs(previous, current); + expect(changes).toEqual([ + { field: "Secret", from: "(none)", to: "NEW (added)" }, + { field: "Secret", from: "OLD", to: "(removed)" }, + { field: "Secret", from: "TOKEN", to: "TOKEN (updated)" }, + ]); + expect(JSON.stringify(changes)).not.toContain("do-not-leak"); + }); + + it("ignores canonical-equivalent array ordering and identical specs", () => { + const previous = spec(); + previous.placements.push({ serverId: "server-b", count: 2 }); + previous.ports.push({ + containerPort: 53, + protocol: "udp", + isPublic: false, + domain: null, + externalPort: null, + tlsPassthrough: false, + }); + previous.secrets.push({ + key: "OTHER", + encryptedValue: "cipher-two", + updatedAt: "2", + }); + previous.volumes.push({ name: "logs", containerPath: "/logs" }); + const reordered = structuredClone(previous); + reordered.placements.reverse(); + reordered.ports.reverse(); + reordered.secrets.reverse(); + reordered.volumes.reverse(); + + expect(diffServiceRevisionSpecs(previous, reordered)).toEqual([]); + expect(diffServiceRevisionSpecs(previous, previous)).toEqual([]); + }); +}); diff --git a/web/tests/service-revisions-route.test.ts b/web/tests/service-revisions-route.test.ts new file mode 100644 index 0000000..7286125 --- /dev/null +++ b/web/tests/service-revisions-route.test.ts @@ -0,0 +1,219 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; + +const mocks = vi.hoisted(() => { + const queryResults: unknown[][] = []; + + function createQuery(result: unknown[]) { + const query = { + from: vi.fn(() => query), + where: vi.fn(() => query), + orderBy: vi.fn(() => query), + limit: vi.fn(() => query), + // biome-ignore lint/suspicious/noThenProperty: Drizzle query builders are awaitable. + then: ( + resolve: (value: unknown[]) => unknown, + reject?: (reason: unknown) => unknown, + ) => Promise.resolve(result).then(resolve, reject), + }; + return query; + } + + return { + queryResults, + db: { + select: vi.fn(() => createQuery(queryResults.shift() ?? [])), + }, + requireRequestSession: vi.fn(), + }; +}); + +vi.mock("@/db", () => ({ db: mocks.db })); +vi.mock("@/lib/api-auth", () => ({ + requireRequestSession: mocks.requireRequestSession, +})); + +import { GET } from "@/app/api/services/[id]/revisions/route"; + +function revisionSpec( + image = "app:v1", + encryptedValue = "cipher", +): ServiceRevisionSpec { + return { + schemaVersion: 1, + image, + hostname: "app", + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "server-1", count: 1 }], + ports: [], + secrets: [{ key: "TOKEN", encryptedValue, updatedAt: "2026-01-01" }], + volumes: [], + }; +} + +function request(cursor?: string) { + const url = new URL("http://localhost/api/services/service-1/revisions"); + if (cursor) url.searchParams.set("cursor", cursor); + return GET(new Request(url), { + params: Promise.resolve({ id: "service-1" }), + }); +} + +describe("service revisions route", () => { + beforeEach(() => { + mocks.queryResults.length = 0; + mocks.db.select.mockClear(); + mocks.requireRequestSession.mockReset(); + mocks.requireRequestSession.mockResolvedValue({ + ok: true, + session: { user: { id: "user-1" } }, + }); + }); + + it("requires authentication", async () => { + mocks.requireRequestSession.mockResolvedValue({ + ok: false, + response: Response.json({ message: "Unauthorized" }, { status: 401 }), + }); + + const response = await request(); + + expect(response.status).toBe(401); + expect(mocks.db.select).not.toHaveBeenCalled(); + }); + + it("rejects malformed cursors before querying the service", async () => { + const response = await request("not-a-cursor"); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + message: "Invalid revision cursor", + }); + expect(mocks.db.select).not.toHaveBeenCalled(); + }); + + it("rejects parseable timestamps that PostgreSQL may not accept", async () => { + const cursor = Buffer.from( + JSON.stringify({ + createdAt: "Mon Jul 13 2026 02:00:00 GMT+0000", + id: "revision-1", + }), + ).toString("base64url"); + + const response = await request(cursor); + + expect(response.status).toBe(400); + expect(mocks.db.select).not.toHaveBeenCalled(); + }); + + it("returns safe changes and rollout metadata", async () => { + mocks.queryResults.push( + [{ id: "service-1" }], + [ + { + id: "revision-2", + createdAt: new Date("2026-07-13T02:00:00Z"), + cursorCreatedAt: "2026-07-13 02:00:00+00", + specification: revisionSpec("app:v2", "secret-new"), + }, + { + id: "revision-1", + createdAt: new Date("2026-07-13T01:00:00Z"), + cursorCreatedAt: "2026-07-13 01:00:00+00", + specification: revisionSpec("app:v1", "secret-old"), + }, + ], + [{ id: "server-1", name: "Sydney" }], + [ + { + id: "rollout-2", + serviceRevisionId: "revision-2", + status: "completed", + createdAt: new Date("2026-07-13T02:00:00Z"), + }, + ], + ); + + const response = await request(); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.revisions[0]).toMatchObject({ + id: "revision-2", + rollout: { id: "rollout-2", status: "completed" }, + comparison: { + kind: "changes", + changes: expect.arrayContaining([ + { field: "Image", from: "app:v1", to: "app:v2" }, + { field: "Secret", from: "TOKEN", to: "TOKEN (updated)" }, + ]), + }, + }); + expect(body.revisions[1].comparison).toEqual({ kind: "initial" }); + expect(JSON.stringify(body)).not.toContain("secret-new"); + expect(JSON.stringify(body)).not.toContain("secret-old"); + expect(JSON.stringify(body)).not.toContain("specification"); + }); + + it("uses the extra revision as the page boundary comparison", async () => { + const revisions = Array.from({ length: 26 }, (_, index) => ({ + id: `revision-${String(26 - index).padStart(2, "0")}`, + createdAt: new Date(Date.UTC(2026, 6, 13, 2, 0, 26 - index)), + cursorCreatedAt: `2026-07-13 02:00:${String(26 - index).padStart(2, "0")}.123456+00`, + specification: revisionSpec(`app:v${26 - index}`), + })); + mocks.queryResults.push([{ id: "service-1" }], revisions, [], []); + + const response = await request(); + const body = await response.json(); + + expect(body.revisions).toHaveLength(25); + expect(body.nextCursor).toEqual(expect.any(String)); + expect( + JSON.parse(Buffer.from(body.nextCursor, "base64url").toString("utf8")), + ).toMatchObject({ createdAt: expect.stringContaining(".123456") }); + expect(body.revisions[24].comparison).toMatchObject({ + kind: "changes", + changes: [{ field: "Image", from: "app:v1", to: "app:v2" }], + }); + }); + + it("rejects malformed v1 specifications without exposing their values", async () => { + mocks.queryResults.push( + [{ id: "service-1" }], + [ + { + id: "revision-2", + createdAt: new Date("2026-07-13T02:00:00Z"), + cursorCreatedAt: "2026-07-13 02:00:00+00", + specification: { + ...revisionSpec(), + image: { encryptedValue: "must-not-leak" }, + }, + }, + { + id: "revision-1", + createdAt: new Date("2026-07-13T01:00:00Z"), + cursorCreatedAt: "2026-07-13 01:00:00+00", + specification: revisionSpec(), + }, + ], + [], + [], + ); + + const response = await request(); + const body = await response.json(); + + expect(body.revisions[0].comparison).toEqual({ kind: "unavailable" }); + expect(JSON.stringify(body)).not.toContain("must-not-leak"); + }); +}); From 4365e4da24984d698e8faec2d897d721956221ba Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Tue, 14 Jul 2026 20:22:02 +1000 Subject: [PATCH 10/10] fix(web): align auth schema table names --- web/db/schema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/db/schema.ts b/web/db/schema.ts index 4e158cf..7f3ed25 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -96,7 +96,7 @@ export const verification = pgTable( ); export const twoFactor = pgTable( - "twoFactor", + "two_factor", { id: text("id").primaryKey(), secret: text("secret").notNull(), @@ -117,7 +117,7 @@ export const twoFactor = pgTable( ); export const deviceCode = pgTable( - "deviceCode", + "device_code", { id: text("id").primaryKey(), deviceCode: text("device_code").notNull(),