diff --git a/agent/internal/agent/backup.go b/agent/internal/agent/backup.go index 5b121c7a..e5272563 100644 --- a/agent/internal/agent/backup.go +++ b/agent/internal/agent/backup.go @@ -101,7 +101,10 @@ func (a *Agent) processVolumeBackup(backupID, serviceID, containerID, volumeName } } - tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("backup-%s.tar.gz", backupID)) + tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("backup-%s.tar.gz", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err)) + } defer os.Remove(tarPath) size, checksum, err := createTarGzWithChecksum(volumePath, tarPath) @@ -158,7 +161,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return err } - tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-%s.tar.gz", backupID)) + tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-%s.tar.gz", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err)) + } defer os.Remove(tarPath) if !strings.HasSuffix(storagePath, ".tar.gz") { @@ -185,7 +191,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return reportFailure(fmt.Errorf("checksum mismatch: expected %s, got %s", expectedChecksum, checksum)) } - tempExtractPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-extract-%s", backupID)) + tempExtractPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-extract-%s", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp extract path: %w", err)) + } defer os.RemoveAll(tempExtractPath) if err := os.MkdirAll(tempExtractPath, 0755); err != nil { @@ -258,6 +267,19 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return nil } +func tempArtifactPath(dataDir, name string) (string, error) { + if name == "" || name != filepath.Base(name) { + return "", fmt.Errorf("invalid temp artifact name: %s", name) + } + + tmpDir := filepath.Join(dataDir, "tmp") + if err := os.MkdirAll(tmpDir, 0700); err != nil { + return "", err + } + + return filepath.Join(tmpDir, name), nil +} + func moveDir(src, dst string) error { if err := os.Rename(src, dst); err == nil { return nil diff --git a/agent/internal/agent/backup_test.go b/agent/internal/agent/backup_test.go new file mode 100644 index 00000000..7a9a127c --- /dev/null +++ b/agent/internal/agent/backup_test.go @@ -0,0 +1,32 @@ +package agent + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTempArtifactPathStaysUnderAgentDataDir(t *testing.T) { + dataDir := t.TempDir() + + path, err := tempArtifactPath(dataDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz") + if err != nil { + t.Fatal(err) + } + + expectedPrefix := filepath.Join(dataDir, "tmp") + string(os.PathSeparator) + if !strings.HasPrefix(path, expectedPrefix) { + t.Fatalf("expected %s to be under %s", path, expectedPrefix) + } + + if _, err := os.Stat(filepath.Join(dataDir, "tmp")); err != nil { + t.Fatalf("expected temp directory to exist: %v", err) + } +} + +func TestTempArtifactPathRejectsNestedNames(t *testing.T) { + if _, err := tempArtifactPath(t.TempDir(), "../backup.tar.gz"); err == nil { + t.Fatal("expected nested temp artifact name to be rejected") + } +} diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index 9b39c419..ac10ee3c 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -44,6 +44,7 @@ func (a *Agent) Run(ctx context.Context) { var cleanupTickerC <-chan time.Time if a.Builder != nil { + go a.RunBuildCleanup() cleanupTicker := time.NewTicker(BuildCleanupInterval) defer cleanupTicker.Stop() cleanupTickerC = cleanupTicker.C diff --git a/agent/internal/build/build.go b/agent/internal/build/build.go index b55f2816..00c343d1 100644 --- a/agent/internal/build/build.go +++ b/agent/internal/build/build.go @@ -5,11 +5,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "log" "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -43,6 +45,13 @@ type Builder struct { logSender LogSender } +const ( + staleBuildDirAge = 1 * time.Hour + staleTempArtifactAge = 24 * time.Hour +) + +var managedTempArtifactPattern = regexp.MustCompile(`^(backup|restore)-[0-9a-fA-F-]{36}\.tar\.gz$|^restore-extract-[0-9a-fA-F-]{36}$`) + func NewBuilder(dataDir string, logSender LogSender) *Builder { return &Builder{ dataDir: dataDir, @@ -408,8 +417,26 @@ func (b *Builder) sendLog(config *Config, message string) { } func (b *Builder) Cleanup() error { - buildsDir := filepath.Join(b.dataDir, "builds") + now := time.Now() + var cleanupErrors []error + + if err := cleanupStaleBuildDirs(filepath.Join(b.dataDir, "builds"), now.Add(-staleBuildDirAge)); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + + if err := cleanupManagedTempArtifacts(filepath.Join(b.dataDir, "tmp"), now.Add(-staleTempArtifactAge)); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + + log.Printf("[build:cleanup] pruning unused images") + if err := container.ImagePrune(); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + return errors.Join(cleanupErrors...) +} + +func cleanupStaleBuildDirs(buildsDir string, cutoff time.Time) error { entries, err := os.ReadDir(buildsDir) if err != nil { if os.IsNotExist(err) { @@ -418,7 +445,7 @@ func (b *Builder) Cleanup() error { return err } - cutoff := time.Now().Add(-1 * time.Hour) + var cleanupErrors []error for _, entry := range entries { if !entry.IsDir() { continue @@ -432,14 +459,47 @@ func (b *Builder) Cleanup() error { if info.ModTime().Before(cutoff) { path := filepath.Join(buildsDir, entry.Name()) log.Printf("[build:cleanup] removing stale build dir: %s", entry.Name()) - os.RemoveAll(path) + if err := os.RemoveAll(path); err != nil { + cleanupErrors = append(cleanupErrors, err) + } } } - log.Printf("[build:cleanup] pruning unused images") - container.ImagePrune() + return errors.Join(cleanupErrors...) +} - return nil +func cleanupManagedTempArtifacts(tmpDir string, cutoff time.Time) error { + entries, err := os.ReadDir(tmpDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var cleanupErrors []error + for _, entry := range entries { + if !managedTempArtifactPattern.MatchString(entry.Name()) { + continue + } + + info, err := entry.Info() + if err != nil { + continue + } + + if info.ModTime().After(cutoff) { + continue + } + + path := filepath.Join(tmpDir, entry.Name()) + log.Printf("[build:cleanup] removing stale temp artifact: %s", entry.Name()) + if err := os.RemoveAll(path); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + } + + return errors.Join(cleanupErrors...) } func CheckPrerequisites() error { diff --git a/agent/internal/build/build_test.go b/agent/internal/build/build_test.go new file mode 100644 index 00000000..4efea471 --- /dev/null +++ b/agent/internal/build/build_test.go @@ -0,0 +1,92 @@ +package build + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestCleanupManagedTempArtifactsRemovesOnlyOldManagedArtifacts(t *testing.T) { + tmpDir := t.TempDir() + oldTime := time.Now().Add(-48 * time.Hour) + cutoff := time.Now().Add(-24 * time.Hour) + + oldArchive := filepath.Join(tmpDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz") + youngArchive := filepath.Join(tmpDir, "restore-22222222-2222-2222-2222-222222222222.tar.gz") + unmanagedArchive := filepath.Join(tmpDir, "backup-not-a-build-id.tar.gz") + oldExtractDir := filepath.Join(tmpDir, "restore-extract-33333333-3333-3333-3333-333333333333") + + if err := os.WriteFile(oldArchive, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(youngArchive, []byte("young"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unmanagedArchive, []byte("unmanaged"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(oldExtractDir, 0700); err != nil { + t.Fatal(err) + } + + for _, path := range []string{oldArchive, unmanagedArchive, oldExtractDir} { + if err := os.Chtimes(path, oldTime, oldTime); err != nil { + t.Fatal(err) + } + } + + if err := cleanupManagedTempArtifacts(tmpDir, cutoff); err != nil { + t.Fatal(err) + } + + assertMissing(t, oldArchive) + assertMissing(t, oldExtractDir) + assertExists(t, youngArchive) + assertExists(t, unmanagedArchive) +} + +func TestCleanupStaleBuildDirsRemovesOnlyOldDirectories(t *testing.T) { + buildsDir := t.TempDir() + oldTime := time.Now().Add(-2 * time.Hour) + cutoff := time.Now().Add(-1 * time.Hour) + + oldDir := filepath.Join(buildsDir, "old-build") + youngDir := filepath.Join(buildsDir, "young-build") + filePath := filepath.Join(buildsDir, "not-a-dir") + + if err := os.Mkdir(oldDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(youngDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filePath, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(oldDir, oldTime, oldTime); err != nil { + t.Fatal(err) + } + + if err := cleanupStaleBuildDirs(buildsDir, cutoff); err != nil { + t.Fatal(err) + } + + assertMissing(t, oldDir) + assertExists(t, youngDir) + assertExists(t, filePath) +} + +func assertExists(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected %s to exist: %v", path, err) + } +} + +func assertMissing(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected %s to be removed, got err=%v", path, err) + } +} diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index 21a8ba5d..912e694f 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -456,8 +456,12 @@ func writeDockerConfig(registryURL, username, password string) error { return nil } -func ImagePrune() { - exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h").Run() +func ImagePrune() error { + cmd := exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to prune images: %s: %w", string(output), err) + } + return nil } type podmanContainer struct { diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 9d513fbc..8ec46747 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -109,7 +109,6 @@ type ServerlessRoute struct { Port int `json:"port"` SleepAfterSeconds int `json:"sleepAfterSeconds"` WakeTimeoutSeconds int `json:"wakeTimeoutSeconds"` - MinReadyReplicas int `json:"minReadyReplicas"` LocalDeploymentIDs []string `json:"localDeploymentIds"` Upstreams []ServerlessUpstream `json:"upstreams"` } diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index 4d8f8c18..c7002585 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -236,7 +236,7 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, } wakeStartedAt := time.Now() - if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && hasAlwaysOnUpstream(ready)) { + if len(ready) > 0 { log.Printf( "[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=background", host, @@ -380,11 +380,11 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi for { state := g.runtime.ExpectedState() - ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + ready, _, err := g.readyUpstreams(route, state) if err != nil { return nil, err } - if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) { + if len(ready) > 0 { pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready) if len(pendingIDs) > 0 { go g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, startedAt, pendingIDs) @@ -909,15 +909,6 @@ func localUpstream(route *agenthttp.ServerlessRoute, expected agenthttp.Expected return agenthttp.ServerlessUpstream{}, false } -func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool { - for _, upstream := range upstreams { - if upstream.AlwaysOn { - return true - } - } - return false -} - func hasRunningLocalDeployment(deploymentIDs []string, actualByDeploymentID map[string]container.Container) bool { for _, deploymentID := range deploymentIDs { if actual, ok := actualByDeploymentID[deploymentID]; ok && actual.State == "running" { diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index c39123bb..ad1e3dec 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -481,7 +481,6 @@ func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { Port: 3000, SleepAfterSeconds: 300, WakeTimeoutSeconds: 5, - MinReadyReplicas: 1, LocalDeploymentIDs: []string{"dep_local"}, Upstreams: []agenthttp.ServerlessUpstream{ { diff --git a/docs/architecture.mdx b/docs/architecture.mdx index fa9729cb..90dd7a29 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -254,10 +254,9 @@ to proxy nodes that own a local proxy replica for that service. Cross-proxy wake coordination is intentionally out of scope. The wake gateway keeps the incoming request open while it starts local proxy -replicas, unless an always-on worker upstream is already ready. `Min Ready` -controls how many ready upstreams are preferred before releasing queued requests. -If fewer replicas become ready and no local wake is still in progress, the -gateway serves any available ready upstream instead of returning a 503. +replicas, unless an always-on worker upstream is already ready. Queued requests +resume when one upstream is ready. If no upstream becomes ready and no local wake +is still in progress, the gateway returns a 503. Sleep is driven by the proxy gateway's in-memory activity timer. The control plane does not scan request activity and does not enqueue sleep work. It accepts diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 46aa96ec..387d0115 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -34,15 +34,13 @@ Serverless settings are configured per service: | Enable serverless | Off | Allows proxy-hosted HTTP deployments to sleep and wake on request | | Sleep after | `300s` | Idle period before running containers are stopped | | Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | -| Min Ready | `1` | Preferred number of ready deployments before queued requests resume | -`Min Ready` does not cap how many local proxy replicas are started. A cold wake -starts the sleeping local proxy replicas for that host. `Min Ready` only controls -when held requests can resume. If a worker upstream is already ready, the gateway -can serve it immediately while local proxy replicas wake in the background. +A cold wake starts the sleeping local proxy replicas for that host. Held +requests resume when one upstream is ready. If a worker upstream is already +ready, the gateway can serve it immediately while local proxy replicas wake in +the background. -Serverless services require at least one configured replica. `Min Ready` must be -between `1` and `10`, and cannot exceed the configured replica count. +Serverless services require at least one configured replica. ## Placement diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 3e3865c8..1b6e9c82 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -1004,7 +1004,6 @@ const serverlessSettingsSchema = z.object({ enabled: z.boolean(), sleepAfterSeconds: z.number().int().min(60).max(86_400), wakeTimeoutSeconds: z.number().int().min(10).max(900), - minReadyReplicas: z.number().int().min(1).max(10), }); export async function updateServiceServerlessSettings( @@ -1013,7 +1012,6 @@ export async function updateServiceServerlessSettings( enabled: boolean; sleepAfterSeconds: number; wakeTimeoutSeconds: number; - minReadyReplicas: number; }, ) { await requireDeveloperRole(); @@ -1064,11 +1062,6 @@ export async function updateServiceServerlessSettings( if (totalConfiguredReplicas < 1) { throw new Error("Serverless services require at least one replica"); } - if (validated.minReadyReplicas > totalConfiguredReplicas) { - throw new Error( - "Minimum ready replicas cannot exceed configured replicas", - ); - } } await tx @@ -1077,7 +1070,6 @@ export async function updateServiceServerlessSettings( serverlessEnabled: validated.enabled, serverlessSleepAfterSeconds: validated.sleepAfterSeconds, serverlessWakeTimeoutSeconds: validated.wakeTimeoutSeconds, - serverlessMinReadyReplicas: validated.minReadyReplicas, }) .where(eq(services.id, serviceId)); }); @@ -1219,10 +1211,8 @@ export async function updateServiceConfig( .delete(serviceReplicas) .where(eq(serviceReplicas.serviceId, serviceId)); - let totalReplicas = 0; for (const replica of config.replicas) { if (replica.count > 0) { - totalReplicas += replica.count; await db.insert(serviceReplicas).values({ id: randomUUID(), serviceId, @@ -1231,15 +1221,6 @@ export async function updateServiceConfig( }); } } - - await db - .update(services) - .set({ - serverlessMinReadyReplicas: sql`LEAST(${services.serverlessMinReadyReplicas}, ${Math.max(1, totalReplicas)})`, - }) - .where( - and(eq(services.id, serviceId), eq(services.serverlessEnabled, true)), - ); } return { success: true }; diff --git a/web/components/service/details/serverless-section.tsx b/web/components/service/details/serverless-section.tsx index 4e2691ee..17664b35 100644 --- a/web/components/service/details/serverless-section.tsx +++ b/web/components/service/details/serverless-section.tsx @@ -25,9 +25,6 @@ export const ServerlessSection = memo(function ServerlessSection({ const [wakeTimeoutSeconds, setWakeTimeoutSeconds] = useState( String(service.serverlessWakeTimeoutSeconds ?? 300), ); - const [minReadyReplicas, setMinReadyReplicas] = useState( - String(service.serverlessMinReadyReplicas ?? 1), - ); const hasPublicHttpEndpoint = useMemo( () => service.ports.some( @@ -44,9 +41,8 @@ export const ServerlessSection = memo(function ServerlessSection({ () => ({ sleepAfterSeconds: Number.parseInt(sleepAfterSeconds, 10), wakeTimeoutSeconds: Number.parseInt(wakeTimeoutSeconds, 10), - minReadyReplicas: Number.parseInt(minReadyReplicas, 10), }), - [sleepAfterSeconds, wakeTimeoutSeconds, minReadyReplicas], + [sleepAfterSeconds, wakeTimeoutSeconds], ); const validationError = useMemo(() => { @@ -67,21 +63,13 @@ export const ServerlessSection = memo(function ServerlessSection({ ) { return "Wake timeout must be between 10 and 900 seconds"; } - if ( - !Number.isInteger(parsed.minReadyReplicas) || - parsed.minReadyReplicas < 1 || - parsed.minReadyReplicas > 10 - ) { - return "Minimum ready replicas must be between 1 and 10"; - } return null; }, [enabled, parsed, unavailableReason]); const hasChanges = enabled !== service.serverlessEnabled || parsed.sleepAfterSeconds !== service.serverlessSleepAfterSeconds || - parsed.wakeTimeoutSeconds !== service.serverlessWakeTimeoutSeconds || - parsed.minReadyReplicas !== service.serverlessMinReadyReplicas; + parsed.wakeTimeoutSeconds !== service.serverlessWakeTimeoutSeconds; const handleSave = async () => { setIsSaving(true); @@ -90,7 +78,6 @@ export const ServerlessSection = memo(function ServerlessSection({ enabled, sleepAfterSeconds: parsed.sleepAfterSeconds, wakeTimeoutSeconds: parsed.wakeTimeoutSeconds, - minReadyReplicas: parsed.minReadyReplicas, }); onUpdate(); toast.success("Serverless settings saved. Deploy to apply."); @@ -123,7 +110,7 @@ export const ServerlessSection = memo(function ServerlessSection({ />
-
+
-
- - setMinReadyReplicas(event.target.value)} - /> -

diff --git a/web/components/service/details/service-details-overview.tsx b/web/components/service/details/service-details-overview.tsx index 9f6f33b7..a3330a2b 100644 --- a/web/components/service/details/service-details-overview.tsx +++ b/web/components/service/details/service-details-overview.tsx @@ -1,13 +1,10 @@ "use client"; -import cronstrue from "cronstrue"; import { Activity, Box, - Cpu, Github, Globe, - HardDrive, type LucideIcon, Server, } from "lucide-react"; @@ -160,7 +157,7 @@ export function ServiceDetailsOverview({ service }: { service: Service }) { }); return ( - +

0} @@ -186,7 +183,7 @@ function RequestStatsPanel({ error?: unknown; isLoading: boolean; }) { - const [chartMode, setChartMode] = useState("rate"); + const [chartMode, setChartMode] = useState("total"); const chartRows = useMemo(() => buildChartRows(stats), [stats]); const statusSeries = useMemo(() => buildStatusSeries(stats), [stats]); const hasChartData = chartRows.some((row) => row.totalRequests > 0); @@ -358,12 +355,12 @@ function RequestChartModeToggle({ disabled: boolean; }) { const options: Array<{ value: RequestChartMode; label: string }> = [ - { value: "rate", label: "RPS" }, { value: "total", label: "Total" }, + { value: "rate", label: "RPS" }, ]; return ( -
+
{options.map((option) => { const isSelected = value === option.value; @@ -401,7 +398,7 @@ function ServiceConfigPanel({ const primaryEndpoint = getPrimaryEndpoint(overview.endpoints); return ( -
+
@@ -411,6 +408,7 @@ function ServiceConfigPanel({ + {formatResources(service)} - - {formatReplicaCompact(overview)} - {formatPlacementLabel(service)} - {overview.serverSummaries.length > 0 ? ( - - {formatCount(overview.serverSummaries.length, "server")} - - ) : null} - - } > - {formatEndpointCount(overview.endpoints)} {formatPortSummary(service.ports || [])} {primaryEndpoint.kind !== "private" ? ( {`${service.hostname || service.name}.internal`} ) : null} - - - {formatBackupLabel(service)} - {formatDeployScheduleLabel(service)} -
); @@ -483,7 +457,12 @@ function SummaryItem({ className?: string; }) { return ( -
+
{label} @@ -509,7 +488,12 @@ function ConfigDigestItem({ className?: string; }) { return ( -
+
{label} @@ -630,7 +614,7 @@ function LegendMetric({ function RequestStatsState({ message }: { message: string }) { return ( -
+
{message}
); @@ -649,7 +633,7 @@ function RequestStatsTooltip({ const items = visiblePayload.length > 0 ? visiblePayload : payload; return ( -
+

{formatTooltipDate(String(label))}

{items.map((item) => ( @@ -922,18 +906,6 @@ function formatInstanceSummary(overview: OverviewData): string { return `${overview.runningDeployments}/${configured} Running Across ${formatCount(serverCount, "server")}`; } -function getLockedServerLabel(service: Service): string | null { - if (!service.lockedServerId) return null; - - return ( - service.lockedServer?.name ?? - service.configuredReplicas.find( - (replica) => replica.serverId === service.lockedServerId, - )?.serverName ?? - service.lockedServerId - ); -} - function getPrimaryEndpoint(endpoints: EndpointItem[]): EndpointItem { return ( endpoints.find((endpoint) => endpoint.kind === "public") ?? @@ -948,34 +920,6 @@ function getPrimaryEndpoint(endpoints: EndpointItem[]): EndpointItem { ); } -function formatReplicaCompact(overview: OverviewData): string { - const configured = getConfiguredReplicaCount(overview); - - if (configured === 0) return "No Replicas"; - - return `${overview.runningDeployments}/${configured} Running`; -} - -function formatPlacementLabel(service: Service): string { - const lockedServer = getLockedServerLabel(service); - - if (service.stateful) { - return lockedServer ? `Stateful on ${lockedServer}` : "Stateful"; - } - - return lockedServer ? `Pinned to ${lockedServer}` : "Stateless"; -} - -function formatEndpointCount(endpoints: EndpointItem[]): string { - const publicCount = endpoints.filter( - (endpoint) => endpoint.kind !== "private", - ).length; - - if (publicCount === 0) return "Private Only"; - - return `${formatCount(publicCount, "public endpoint")} + Private`; -} - function formatPortSummary(ports: Service["ports"]): string { if (ports.length === 0) return "No Ports"; if (ports.length === 1) return formatPortLabel(ports[0]); @@ -990,43 +934,6 @@ function formatPortLabel(port: Service["ports"][number]): string { return `${protocol} :${port.port}${external}`; } -function formatDataSummary(service: Service): string { - const volumeCount = service.volumes?.length ?? 0; - const secretCount = service.secrets?.length ?? 0; - const parts = []; - - if (volumeCount > 0) parts.push(formatCount(volumeCount, "volume")); - if (secretCount > 0) parts.push(formatCount(secretCount, "secret")); - - return parts.length > 0 ? parts.join(" · ") : "No Volumes or Secrets"; -} - -function formatBackupLabel(service: Service): string { - if ((service.volumes?.length ?? 0) === 0) return "No Backups"; - if (!service.backupEnabled) return "Manual Backups"; - - return service.backupSchedule - ? `Backup ${service.backupSchedule}` - : "Backups On"; -} - -function formatDeployScheduleLabel(service: Service): string { - if (!service.deploymentSchedule) return "Manual Deploy"; - - try { - const scheduleDescription = cronstrue.toString(service.deploymentSchedule, { - verbose: true, - }); - return `Deploy ${lowercaseFirstLetter(scheduleDescription)}`; - } catch { - return "Scheduled Deploy"; - } -} - -function lowercaseFirstLetter(value: string): string { - return value.charAt(0).toLowerCase() + value.slice(1); -} - function formatCount(count: number, singular: string): string { const label = singular .split(" ") diff --git a/web/db/schema.ts b/web/db/schema.ts index 23883407..2a9137a4 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -387,9 +387,6 @@ export const services = pgTable("services", { serverlessWakeTimeoutSeconds: integer("serverless_wake_timeout_seconds") .notNull() .default(300), - serverlessMinReadyReplicas: integer("serverless_min_ready_replicas") - .notNull() - .default(1), deployedConfig: text("deployed_config"), deploymentSchedule: text("deployment_schedule"), lastScheduledDeploymentRunAt: timestamp("last_scheduled_deployment_run_at", { diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index 78bd7d6e..dde83bc6 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -19,10 +19,10 @@ import { getSteadyStateRecreateDecision, } from "@/lib/autoheal-policy"; import { - type ObservedPhase, isObservedActiveContainer, isObservedReady, markDeploymentFailedRemoved, + type ObservedPhase, observedStartingPhases, runtimeExpectedStates, } from "@/lib/deployment-status"; @@ -149,7 +149,6 @@ async function applyDeploymentErrors( serverlessEnabled: services.serverlessEnabled, serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, - serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, stateful: services.stateful, deployedConfig: services.deployedConfig, rolloutStatus: rollouts.status, @@ -281,7 +280,6 @@ async function applyServerlessTransitions( serverlessEnabled: services.serverlessEnabled, serverlessSleepAfterSeconds: services.serverlessSleepAfterSeconds, serverlessWakeTimeoutSeconds: services.serverlessWakeTimeoutSeconds, - serverlessMinReadyReplicas: services.serverlessMinReadyReplicas, stateful: services.stateful, deployedConfig: services.deployedConfig, serverIsProxy: servers.isProxy, @@ -480,7 +478,9 @@ function getServerlessTransitionResultBase( function isServerlessTransitionType( value: unknown, ): value is ServerlessTransition["type"] { - return value === "sleep" || value === "wake_started" || value === "wake_failed"; + return ( + value === "sleep" || value === "wake_started" || value === "wake_failed" + ); } export function getSleepTransitionDeploymentIds( @@ -561,7 +561,6 @@ function getInvalidServerlessTransitionReason({ serverlessEnabled: boolean; serverlessSleepAfterSeconds: number; serverlessWakeTimeoutSeconds: number; - serverlessMinReadyReplicas: number; deployedConfig: string | null; serverIsProxy: boolean; } @@ -729,8 +728,9 @@ export async function applyStatusReport( serverId, serverlessTransitions, ); - const sleepTransitionDeploymentIds = - getSleepTransitionDeploymentIds(serverlessTransitions); + const sleepTransitionDeploymentIds = getSleepTransitionDeploymentIds( + serverlessTransitions, + ); const reportedDeploymentIds = report.containers .map((c) => c.deploymentId) diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index c4838999..6340a112 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -101,7 +101,6 @@ export type ServerlessRoute = { port: number; sleepAfterSeconds: number; wakeTimeoutSeconds: number; - minReadyReplicas: number; localDeploymentIds: string[]; upstreams: ServerlessRouteUpstream[]; }; @@ -462,10 +461,6 @@ export function buildServerlessRoutesFromRows({ getDeployedServerlessConfig(service).sleepAfterSeconds, wakeTimeoutSeconds: getDeployedServerlessConfig(service).wakeTimeoutSeconds, - minReadyReplicas: Math.max( - 1, - getDeployedServerlessConfig(service).minReadyReplicas, - ), localDeploymentIds, upstreams, }, @@ -594,15 +589,16 @@ async function buildTraefikConfig(server: Server, allServices: Service[]) { ), ); + const routePorts = buildRuntimeRoutePorts(allServices, ports); const routes = buildTraefikRoutes({ serverId: server.id, - ports: buildRuntimeRoutePorts(allServices, ports), + ports: routePorts, routableDeployments, serverlessServiceIds, serverlessRouteSuppressedServiceIds, }); - const routedDomains = routes.httpRoutes.map((r) => r.domain); - const certificates = await getAllCertificatesForDomains(routedDomains); + const certificateDomains = buildTraefikCertificateDomains(routePorts); + const certificates = await getAllCertificatesForDomains(certificateDomains); const controlPlaneUrl = process.env.APP_URL; if (!controlPlaneUrl) { @@ -722,6 +718,17 @@ export function buildTraefikRoutes({ return { httpRoutes, tcpRoutes, udpRoutes }; } +export function buildTraefikCertificateDomains(ports: RouteServicePort[]) { + return Array.from( + new Set( + ports + .filter((port) => port.isPublic && port.protocol === "http") + .map((port) => port.domain?.trim()) + .filter((domain): domain is string => Boolean(domain)), + ), + ).sort(); +} + function buildHealthCheck(service: Service): ExpectedContainer["healthCheck"] { if (!service.healthCheckCmd) return null; diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index abe53c72..20273c2d 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -41,7 +41,6 @@ export function isActiveDeploymentForRollout( serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; serverlessWakeTimeoutSeconds?: number | null; - serverlessMinReadyReplicas?: number | null; }, ) { void service; @@ -192,8 +191,8 @@ export async function prepareRollingUpdate( .from(deployments) .where(eq(deployments.serviceId, serviceId)); - const runningDeployments = existingDeployments.filter( - (d) => isActiveDeploymentForRollout(d, service), + const runningDeployments = existingDeployments.filter((d) => + isActiveDeploymentForRollout(d, service), ); return { deploymentIds: runningDeployments.map((d) => d.id) }; @@ -487,8 +486,8 @@ export async function checkForRollingUpdate( .from(deployments) .where(eq(deployments.serviceId, serviceId)); - const runningDeployments = existingDeployments.filter( - (d) => isActiveDeploymentForRollout(d, service), + const runningDeployments = existingDeployments.filter((d) => + isActiveDeploymentForRollout(d, service), ); return runningDeployments.length > 0; diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 263e34c9..6cf876ff 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -48,7 +48,6 @@ export type ServerlessConfig = { enabled: boolean; sleepAfterSeconds: number; wakeTimeoutSeconds: number; - minReadyReplicas: number; }; export type DeployedConfig = { @@ -90,7 +89,6 @@ export function buildCurrentConfig( serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; serverlessWakeTimeoutSeconds?: number | null; - serverlessMinReadyReplicas?: number | null; }, replicas: { serverId: string; serverName: string; count: number }[], ports: { @@ -148,7 +146,6 @@ export function buildCurrentConfig( enabled: service.serverlessEnabled ?? false, sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, - minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, }, secrets: (secrets ?? []).map((s) => ({ key: s.key, @@ -289,16 +286,6 @@ export function diffConfigs( to: `${currentServerless.wakeTimeoutSeconds}s`, }); } - if ( - deployedServerless.minReadyReplicas !== - currentServerless.minReadyReplicas - ) { - changes.push({ - field: "Serverless min ready", - from: String(deployedServerless.minReadyReplicas), - to: String(currentServerless.minReadyReplicas), - }); - } } if (deployed.source.image !== current.source.image) { @@ -568,7 +555,6 @@ export function normalizeServerlessConfig( enabled: config?.enabled ?? false, sleepAfterSeconds: config?.sleepAfterSeconds ?? 300, wakeTimeoutSeconds: config?.wakeTimeoutSeconds ?? 300, - minReadyReplicas: config?.minReadyReplicas ?? 1, }; } @@ -576,13 +562,11 @@ export function getCurrentServerlessConfig(service: { serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; serverlessWakeTimeoutSeconds?: number | null; - serverlessMinReadyReplicas?: number | null; }): ServerlessConfig { return { enabled: service.serverlessEnabled ?? false, sleepAfterSeconds: service.serverlessSleepAfterSeconds ?? 300, wakeTimeoutSeconds: service.serverlessWakeTimeoutSeconds ?? 300, - minReadyReplicas: service.serverlessMinReadyReplicas ?? 1, }; } @@ -591,7 +575,6 @@ export function getDeployedServerlessConfig(service: { serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; serverlessWakeTimeoutSeconds?: number | null; - serverlessMinReadyReplicas?: number | null; }): ServerlessConfig { const deployed = parseDeployedConfig(service.deployedConfig ?? null); return deployed?.serverless @@ -613,7 +596,6 @@ export function isDeployedServerlessService(service: { serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; serverlessWakeTimeoutSeconds?: number | null; - serverlessMinReadyReplicas?: number | null; }) { return getDeployedServerlessConfig(service).enabled; } diff --git a/web/public/setup.sh b/web/public/setup.sh index ed91d52b..30725e66 100644 --- a/web/public/setup.sh +++ b/web/public/setup.sh @@ -61,6 +61,31 @@ pkg_hold() { fi } +configure_journald_limits() { + step "Configuring journal storage limits..." + mkdir -p /etc/systemd/journald.conf.d + cat > /etc/systemd/journald.conf.d/techulus.conf << 'EOF' +[Journal] +SystemMaxUse=1G +SystemKeepFree=1G +RuntimeMaxUse=256M +MaxRetentionSec=7day +EOF + systemctl restart systemd-journald 2>/dev/null || true + echo "✓ Journald storage limits configured" +} + +ensure_logrotate() { + if command -v logrotate &>/dev/null; then + return + fi + + pkg_install logrotate + if ! command -v logrotate &>/dev/null; then + error "Failed to install logrotate" + fi +} + echo "" echo "==========================================" echo " Techulus Cloud Agent Installer" @@ -118,6 +143,8 @@ case $ARCH in esac echo "✓ Architecture: $ARCH ($AGENT_ARCH)" +configure_journald_limits + step "Collecting configuration..." prompt CONTROL_PLANE_URL "Enter Control Plane URL (e.g., https://api.example.com): " required @@ -409,6 +436,26 @@ EOF fi echo "✓ Traefik config created" + step "Configuring Traefik access log rotation..." + ensure_logrotate + cat > /etc/logrotate.d/techulus-traefik << 'EOF' +/var/log/traefik/access.log { + daily + maxsize 100M + rotate 7 + compress + delaycompress + missingok + notifempty + copytruncate + create 0644 root root +} +EOF + if [ ! -f /etc/logrotate.d/techulus-traefik ]; then + error "Failed to create Traefik logrotate config" + fi + echo "✓ Traefik access log rotation configured" + cat > /etc/systemd/system/traefik.service << 'EOF' [Unit] Description=Traefik Reverse Proxy @@ -541,6 +588,23 @@ mkdir -p /etc/buildkit cat > /etc/buildkit/buildkitd.toml << 'EOF' [dns] nameservers = ["8.8.8.8", "1.1.1.1"] + +[worker.oci] + gc = true + reservedSpace = "15%" + maxUsedSpace = "30%" + minFreeSpace = "15%" + +[[worker.oci.gcpolicy]] + filters = ["type==source.local", "type==source.git.checkout", "type==exec.cachemount"] + keepDuration = "48h" + maxUsedSpace = "10%" + +[[worker.oci.gcpolicy]] + all = true + keepDuration = "168h" + reservedSpace = "15%" + maxUsedSpace = "30%" EOF if [ ! -f /etc/buildkit/buildkitd.toml ]; then error "Failed to create BuildKit config" @@ -568,7 +632,7 @@ fi systemctl daemon-reload systemctl enable buildkitd -systemctl start buildkitd +systemctl restart buildkitd sleep 2 if ! systemctl is-active --quiet buildkitd; then error "Failed to start BuildKit daemon" diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 52d93b25..2b01748d 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -10,6 +10,7 @@ import { buildExpectedContainersFromRows, buildRuntimeRoutePorts, buildServerlessRoutesFromRows, + buildTraefikCertificateDomains, buildTraefikRoutes, } from "@/lib/agent/expected-state"; @@ -31,7 +32,6 @@ describe("expected-state pure builders", () => { enabled: true, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, }, }); @@ -358,6 +358,33 @@ describe("expected-state pure builders", () => { expect(routes.httpRoutes).toEqual([]); }); + it("keeps suppressed serverless HTTP domains in certificate selection", () => { + const ports: Parameters[0] = [ + { + id: "port_1", + serviceId: "svc_serverless", + port: 3000, + isPublic: true, + protocol: "http", + domain: "sleepy.example.com", + externalPort: null, + tlsPassthrough: false, + }, + ]; + + const routes = buildTraefikRoutes({ + serverId: "proxy_2", + ports, + routableDeployments: [], + serverlessRouteSuppressedServiceIds: new Set(["svc_serverless"]), + }); + + expect(routes.httpRoutes).toEqual([]); + expect(buildTraefikCertificateDomains(ports)).toEqual([ + "sleepy.example.com", + ]); + }); + it("keeps worker-only serverless services on direct routes", () => { const routes = buildTraefikRoutes({ serverId: "proxy_1", @@ -459,7 +486,6 @@ describe("expected-state pure builders", () => { stateful: false, serverlessSleepAfterSeconds: 300, serverlessWakeTimeoutSeconds: 120, - serverlessMinReadyReplicas: 1, }, ] as any, ports: [ @@ -519,7 +545,6 @@ describe("expected-state pure builders", () => { port: 3000, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, localDeploymentIds: ["dep_proxy"], upstreams: [ { @@ -544,7 +569,6 @@ describe("expected-state pure builders", () => { stateful: true, serverlessSleepAfterSeconds: 300, serverlessWakeTimeoutSeconds: 120, - serverlessMinReadyReplicas: 1, }, ] as any, ports: [ @@ -584,7 +608,6 @@ describe("expected-state pure builders", () => { port: 3000, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, localDeploymentIds: ["dep_stateful"], upstreams: [], }, @@ -601,7 +624,6 @@ describe("expected-state pure builders", () => { stateful: false, serverlessSleepAfterSeconds: 300, serverlessWakeTimeoutSeconds: 120, - serverlessMinReadyReplicas: 1, }, ] as any, ports: [ @@ -658,7 +680,6 @@ describe("expected-state pure builders", () => { stateful: false, serverlessSleepAfterSeconds: 60, serverlessWakeTimeoutSeconds: 60, - serverlessMinReadyReplicas: 1, deployedConfig: deployedServerlessConfig, }, ] as any, @@ -687,7 +708,6 @@ describe("expected-state pure builders", () => { port: 3000, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, localDeploymentIds: ["dep_sleeping"], upstreams: [], }, @@ -734,7 +754,6 @@ describe("expected-state pure builders", () => { stateful: false, serverlessSleepAfterSeconds: 300, serverlessWakeTimeoutSeconds: 120, - serverlessMinReadyReplicas: 1, }, ] as any, ports: [ diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index 5564eabf..98a1c55b 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; import { + type DeployedConfig, diffConfigs, getDeployedServerlessConfig, isDeployedServerlessService, - type DeployedConfig, } from "@/lib/service-config"; function deployedConfig( @@ -19,7 +19,6 @@ function deployedConfig( enabled: true, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, }, ...overrides, }; @@ -31,7 +30,6 @@ describe("service config", () => { serverlessEnabled: false, serverlessSleepAfterSeconds: 60, serverlessWakeTimeoutSeconds: 60, - serverlessMinReadyReplicas: 1, stateful: true, deployedConfig: JSON.stringify(deployedConfig()), }; @@ -50,7 +48,6 @@ describe("service config", () => { stateful: true, serverlessSleepAfterSeconds: 300, serverlessWakeTimeoutSeconds: 120, - serverlessMinReadyReplicas: 1, deployedConfig: JSON.stringify(deployedConfig({ stateful: true })), }; @@ -68,7 +65,6 @@ describe("service config", () => { enabled: false, sleepAfterSeconds: 300, wakeTimeoutSeconds: 120, - minReadyReplicas: 1, }, });