Release v0.27.0

agent/internal/agent/backup.go

@@ -101,7 +101,10 @@ func (a *Agent) processVolumeBackup(backupID, serviceID, containerID, volumeName
 		}
 	}
 
-	tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("backup-%s.tar.gz", backupID))
+	tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("backup-%s.tar.gz", backupID))
+	if err != nil {
+		return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err))
+	}
 	defer os.Remove(tarPath)
 
 	size, checksum, err := createTarGzWithChecksum(volumePath, tarPath)
@@ -158,7 +161,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam
 		return err
 	}
 
-	tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-%s.tar.gz", backupID))
+	tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-%s.tar.gz", backupID))
+	if err != nil {
+		return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err))
+	}
 	defer os.Remove(tarPath)
 
 	if !strings.HasSuffix(storagePath, ".tar.gz") {
@@ -185,7 +191,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam
 		return reportFailure(fmt.Errorf("checksum mismatch: expected %s, got %s", expectedChecksum, checksum))
 	}
 
-	tempExtractPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-extract-%s", backupID))
+	tempExtractPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-extract-%s", backupID))
+	if err != nil {
+		return reportFailure(fmt.Errorf("failed to create temp extract path: %w", err))
+	}
 	defer os.RemoveAll(tempExtractPath)
 
 	if err := os.MkdirAll(tempExtractPath, 0755); err != nil {
@@ -258,6 +267,19 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam
 	return nil
 }
 
+func tempArtifactPath(dataDir, name string) (string, error) {
+	if name == "" || name != filepath.Base(name) {
+		return "", fmt.Errorf("invalid temp artifact name: %s", name)
+	}
+
+	tmpDir := filepath.Join(dataDir, "tmp")
+	if err := os.MkdirAll(tmpDir, 0700); err != nil {
+		return "", err
+	}
+
+	return filepath.Join(tmpDir, name), nil
+}
+
 func moveDir(src, dst string) error {
 	if err := os.Rename(src, dst); err == nil {
 		return nil

agent/internal/agent/backup_test.go

@@ -0,0 +1,32 @@
+package agent
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestTempArtifactPathStaysUnderAgentDataDir(t *testing.T) {
+	dataDir := t.TempDir()
+
+	path, err := tempArtifactPath(dataDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedPrefix := filepath.Join(dataDir, "tmp") + string(os.PathSeparator)
+	if !strings.HasPrefix(path, expectedPrefix) {
+		t.Fatalf("expected %s to be under %s", path, expectedPrefix)
+	}
+
+	if _, err := os.Stat(filepath.Join(dataDir, "tmp")); err != nil {
+		t.Fatalf("expected temp directory to exist: %v", err)
+	}
+}
+
+func TestTempArtifactPathRejectsNestedNames(t *testing.T) {
+	if _, err := tempArtifactPath(t.TempDir(), "../backup.tar.gz"); err == nil {
+		t.Fatal("expected nested temp artifact name to be rejected")
+	}
+}

agent/internal/agent/run.go

@@ -44,6 +44,7 @@ func (a *Agent) Run(ctx context.Context) {
 
 	var cleanupTickerC <-chan time.Time
 	if a.Builder != nil {
+		go a.RunBuildCleanup()
 		cleanupTicker := time.NewTicker(BuildCleanupInterval)
 		defer cleanupTicker.Stop()
 		cleanupTickerC = cleanupTicker.C

agent/internal/build/build.go

@@ -5,11 +5,13 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -43,6 +45,13 @@ type Builder struct {
 	logSender LogSender
 }
 
+const (
+	staleBuildDirAge     = 1 * time.Hour
+	staleTempArtifactAge = 24 * time.Hour
+)
+
+var managedTempArtifactPattern = regexp.MustCompile(`^(backup|restore)-[0-9a-fA-F-]{36}\.tar\.gz$|^restore-extract-[0-9a-fA-F-]{36}$`)
+
 func NewBuilder(dataDir string, logSender LogSender) *Builder {
 	return &Builder{
 		dataDir:   dataDir,
@@ -408,8 +417,26 @@ func (b *Builder) sendLog(config *Config, message string) {
 }
 
 func (b *Builder) Cleanup() error {
-	buildsDir := filepath.Join(b.dataDir, "builds")
+	now := time.Now()
+	var cleanupErrors []error
+
+	if err := cleanupStaleBuildDirs(filepath.Join(b.dataDir, "builds"), now.Add(-staleBuildDirAge)); err != nil {
+		cleanupErrors = append(cleanupErrors, err)
+	}
+
+	if err := cleanupManagedTempArtifacts(filepath.Join(b.dataDir, "tmp"), now.Add(-staleTempArtifactAge)); err != nil {
+		cleanupErrors = append(cleanupErrors, err)
+	}
+
+	log.Printf("[build:cleanup] pruning unused images")
+	if err := container.ImagePrune(); err != nil {
+		cleanupErrors = append(cleanupErrors, err)
+	}
 
+	return errors.Join(cleanupErrors...)
+}
+
+func cleanupStaleBuildDirs(buildsDir string, cutoff time.Time) error {
 	entries, err := os.ReadDir(buildsDir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -418,7 +445,7 @@ func (b *Builder) Cleanup() error {
 		return err
 	}
 
-	cutoff := time.Now().Add(-1 * time.Hour)
+	var cleanupErrors []error
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
@@ -432,14 +459,47 @@ func (b *Builder) Cleanup() error {
 		if info.ModTime().Before(cutoff) {
 			path := filepath.Join(buildsDir, entry.Name())
 			log.Printf("[build:cleanup] removing stale build dir: %s", entry.Name())
-			os.RemoveAll(path)
+			if err := os.RemoveAll(path); err != nil {
+				cleanupErrors = append(cleanupErrors, err)
+			}
 		}
 	}
 
-	log.Printf("[build:cleanup] pruning unused images")
-	container.ImagePrune()
+	return errors.Join(cleanupErrors...)
+}
 
-	return nil
+func cleanupManagedTempArtifacts(tmpDir string, cutoff time.Time) error {
+	entries, err := os.ReadDir(tmpDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	var cleanupErrors []error
+	for _, entry := range entries {
+		if !managedTempArtifactPattern.MatchString(entry.Name()) {
+			continue
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			continue
+		}
+
+		if info.ModTime().After(cutoff) {
+			continue
+		}
+
+		path := filepath.Join(tmpDir, entry.Name())
+		log.Printf("[build:cleanup] removing stale temp artifact: %s", entry.Name())
+		if err := os.RemoveAll(path); err != nil {
+			cleanupErrors = append(cleanupErrors, err)
+		}
+	}
+
+	return errors.Join(cleanupErrors...)
 }
 
 func CheckPrerequisites() error {

agent/internal/build/build_test.go

@@ -0,0 +1,92 @@
+package build
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestCleanupManagedTempArtifactsRemovesOnlyOldManagedArtifacts(t *testing.T) {
+	tmpDir := t.TempDir()
+	oldTime := time.Now().Add(-48 * time.Hour)
+	cutoff := time.Now().Add(-24 * time.Hour)
+
+	oldArchive := filepath.Join(tmpDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz")
+	youngArchive := filepath.Join(tmpDir, "restore-22222222-2222-2222-2222-222222222222.tar.gz")
+	unmanagedArchive := filepath.Join(tmpDir, "backup-not-a-build-id.tar.gz")
+	oldExtractDir := filepath.Join(tmpDir, "restore-extract-33333333-3333-3333-3333-333333333333")
+
+	if err := os.WriteFile(oldArchive, []byte("old"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(youngArchive, []byte("young"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(unmanagedArchive, []byte("unmanaged"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(oldExtractDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, path := range []string{oldArchive, unmanagedArchive, oldExtractDir} {
+		if err := os.Chtimes(path, oldTime, oldTime); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if err := cleanupManagedTempArtifacts(tmpDir, cutoff); err != nil {
+		t.Fatal(err)
+	}
+
+	assertMissing(t, oldArchive)
+	assertMissing(t, oldExtractDir)
+	assertExists(t, youngArchive)
+	assertExists(t, unmanagedArchive)
+}
+
+func TestCleanupStaleBuildDirsRemovesOnlyOldDirectories(t *testing.T) {
+	buildsDir := t.TempDir()
+	oldTime := time.Now().Add(-2 * time.Hour)
+	cutoff := time.Now().Add(-1 * time.Hour)
+
+	oldDir := filepath.Join(buildsDir, "old-build")
+	youngDir := filepath.Join(buildsDir, "young-build")
+	filePath := filepath.Join(buildsDir, "not-a-dir")
+
+	if err := os.Mkdir(oldDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(youngDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filePath, []byte("keep"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(oldDir, oldTime, oldTime); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cleanupStaleBuildDirs(buildsDir, cutoff); err != nil {
+		t.Fatal(err)
+	}
+
+	assertMissing(t, oldDir)
+	assertExists(t, youngDir)
+	assertExists(t, filePath)
+}
+
+func assertExists(t *testing.T, path string) {
+	t.Helper()
+	if _, err := os.Stat(path); err != nil {
+		t.Fatalf("expected %s to exist: %v", path, err)
+	}
+}
+
+func assertMissing(t *testing.T, path string) {
+	t.Helper()
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		t.Fatalf("expected %s to be removed, got err=%v", path, err)
+	}
+}

agent/internal/container/runtime.go

@@ -456,8 +456,12 @@ func writeDockerConfig(registryURL, username, password string) error {
 	return nil
 }
 
-func ImagePrune() {
-	exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h").Run()
+func ImagePrune() error {
+	cmd := exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h")
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to prune images: %s: %w", string(output), err)
+	}
+	return nil
 }
 
 type podmanContainer struct {

agent/internal/http/client.go

@@ -109,7 +109,6 @@ type ServerlessRoute struct {
 	Port               int                  `json:"port"`
 	SleepAfterSeconds  int                  `json:"sleepAfterSeconds"`
 	WakeTimeoutSeconds int                  `json:"wakeTimeoutSeconds"`
-	MinReadyReplicas   int                  `json:"minReadyReplicas"`
 	LocalDeploymentIDs []string             `json:"localDeploymentIds"`
 	Upstreams          []ServerlessUpstream `json:"upstreams"`
 }

agent/internal/serverless/gateway.go

@@ -236,7 +236,7 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream,
 	}
 
 	wakeStartedAt := time.Now()
-	if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && hasAlwaysOnUpstream(ready)) {
+	if len(ready) > 0 {
 		log.Printf(
 			"[serverless-gateway] wake requested host=%s deployments=%d ready_upstreams=%d mode=background",
 			host,
@@ -380,11 +380,11 @@ func (g *Gateway) waitForReadyUpstreams(route *agenthttp.ServerlessRoute, wakeTi
 
 	for {
 		state := g.runtime.ExpectedState()
-		ready, sleepingLocalIDs, err := g.readyUpstreams(route, state)
+		ready, _, err := g.readyUpstreams(route, state)
 		if err != nil {
 			return nil, err
 		}
-		if len(ready) >= max(1, route.MinReadyReplicas) || (len(ready) > 0 && len(sleepingLocalIDs) == 0) {
+		if len(ready) > 0 {
 			pendingIDs := pendingWakeDeploymentIDs(wokenDeploymentIDs, ready)
 			if len(pendingIDs) > 0 {
 				go g.waitForWokenDeployments(route, route.WakeTimeoutSeconds, startedAt, pendingIDs)
@@ -909,15 +909,6 @@ func localUpstream(route *agenthttp.ServerlessRoute, expected agenthttp.Expected
 	return agenthttp.ServerlessUpstream{}, false
 }
 
-func hasAlwaysOnUpstream(upstreams []agenthttp.ServerlessUpstream) bool {
-	for _, upstream := range upstreams {
-		if upstream.AlwaysOn {
-			return true
-		}
-	}
-	return false
-}
-
 func hasRunningLocalDeployment(deploymentIDs []string, actualByDeploymentID map[string]container.Container) bool {
 	for _, deploymentID := range deploymentIDs {
 		if actual, ok := actualByDeploymentID[deploymentID]; ok && actual.State == "running" {

agent/internal/serverless/gateway_test.go

@@ -481,7 +481,6 @@ func testExpectedState(localDesiredState string) *agenthttp.ExpectedState {
 			Port:               3000,
 			SleepAfterSeconds:  300,
 			WakeTimeoutSeconds: 5,
-			MinReadyReplicas:   1,
 			LocalDeploymentIDs: []string{"dep_local"},
 			Upstreams: []agenthttp.ServerlessUpstream{
 				{

docs/architecture.mdx

@@ -254,10 +254,9 @@ to proxy nodes that own a local proxy replica for that service. Cross-proxy wake
 coordination is intentionally out of scope.
 
 The wake gateway keeps the incoming request open while it starts local proxy
-replicas, unless an always-on worker upstream is already ready. `Min Ready`
-controls how many ready upstreams are preferred before releasing queued requests.
-If fewer replicas become ready and no local wake is still in progress, the
-gateway serves any available ready upstream instead of returning a 503.
+replicas, unless an always-on worker upstream is already ready. Queued requests
+resume when one upstream is ready. If no upstream becomes ready and no local wake
+is still in progress, the gateway returns a 503.
 
 Sleep is driven by the proxy gateway's in-memory activity timer. The control
 plane does not scan request activity and does not enqueue sleep work. It accepts