diff --git a/agent/internal/agent/backup.go b/agent/internal/agent/backup.go index 5b121c7a..e5272563 100644 --- a/agent/internal/agent/backup.go +++ b/agent/internal/agent/backup.go @@ -101,7 +101,10 @@ func (a *Agent) processVolumeBackup(backupID, serviceID, containerID, volumeName } } - tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("backup-%s.tar.gz", backupID)) + tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("backup-%s.tar.gz", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err)) + } defer os.Remove(tarPath) size, checksum, err := createTarGzWithChecksum(volumePath, tarPath) @@ -158,7 +161,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return err } - tarPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-%s.tar.gz", backupID)) + tarPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-%s.tar.gz", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp archive path: %w", err)) + } defer os.Remove(tarPath) if !strings.HasSuffix(storagePath, ".tar.gz") { @@ -185,7 +191,10 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return reportFailure(fmt.Errorf("checksum mismatch: expected %s, got %s", expectedChecksum, checksum)) } - tempExtractPath := filepath.Join(os.TempDir(), fmt.Sprintf("restore-extract-%s", backupID)) + tempExtractPath, err := tempArtifactPath(a.DataDir, fmt.Sprintf("restore-extract-%s", backupID)) + if err != nil { + return reportFailure(fmt.Errorf("failed to create temp extract path: %w", err)) + } defer os.RemoveAll(tempExtractPath) if err := os.MkdirAll(tempExtractPath, 0755); err != nil { @@ -258,6 +267,19 @@ func (a *Agent) processVolumeRestore(backupID, serviceID, containerID, volumeNam return nil } +func tempArtifactPath(dataDir, name string) (string, error) { + if name == "" || name != filepath.Base(name) { + return "", fmt.Errorf("invalid temp artifact name: %s", name) + } + + tmpDir := filepath.Join(dataDir, "tmp") + if err := os.MkdirAll(tmpDir, 0700); err != nil { + return "", err + } + + return filepath.Join(tmpDir, name), nil +} + func moveDir(src, dst string) error { if err := os.Rename(src, dst); err == nil { return nil diff --git a/agent/internal/agent/backup_test.go b/agent/internal/agent/backup_test.go new file mode 100644 index 00000000..7a9a127c --- /dev/null +++ b/agent/internal/agent/backup_test.go @@ -0,0 +1,32 @@ +package agent + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTempArtifactPathStaysUnderAgentDataDir(t *testing.T) { + dataDir := t.TempDir() + + path, err := tempArtifactPath(dataDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz") + if err != nil { + t.Fatal(err) + } + + expectedPrefix := filepath.Join(dataDir, "tmp") + string(os.PathSeparator) + if !strings.HasPrefix(path, expectedPrefix) { + t.Fatalf("expected %s to be under %s", path, expectedPrefix) + } + + if _, err := os.Stat(filepath.Join(dataDir, "tmp")); err != nil { + t.Fatalf("expected temp directory to exist: %v", err) + } +} + +func TestTempArtifactPathRejectsNestedNames(t *testing.T) { + if _, err := tempArtifactPath(t.TempDir(), "../backup.tar.gz"); err == nil { + t.Fatal("expected nested temp artifact name to be rejected") + } +} diff --git a/agent/internal/agent/run.go b/agent/internal/agent/run.go index 9b39c419..ac10ee3c 100644 --- a/agent/internal/agent/run.go +++ b/agent/internal/agent/run.go @@ -44,6 +44,7 @@ func (a *Agent) Run(ctx context.Context) { var cleanupTickerC <-chan time.Time if a.Builder != nil { + go a.RunBuildCleanup() cleanupTicker := time.NewTicker(BuildCleanupInterval) defer cleanupTicker.Stop() cleanupTickerC = cleanupTicker.C diff --git a/agent/internal/build/build.go b/agent/internal/build/build.go index b55f2816..00c343d1 100644 --- a/agent/internal/build/build.go +++ b/agent/internal/build/build.go @@ -5,11 +5,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "log" "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -43,6 +45,13 @@ type Builder struct { logSender LogSender } +const ( + staleBuildDirAge = 1 * time.Hour + staleTempArtifactAge = 24 * time.Hour +) + +var managedTempArtifactPattern = regexp.MustCompile(`^(backup|restore)-[0-9a-fA-F-]{36}\.tar\.gz$|^restore-extract-[0-9a-fA-F-]{36}$`) + func NewBuilder(dataDir string, logSender LogSender) *Builder { return &Builder{ dataDir: dataDir, @@ -408,8 +417,26 @@ func (b *Builder) sendLog(config *Config, message string) { } func (b *Builder) Cleanup() error { - buildsDir := filepath.Join(b.dataDir, "builds") + now := time.Now() + var cleanupErrors []error + + if err := cleanupStaleBuildDirs(filepath.Join(b.dataDir, "builds"), now.Add(-staleBuildDirAge)); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + + if err := cleanupManagedTempArtifacts(filepath.Join(b.dataDir, "tmp"), now.Add(-staleTempArtifactAge)); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + + log.Printf("[build:cleanup] pruning unused images") + if err := container.ImagePrune(); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + return errors.Join(cleanupErrors...) +} + +func cleanupStaleBuildDirs(buildsDir string, cutoff time.Time) error { entries, err := os.ReadDir(buildsDir) if err != nil { if os.IsNotExist(err) { @@ -418,7 +445,7 @@ func (b *Builder) Cleanup() error { return err } - cutoff := time.Now().Add(-1 * time.Hour) + var cleanupErrors []error for _, entry := range entries { if !entry.IsDir() { continue @@ -432,14 +459,47 @@ func (b *Builder) Cleanup() error { if info.ModTime().Before(cutoff) { path := filepath.Join(buildsDir, entry.Name()) log.Printf("[build:cleanup] removing stale build dir: %s", entry.Name()) - os.RemoveAll(path) + if err := os.RemoveAll(path); err != nil { + cleanupErrors = append(cleanupErrors, err) + } } } - log.Printf("[build:cleanup] pruning unused images") - container.ImagePrune() + return errors.Join(cleanupErrors...) +} - return nil +func cleanupManagedTempArtifacts(tmpDir string, cutoff time.Time) error { + entries, err := os.ReadDir(tmpDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var cleanupErrors []error + for _, entry := range entries { + if !managedTempArtifactPattern.MatchString(entry.Name()) { + continue + } + + info, err := entry.Info() + if err != nil { + continue + } + + if info.ModTime().After(cutoff) { + continue + } + + path := filepath.Join(tmpDir, entry.Name()) + log.Printf("[build:cleanup] removing stale temp artifact: %s", entry.Name()) + if err := os.RemoveAll(path); err != nil { + cleanupErrors = append(cleanupErrors, err) + } + } + + return errors.Join(cleanupErrors...) } func CheckPrerequisites() error { diff --git a/agent/internal/build/build_test.go b/agent/internal/build/build_test.go new file mode 100644 index 00000000..4efea471 --- /dev/null +++ b/agent/internal/build/build_test.go @@ -0,0 +1,92 @@ +package build + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestCleanupManagedTempArtifactsRemovesOnlyOldManagedArtifacts(t *testing.T) { + tmpDir := t.TempDir() + oldTime := time.Now().Add(-48 * time.Hour) + cutoff := time.Now().Add(-24 * time.Hour) + + oldArchive := filepath.Join(tmpDir, "backup-11111111-1111-1111-1111-111111111111.tar.gz") + youngArchive := filepath.Join(tmpDir, "restore-22222222-2222-2222-2222-222222222222.tar.gz") + unmanagedArchive := filepath.Join(tmpDir, "backup-not-a-build-id.tar.gz") + oldExtractDir := filepath.Join(tmpDir, "restore-extract-33333333-3333-3333-3333-333333333333") + + if err := os.WriteFile(oldArchive, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(youngArchive, []byte("young"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unmanagedArchive, []byte("unmanaged"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(oldExtractDir, 0700); err != nil { + t.Fatal(err) + } + + for _, path := range []string{oldArchive, unmanagedArchive, oldExtractDir} { + if err := os.Chtimes(path, oldTime, oldTime); err != nil { + t.Fatal(err) + } + } + + if err := cleanupManagedTempArtifacts(tmpDir, cutoff); err != nil { + t.Fatal(err) + } + + assertMissing(t, oldArchive) + assertMissing(t, oldExtractDir) + assertExists(t, youngArchive) + assertExists(t, unmanagedArchive) +} + +func TestCleanupStaleBuildDirsRemovesOnlyOldDirectories(t *testing.T) { + buildsDir := t.TempDir() + oldTime := time.Now().Add(-2 * time.Hour) + cutoff := time.Now().Add(-1 * time.Hour) + + oldDir := filepath.Join(buildsDir, "old-build") + youngDir := filepath.Join(buildsDir, "young-build") + filePath := filepath.Join(buildsDir, "not-a-dir") + + if err := os.Mkdir(oldDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(youngDir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filePath, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(oldDir, oldTime, oldTime); err != nil { + t.Fatal(err) + } + + if err := cleanupStaleBuildDirs(buildsDir, cutoff); err != nil { + t.Fatal(err) + } + + assertMissing(t, oldDir) + assertExists(t, youngDir) + assertExists(t, filePath) +} + +func assertExists(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected %s to exist: %v", path, err) + } +} + +func assertMissing(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected %s to be removed, got err=%v", path, err) + } +} diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index 21a8ba5d..912e694f 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -456,8 +456,12 @@ func writeDockerConfig(registryURL, username, password string) error { return nil } -func ImagePrune() { - exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h").Run() +func ImagePrune() error { + cmd := exec.Command("podman", "image", "prune", "-a", "-f", "--filter", "until=168h") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to prune images: %s: %w", string(output), err) + } + return nil } type podmanContainer struct { diff --git a/web/public/setup.sh b/web/public/setup.sh index ed91d52b..30725e66 100644 --- a/web/public/setup.sh +++ b/web/public/setup.sh @@ -61,6 +61,31 @@ pkg_hold() { fi } +configure_journald_limits() { + step "Configuring journal storage limits..." + mkdir -p /etc/systemd/journald.conf.d + cat > /etc/systemd/journald.conf.d/techulus.conf << 'EOF' +[Journal] +SystemMaxUse=1G +SystemKeepFree=1G +RuntimeMaxUse=256M +MaxRetentionSec=7day +EOF + systemctl restart systemd-journald 2>/dev/null || true + echo "✓ Journald storage limits configured" +} + +ensure_logrotate() { + if command -v logrotate &>/dev/null; then + return + fi + + pkg_install logrotate + if ! command -v logrotate &>/dev/null; then + error "Failed to install logrotate" + fi +} + echo "" echo "==========================================" echo " Techulus Cloud Agent Installer" @@ -118,6 +143,8 @@ case $ARCH in esac echo "✓ Architecture: $ARCH ($AGENT_ARCH)" +configure_journald_limits + step "Collecting configuration..." prompt CONTROL_PLANE_URL "Enter Control Plane URL (e.g., https://api.example.com): " required @@ -409,6 +436,26 @@ EOF fi echo "✓ Traefik config created" + step "Configuring Traefik access log rotation..." + ensure_logrotate + cat > /etc/logrotate.d/techulus-traefik << 'EOF' +/var/log/traefik/access.log { + daily + maxsize 100M + rotate 7 + compress + delaycompress + missingok + notifempty + copytruncate + create 0644 root root +} +EOF + if [ ! -f /etc/logrotate.d/techulus-traefik ]; then + error "Failed to create Traefik logrotate config" + fi + echo "✓ Traefik access log rotation configured" + cat > /etc/systemd/system/traefik.service << 'EOF' [Unit] Description=Traefik Reverse Proxy @@ -541,6 +588,23 @@ mkdir -p /etc/buildkit cat > /etc/buildkit/buildkitd.toml << 'EOF' [dns] nameservers = ["8.8.8.8", "1.1.1.1"] + +[worker.oci] + gc = true + reservedSpace = "15%" + maxUsedSpace = "30%" + minFreeSpace = "15%" + +[[worker.oci.gcpolicy]] + filters = ["type==source.local", "type==source.git.checkout", "type==exec.cachemount"] + keepDuration = "48h" + maxUsedSpace = "10%" + +[[worker.oci.gcpolicy]] + all = true + keepDuration = "168h" + reservedSpace = "15%" + maxUsedSpace = "30%" EOF if [ ! -f /etc/buildkit/buildkitd.toml ]; then error "Failed to create BuildKit config" @@ -568,7 +632,7 @@ fi systemctl daemon-reload systemctl enable buildkitd -systemctl start buildkitd +systemctl restart buildkitd sleep 2 if ! systemctl is-active --quiet buildkitd; then error "Failed to start BuildKit daemon"