release: Retry PPA uploads

tools/ci/launchpad-ppa/main.go

@@ -213,13 +213,12 @@ func run(log *silog.Logger, req publishRequest) error {
 
 		if plan.Dput {
 			log.Info("Uploading source package", "series", series, "changes", changes)
-			output, flushOutput := silog.Writer(log.WithPrefix("dput"), silog.LevelInfo)
-			err := xec.Command(ctx, log, "dput", "--unchecked", plan.DputTarget, changes).
-				WithStdout(output).
-				WithStderr(output).
-				Run()
-			flushOutput()
-			if err != nil {
+			if err := uploadSourcePackage(
+				ctx,
+				log,
+				plan.DputTarget,
+				changes,
+			); err != nil {
 				return fmt.Errorf("dput %s: %w", series, err)
 			}
 		} else {
@@ -237,6 +236,43 @@ func run(log *silog.Logger, req publishRequest) error {
 	return nil
 }
 
+// uploadSourcePackage runs dput up to three times with exponential backoff.
+func uploadSourcePackage(
+	ctx context.Context,
+	log *silog.Logger,
+	target string,
+	changes string,
+) error {
+	const maxAttempts = 3
+	for attempt := 1; ; attempt++ {
+		output, flushOutput := silog.Writer(log.WithPrefix("dput"), silog.LevelInfo)
+		err := xec.Command(ctx, log, "dput", "--unchecked", target, changes).
+			WithStdout(output).
+			WithStderr(output).
+			Run()
+		flushOutput()
+		if err == nil {
+			return nil
+		}
+		if attempt >= maxAttempts {
+			return err
+		}
+
+		delay := 15 * time.Second << (attempt - 1)
+		log.Warn("dput failed; retrying",
+			"attempt", attempt,
+			"maxAttempts", maxAttempts,
+			"delay", delay,
+			"error", err)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(delay):
+		}
+	}
+}
+
 func writeDebianPackaging(log *silog.Logger, sourceDir string) error {
 	log.Info("Writing embedded Debian packaging")
 

tools/ci/launchpad-ppa/upload_test.go

@@ -0,0 +1,89 @@
+//go:build linux
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.abhg.dev/gs/internal/silog/silogtest"
+)
+
+func TestUploadSourcePackage_retriesTwice(t *testing.T) {
+	binDir := t.TempDir()
+	attemptsFile := filepath.Join(t.TempDir(), "attempts")
+	writeRetryExecutable(t, filepath.Join(binDir, "dput"), `#!/bin/sh
+set -eu
+attempts=0
+if [ -f "$DPUT_ATTEMPTS_FILE" ]; then
+  attempts=$(cat "$DPUT_ATTEMPTS_FILE")
+fi
+attempts=$((attempts + 1))
+printf '%s' "$attempts" > "$DPUT_ATTEMPTS_FILE"
+if [ "$attempts" -lt 3 ]; then
+  exit 1
+fi
+`)
+	t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+	t.Setenv("DPUT_ATTEMPTS_FILE", attemptsFile)
+
+	synctest.Test(t, func(t *testing.T) {
+		start := time.Now()
+		err := uploadSourcePackage(
+			t.Context(),
+			silogtest.New(t),
+			"ppa:test/git-spice",
+			"package_source.changes",
+		)
+		require.NoError(t, err)
+		assert.Equal(t, 45*time.Second, time.Since(start))
+	})
+
+	attempts, err := os.ReadFile(attemptsFile)
+	require.NoError(t, err)
+	assert.Equal(t, "3", string(attempts))
+}
+
+func TestUploadSourcePackage_stopsAfterThirdAttempt(t *testing.T) {
+	binDir := t.TempDir()
+	attemptsFile := filepath.Join(t.TempDir(), "attempts")
+	writeRetryExecutable(t, filepath.Join(binDir, "dput"), `#!/bin/sh
+set -eu
+attempts=0
+if [ -f "$DPUT_ATTEMPTS_FILE" ]; then
+  attempts=$(cat "$DPUT_ATTEMPTS_FILE")
+fi
+attempts=$((attempts + 1))
+printf '%s' "$attempts" > "$DPUT_ATTEMPTS_FILE"
+exit 1
+`)
+	t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+	t.Setenv("DPUT_ATTEMPTS_FILE", attemptsFile)
+
+	synctest.Test(t, func(t *testing.T) {
+		start := time.Now()
+		err := uploadSourcePackage(
+			t.Context(),
+			silogtest.New(t),
+			"ppa:test/git-spice",
+			"package_source.changes",
+		)
+		require.Error(t, err)
+		assert.Equal(t, 45*time.Second, time.Since(start))
+	})
+
+	attempts, err := os.ReadFile(attemptsFile)
+	require.NoError(t, err)
+	assert.Equal(t, "3", string(attempts))
+}
+
+func writeRetryExecutable(t *testing.T, path string, body string) {
+	t.Helper()
+
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o755))
+}