diff --git a/tools/ci/launchpad-ppa/main.go b/tools/ci/launchpad-ppa/main.go index 20d5afd5..392a28c1 100644 --- a/tools/ci/launchpad-ppa/main.go +++ b/tools/ci/launchpad-ppa/main.go @@ -213,13 +213,12 @@ func run(log *silog.Logger, req publishRequest) error { if plan.Dput { log.Info("Uploading source package", "series", series, "changes", changes) - output, flushOutput := silog.Writer(log.WithPrefix("dput"), silog.LevelInfo) - err := xec.Command(ctx, log, "dput", "--unchecked", plan.DputTarget, changes). - WithStdout(output). - WithStderr(output). - Run() - flushOutput() - if err != nil { + if err := uploadSourcePackage( + ctx, + log, + plan.DputTarget, + changes, + ); err != nil { return fmt.Errorf("dput %s: %w", series, err) } } else { @@ -237,6 +236,43 @@ func run(log *silog.Logger, req publishRequest) error { return nil } +// uploadSourcePackage runs dput up to three times with exponential backoff. +func uploadSourcePackage( + ctx context.Context, + log *silog.Logger, + target string, + changes string, +) error { + const maxAttempts = 3 + for attempt := 1; ; attempt++ { + output, flushOutput := silog.Writer(log.WithPrefix("dput"), silog.LevelInfo) + err := xec.Command(ctx, log, "dput", "--unchecked", target, changes). + WithStdout(output). + WithStderr(output). + Run() + flushOutput() + if err == nil { + return nil + } + if attempt >= maxAttempts { + return err + } + + delay := 15 * time.Second << (attempt - 1) + log.Warn("dput failed; retrying", + "attempt", attempt, + "maxAttempts", maxAttempts, + "delay", delay, + "error", err) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } +} + func writeDebianPackaging(log *silog.Logger, sourceDir string) error { log.Info("Writing embedded Debian packaging") diff --git a/tools/ci/launchpad-ppa/upload_test.go b/tools/ci/launchpad-ppa/upload_test.go new file mode 100644 index 00000000..3bb5e1d5 --- /dev/null +++ b/tools/ci/launchpad-ppa/upload_test.go @@ -0,0 +1,89 @@ +//go:build linux + +package main + +import ( + "os" + "path/filepath" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.abhg.dev/gs/internal/silog/silogtest" +) + +func TestUploadSourcePackage_retriesTwice(t *testing.T) { + binDir := t.TempDir() + attemptsFile := filepath.Join(t.TempDir(), "attempts") + writeRetryExecutable(t, filepath.Join(binDir, "dput"), `#!/bin/sh +set -eu +attempts=0 +if [ -f "$DPUT_ATTEMPTS_FILE" ]; then + attempts=$(cat "$DPUT_ATTEMPTS_FILE") +fi +attempts=$((attempts + 1)) +printf '%s' "$attempts" > "$DPUT_ATTEMPTS_FILE" +if [ "$attempts" -lt 3 ]; then + exit 1 +fi +`) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("DPUT_ATTEMPTS_FILE", attemptsFile) + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + err := uploadSourcePackage( + t.Context(), + silogtest.New(t), + "ppa:test/git-spice", + "package_source.changes", + ) + require.NoError(t, err) + assert.Equal(t, 45*time.Second, time.Since(start)) + }) + + attempts, err := os.ReadFile(attemptsFile) + require.NoError(t, err) + assert.Equal(t, "3", string(attempts)) +} + +func TestUploadSourcePackage_stopsAfterThirdAttempt(t *testing.T) { + binDir := t.TempDir() + attemptsFile := filepath.Join(t.TempDir(), "attempts") + writeRetryExecutable(t, filepath.Join(binDir, "dput"), `#!/bin/sh +set -eu +attempts=0 +if [ -f "$DPUT_ATTEMPTS_FILE" ]; then + attempts=$(cat "$DPUT_ATTEMPTS_FILE") +fi +attempts=$((attempts + 1)) +printf '%s' "$attempts" > "$DPUT_ATTEMPTS_FILE" +exit 1 +`) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("DPUT_ATTEMPTS_FILE", attemptsFile) + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + err := uploadSourcePackage( + t.Context(), + silogtest.New(t), + "ppa:test/git-spice", + "package_source.changes", + ) + require.Error(t, err) + assert.Equal(t, 45*time.Second, time.Since(start)) + }) + + attempts, err := os.ReadFile(attemptsFile) + require.NoError(t, err) + assert.Equal(t, "3", string(attempts)) +} + +func writeRetryExecutable(t *testing.T, path string, body string) { + t.Helper() + + require.NoError(t, os.WriteFile(path, []byte(body), 0o755)) +}