From 8855706c67efebb5423b0e1d13d853cb29bc3776 Mon Sep 17 00:00:00 2001 From: agarciar Date: Mon, 25 May 2026 12:32:30 +0200 Subject: [PATCH] feat: retry failed Bunny operations with exponential backoff Transient 5xx/network errors during upload, delete, list, or purge aborted the whole deploy. Add a withRetry helper (exponential backoff + jitter) and wrap every BunnyClient network call in it, logging each retry. Uploads re-open the read stream per attempt (streams are single-use). Exposed as a new `retries` builder option (default 3, 0 = fail fast). Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 2 ++ README.md | 1 + src/bunny-client.spec.ts | 34 +++++++++++++++++++++++ src/bunny-client.ts | 60 ++++++++++++++++++++++++++++++---------- src/deploy.spec.ts | 1 + src/deploy.ts | 2 ++ src/retry.spec.ts | 52 ++++++++++++++++++++++++++++++++++ src/retry.ts | 30 ++++++++++++++++++++ src/schema.json | 6 ++++ src/types.ts | 1 + 10 files changed, 175 insertions(+), 14 deletions(-) create mode 100644 src/retry.spec.ts create mode 100644 src/retry.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ef0fe26..ec5e2fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 before syncing and auto-resolves the `/browser` output folder). - Configurable `storageRegion`, `targetFolder`, `ignore` globs, and `concurrency`. +- Automatic `retries` (default 3) with exponential backoff on failed uploads, + deletes, listings, and Pull Zone purges. - `dryRun` mode and a `purgeAfterUpload` toggle. - Credentials read from environment variables or a `.env.local` file. diff --git a/README.md b/README.md index dd81945..abfb35b 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ missing the build aborts before touching the network with a clear message. | `pullZoneId` | `number \| null` | `null` | Required when `purgeAfterUpload` is true. | | `purgeAfterUpload` | `boolean` | `true` | Purge the Pull Zone cache after a successful sync. | | `concurrency` | `number` | `8` | Parallel uploads/deletes. | +| `retries` | `number` | `3` | Retries per failed upload/delete/list/purge (exponential backoff). `0` disables. | | `ignore` | `string[]` | `[]` | Glob patterns to skip. Supports `**`, `*`, and literals. | | `dryRun` | `boolean` | `false` | Compute and print the diff without writing anything. | diff --git a/src/bunny-client.spec.ts b/src/bunny-client.spec.ts index ef62843..61be55d 100644 --- a/src/bunny-client.spec.ts +++ b/src/bunny-client.spec.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const sdkMock = { @@ -103,6 +106,37 @@ describe('BunnyClient', () => { expect((init as RequestInit).headers).toMatchObject({ AccessKey: 'ak' }); }); + it('retries a failed upload and succeeds on a later attempt', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bunny-up-')); + const file = join(dir, 'app.js'); + writeFileSync(file, 'console.log(1)'); + try { + let calls = 0; + sdkMock.file.upload.mockImplementation(async () => { + calls += 1; + if (calls < 2) throw new Error('503 transient'); + return undefined; + }); + const warnings: string[] = []; + const client = new BunnyClient({ + region: 'Falkenstein', + zoneName: 'my-zone', + storagePassword: 'sp', + accountApiKey: 'ak', + retries: 2, + retryBaseDelayMs: 0, + logger: { debug: () => {}, info: () => {}, warn: (m: string) => warnings.push(m) }, + }); + + await client.upload('app.js', file, 'deadbeef', '/'); + + expect(sdkMock.file.upload).toHaveBeenCalledTimes(2); + expect(warnings.some((w) => /retry/i.test(w))).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('purgePullZone throws on a non-2xx response', async () => { const fetchMock = vi.fn(async () => new Response('forbidden', { status: 403 })); vi.stubGlobal('fetch', fetchMock); diff --git a/src/bunny-client.ts b/src/bunny-client.ts index 7522827..141b84b 100644 --- a/src/bunny-client.ts +++ b/src/bunny-client.ts @@ -1,9 +1,14 @@ import { createReadStream } from 'node:fs'; import { Readable } from 'node:stream'; import * as BunnyStorageSDK from '@bunny.net/storage-sdk'; +import { withRetry } from './retry.js'; import { STORAGE_REGIONS, type StorageRegion } from './types.js'; import type { RemoteFile } from './types.js'; +function errText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + export interface BunnyLogger { debug(msg: string): void; info(msg: string): void; @@ -16,6 +21,10 @@ export interface BunnyClientOptions { storagePassword: string; accountApiKey: string | null; logger: BunnyLogger; + /** Additional attempts on a failed network call (0 = no retry). Default 0. */ + retries?: number; + /** Base backoff in ms for retries. Default 300; set 0 in tests. */ + retryBaseDelayMs?: number; } function resolveRegion(region: StorageRegion): BunnyStorageSDK.regions.StorageRegion { @@ -30,6 +39,8 @@ export class BunnyClient { private readonly zone: BunnyStorageSDK.zone.StorageZone; private readonly accountApiKey: string | null; private readonly logger: BunnyLogger; + private readonly retries: number; + private readonly retryBaseDelayMs: number; constructor(opts: BunnyClientOptions) { this.zone = BunnyStorageSDK.zone.connect_with_accesskey( @@ -39,6 +50,17 @@ export class BunnyClient { ); this.accountApiKey = opts.accountApiKey; this.logger = opts.logger; + this.retries = opts.retries ?? 0; + this.retryBaseDelayMs = opts.retryBaseDelayMs ?? 300; + } + + private retry(label: string, fn: () => Promise): Promise { + return withRetry(fn, { + retries: this.retries, + baseDelayMs: this.retryBaseDelayMs, + onRetry: (err, attempt) => + this.logger.warn(`Retry ${attempt}/${this.retries} for ${label}: ${errText(err)}`), + }); } async listAll(targetFolder: string): Promise { @@ -46,7 +68,10 @@ export class BunnyClient { const queue: string[] = [normalizeDir(targetFolder)]; while (queue.length > 0) { const dir = queue.shift()!; - const entries: BunnyStorageSDK.file.StorageFile[] = await BunnyStorageSDK.file.list(this.zone, dir); + const entries: BunnyStorageSDK.file.StorageFile[] = await this.retry( + `list ${dir}`, + () => BunnyStorageSDK.file.list(this.zone, dir), + ); for (const e of entries) { if (e.isDirectory) { queue.push(normalizeDir(`${dir}${e.objectName}/`)); @@ -65,35 +90,42 @@ export class BunnyClient { async upload(relPath: string, absPath: string, sha256: string, targetFolder: string): Promise { const remotePath = joinRemotePath(targetFolder, relPath); - const node = createReadStream(absPath); - const web = Readable.toWeb(node) as unknown as import('node:stream/web').ReadableStream; // Bunny Storage doesn't infer Content-Type from extension; whatever the // client sends is what the Pull Zone serves later. Without this, .mjs/.js/.css // are stored as application/octet-stream and browsers reject module scripts. const contentType = detectContentType(relPath); - await BunnyStorageSDK.file.upload(this.zone, remotePath, web, { sha256Checksum: sha256, contentType }); + // Open the read stream inside the retried fn: a stream is single-use, so a + // retry must start from a fresh one. + await this.retry(`upload ${relPath}`, () => { + const node = createReadStream(absPath); + const web = Readable.toWeb(node) as unknown as import('node:stream/web').ReadableStream; + return BunnyStorageSDK.file.upload(this.zone, remotePath, web, { sha256Checksum: sha256, contentType }); + }); } async remove(relPath: string, targetFolder: string): Promise { const remotePath = joinRemotePath(targetFolder, relPath); - await BunnyStorageSDK.file.remove(this.zone, remotePath); + await this.retry(`remove ${relPath}`, () => BunnyStorageSDK.file.remove(this.zone, remotePath)); } - // Pull Zone purge has no built-in retry: the orchestrator treats a failed - // purge as success-with-warning, and re-running the deploy converges. + // Retries transient failures; if it still fails the orchestrator treats it as + // success-with-warning (the cache expires by TTL and re-running converges). async purgePullZone(pullZoneId: number): Promise { if (!this.accountApiKey) { throw new Error('purgePullZone called without an accountApiKey'); } + const accessKey = this.accountApiKey; const url = `https://api.bunny.net/pullzone/${pullZoneId}/purgeCache`; - const res = await fetch(url, { - method: 'POST', - headers: { AccessKey: this.accountApiKey }, + await this.retry(`purge pull zone ${pullZoneId}`, async () => { + const res = await fetch(url, { + method: 'POST', + headers: { AccessKey: accessKey }, + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`Pull zone purge failed: ${res.status} ${res.statusText} ${body}`.trim()); + } }); - if (!res.ok) { - const body = await res.text().catch(() => ''); - throw new Error(`Pull zone purge failed: ${res.status} ${res.statusText} ${body}`.trim()); - } } } diff --git a/src/deploy.spec.ts b/src/deploy.spec.ts index f5d3316..df354cb 100644 --- a/src/deploy.spec.ts +++ b/src/deploy.spec.ts @@ -40,6 +40,7 @@ function baseOptions(overrides: Partial = {}): DeployOptions { pullZoneId: 12345, purgeAfterUpload: true, concurrency: 4, + retries: 3, ignore: [], dryRun: false, ...overrides, diff --git a/src/deploy.ts b/src/deploy.ts index da913e3..1e227c9 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -30,6 +30,7 @@ export interface Deps { zoneName: string; storagePassword: string; accountApiKey: string | null; + retries: number; logger: BunnyLogger; }) => ClientLike; } @@ -117,6 +118,7 @@ export async function runDeploy( zoneName: options.storageZoneName, storagePassword: secrets.storagePassword, accountApiKey: secrets.accountApiKey, + retries: options.retries, logger: log, }); diff --git a/src/retry.spec.ts b/src/retry.spec.ts new file mode 100644 index 0000000..6cabc70 --- /dev/null +++ b/src/retry.spec.ts @@ -0,0 +1,52 @@ +import { describe, expect, it, vi } from 'vitest'; +import { withRetry } from './retry.js'; + +describe('withRetry', () => { + it('returns the result without retrying when fn succeeds', async () => { + const fn = vi.fn(async () => 'ok'); + const out = await withRetry(fn, { retries: 3, baseDelayMs: 0 }); + expect(out).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('retries until fn succeeds', async () => { + let calls = 0; + const fn = vi.fn(async () => { + calls += 1; + if (calls < 3) throw new Error('transient'); + return 'ok'; + }); + const out = await withRetry(fn, { retries: 3, baseDelayMs: 0 }); + expect(out).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('rethrows the last error after exhausting retries', async () => { + const fn = vi.fn(async () => { + throw new Error('boom'); + }); + await expect(withRetry(fn, { retries: 2, baseDelayMs: 0 })).rejects.toThrow('boom'); + expect(fn).toHaveBeenCalledTimes(3); // initial attempt + 2 retries + }); + + it('does not retry when retries is 0', async () => { + const fn = vi.fn(async () => { + throw new Error('boom'); + }); + await expect(withRetry(fn, { retries: 0, baseDelayMs: 0 })).rejects.toThrow('boom'); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('calls onRetry with the error and the 1-based attempt number', async () => { + const onRetry = vi.fn(); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls < 2) throw new Error('x'); + return 1; + }; + await withRetry(fn, { retries: 3, baseDelayMs: 0, onRetry }); + expect(onRetry).toHaveBeenCalledTimes(1); + expect(onRetry).toHaveBeenCalledWith(expect.any(Error), 1); + }); +}); diff --git a/src/retry.ts b/src/retry.ts new file mode 100644 index 0000000..d7eea3f --- /dev/null +++ b/src/retry.ts @@ -0,0 +1,30 @@ +export interface RetryOptions { + /** Number of additional attempts after the first (0 = a single attempt). */ + retries: number; + /** Base backoff in ms; the wait grows exponentially. Defaults to 300. */ + baseDelayMs?: number; + /** Called before each retry with the error and the 1-based attempt number. */ + onRetry?: (err: unknown, attempt: number) => void; +} + +function sleep(ms: number): Promise { + return ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve(); +} + +export async function withRetry(fn: () => Promise, opts: RetryOptions): Promise { + const retries = Math.max(0, Math.floor(opts.retries)); + const baseDelayMs = opts.baseDelayMs ?? 300; + let attempt = 0; + for (;;) { + try { + return await fn(); + } catch (err) { + if (attempt >= retries) throw err; + attempt += 1; + opts.onRetry?.(err, attempt); + // Exponential backoff with jitter (50–100% of the computed delay). + const backoff = baseDelayMs * 2 ** (attempt - 1); + await sleep(backoff * (0.5 + Math.random() * 0.5)); + } + } +} diff --git a/src/schema.json b/src/schema.json index 14c17b7..1410897 100644 --- a/src/schema.json +++ b/src/schema.json @@ -45,6 +45,12 @@ "minimum": 1, "description": "Maximum number of parallel uploads and deletes." }, + "retries": { + "type": "number", + "default": 3, + "minimum": 0, + "description": "Times to retry a failed upload, delete, list, or purge, with exponential backoff. 0 disables retries." + }, "ignore": { "type": "array", "items": { "type": "string" }, diff --git a/src/types.ts b/src/types.ts index b499d9b..f86ef17 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,6 +22,7 @@ export interface DeployOptions extends JsonObject { pullZoneId: number | null; purgeAfterUpload: boolean; concurrency: number; + retries: number; ignore: string[]; dryRun: boolean; }