From 2f118ccf491d691a72daaaa88dbda7950e092199 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 7 May 2026 11:27:57 -0400 Subject: [PATCH 1/2] config: add DigitalOcean Spaces example and inline R2 ACCOUNT_ID --- .github/workflows/integration.yml | 8 +++--- README.md | 17 ++++++------- .../main_test.go | 6 ++--- .../testdata/config.ini | 3 +-- internal/embed/config.ini | 25 +++++++++++++------ internal/storage/s3_presign.go | 9 +++---- 6 files changed, 38 insertions(+), 30 deletions(-) rename {integration => integration-tests}/main_test.go (97%) rename {integration => integration-tests}/testdata/config.ini (80%) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4209ae7..7901af3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -6,7 +6,7 @@ on: paths: - 'cmd/**' - 'internal/**' - - 'integration/**' + - 'integration-tests/**' - 'go.mod' - 'go.sum' - '.github/workflows/integration.yml' @@ -14,7 +14,7 @@ on: paths: - 'cmd/**' - 'internal/**' - - 'integration/**' + - 'integration-tests/**' - 'go.mod' - 'go.sum' - '.github/workflows/integration.yml' @@ -23,7 +23,7 @@ permissions: contents: read concurrency: - group: integration-${{ github.ref }} + group: integration-tests-${{ github.ref }} cancel-in-progress: true jobs: @@ -69,4 +69,4 @@ jobs: git --version git lfs version - name: Run tests - run: go test ./integration -v -long + run: go test ./integration-tests -v -long diff --git a/README.md b/README.md index 094af66..e833b39 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ Bring Your Own Forge (BYOF) Git LFS server with affordable storage. Assuming 1 TiB is uploaded and stored for a full month, and 3 TiB is downloaded in the same month: -| Provider | Storage | Egress | Yearly total | Savings vs. GitLab LFS | -| --- | ---: | ---: | ---: | ---: | -| GitLab LFS | 109 x 10 GB-month x $5 x 12 = $6,540.00 | Included | $6,540.00 | Baseline | -| GitHub LFS | 1,014 GiB x $0.07 x 12 = $851.76 | 36,744 GiB x $0.0875 = $3,215.10 | $4,066.86 | $2,473.14 (37.81%) | -| DigitalOcean Spaces | ($5 + 774 GiB x $0.02) x 12 = $245.76 | 24,576 GiB x $0.01 = $245.76 | $491.52 | $6,048.48 (92.48%) | -| Cloudflare R2 Standard | 1,090 GB-month x $0.015 x 12 = $196.20 | Included | $196.20 | $6,343.80 (97.00%) | -| Backblaze B2 | 1 TB x $6.95 x 12 = $83.40 | Free up to 3x storage | $83.40 | $6,456.60 (98.72%) | +| Provider | Storage | Egress | Yearly total | Savings vs. GitLab LFS | +|------------------------|----------------------------------------:|---------------------------------:|-------------:|-----------------------:| +| GitLab LFS | 109 x 10 GB-month x $5 x 12 = $6,540.00 | Included | $6,540.00 | Baseline | +| GitHub LFS | 1,014 GiB x $0.07 x 12 = $851.76 | 36,744 GiB x $0.0875 = $3,215.10 | $4,066.86 | $2,473.14 (37.81%) | +| DigitalOcean Spaces | ($5 + 774 GiB x $0.02) x 12 = $245.76 | 24,576 GiB x $0.01 = $245.76 | $491.52 | $6,048.48 (92.48%) | +| Cloudflare R2 Standard | 1,090 GB-month x $0.015 x 12 = $196.20 | Included | $196.20 | $6,343.80 (97.00%) | +| Backblaze B2 | 1 TB x $6.95 x 12 = $83.40 | Free up to 3x storage | $83.40 | $6,456.60 (98.72%) | ## Example setup @@ -37,11 +37,10 @@ STORAGE = r2 [storage "r2"] TYPE = s3-presign SCHEME = r2:// -ACCOUNT_ID = ${LFSD_R2_ACCOUNT_ID} BUCKET = lfs-objects ACCESS_KEY_ID = ${LFSD_R2_ACCESS_KEY_ID} SECRET_ACCESS_KEY = ${LFSD_R2_SECRET_ACCESS_KEY} -ENDPOINT = https://%(ACCOUNT_ID)s.r2.cloudflarestorage.com +ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com ``` `EXTERNAL_URL` is the public origin clients reach the server at. It is used as the base for the object download, upload, and verify URLs the server returns to the client, so it must match what the client sees (typically the public HTTPS URL terminated by your reverse proxy). diff --git a/integration/main_test.go b/integration-tests/main_test.go similarity index 97% rename from integration/main_test.go rename to integration-tests/main_test.go index 8673e21..953a79d 100644 --- a/integration/main_test.go +++ b/integration-tests/main_test.go @@ -1,6 +1,6 @@ //go:build !windows -package integration_test +package integration_tests import ( "context" @@ -29,7 +29,7 @@ var long = flag.Bool("long", false, "run long-running integration tests") func TestMain(m *testing.M) { flag.Parse() if !*long { - fmt.Println("integration: skipping (pass -long to run)") + fmt.Println("integration-tests: skipping (pass -long to run)") os.Exit(0) } os.Exit(m.Run()) @@ -123,7 +123,7 @@ func setupLfsd(ctx context.Context, t *testing.T) func() { Dir(root).Run().Wait(), "go build lfsd") - confPath := filepath.Join(root, "integration", "testdata", "config.ini") + confPath := filepath.Join(root, "integration-tests", "testdata", "config.ini") cmd := exec.CommandContext(ctx, binPath) cmd.Env = append(os.Environ(), "LFSD_CONFIG_PATH="+confPath) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} diff --git a/integration/testdata/config.ini b/integration-tests/testdata/config.ini similarity index 80% rename from integration/testdata/config.ini rename to integration-tests/testdata/config.ini index 045aca7..78391fc 100644 --- a/integration/testdata/config.ini +++ b/integration-tests/testdata/config.ini @@ -8,11 +8,10 @@ PASSWORD = ${LFSD_DATABASE_PASSWORD} [storage "r2"] TYPE = s3-presign SCHEME = r2:// -ACCOUNT_ID = ${LFSD_R2_ACCOUNT_ID} BUCKET = ${LFSD_R2_BUCKET} ACCESS_KEY_ID = ${LFSD_R2_ACCESS_KEY_ID} SECRET_ACCESS_KEY = ${LFSD_R2_SECRET_ACCESS_KEY} -ENDPOINT = https://%(ACCOUNT_ID)s.r2.cloudflarestorage.com +ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com [forge "github.com"] TYPE = github diff --git a/internal/embed/config.ini b/internal/embed/config.ini index 5571790..366e8c4 100644 --- a/internal/embed/config.ini +++ b/internal/embed/config.ini @@ -45,16 +45,27 @@ TEMP_DIR = %(ROOT)s/.tmp ; TYPE = s3-presign ; ; The scheme of the object URI, e.g., "r2://lfs-objects/71205e4480342d973ab4611d557e812dc8f38731d6f407a7d43eda7a5af5b562". ; SCHEME = r2:// -; ; R2 account ID (from Cloudflare dashboard). -; ACCOUNT_ID = ${LFSD_R2_ACCOUNT_ID} -; ; R2 bucket name. Must already exist. +; ; Cloudflare R2 bucket name. Must already exist. ; BUCKET = lfs-objects -; ; Access key ID for an R2 API token with read/write on BUCKET. +; ; Access key ID for an Cloudflare R2 API token with read/write on BUCKET. ; ACCESS_KEY_ID = ${LFSD_R2_ACCESS_KEY_ID} -; ; Secret access key for the R2 API token. +; ; Secret access key for the Cloudflare R2 API token. ; SECRET_ACCESS_KEY = ${LFSD_R2_SECRET_ACCESS_KEY} -; ; Endpoint URL to R2. -; ENDPOINT = https://%(ACCOUNT_ID)s.r2.cloudflarestorage.com +; ; Endpoint URL to Cloudflare R2. +; ENDPOINT = https://${LFSD_R2_ACCOUNT_ID}.r2.cloudflarestorage.com + +; [storage "do"] +; TYPE = s3-presign +; ; The scheme of the object URI, e.g., "do://lfs-objects/71205e4480342d973ab4611d557e812dc8f38731d6f407a7d43eda7a5af5b562". +; SCHEME = do:// +; ; DigitalOcean Spaces bucket name. Must already exist. +; BUCKET = lfs-objects +; ; Access key ID for an DigitalOcean Spaces API token with read/write on BUCKET. +; ACCESS_KEY_ID = ${LFSD_DO_ACCESS_KEY_ID} +; ; Secret access key for the DigitalOcean Spaces API token. +; SECRET_ACCESS_KEY = ${LFSD_DO_SECRET_ACCESS_KEY} +; ; Endpoint URL to DigitalOcean Spaces. DO NOT include the bucket name. +; ENDPOINT = https://nyc3.digitaloceanspaces.com ; Example forge configuration. Copy into your override config file and adjust ; the values. At least one forge must be configured. STORAGE references a diff --git a/internal/storage/s3_presign.go b/internal/storage/s3_presign.go index 5a55902..ce3208f 100644 --- a/internal/storage/s3_presign.go +++ b/internal/storage/s3_presign.go @@ -62,7 +62,6 @@ func NewS3PresignBackend(name, scheme, bucket, accessKeyID, secretAccessKey, end } client := s3.NewFromConfig(aws.Config{ - // R2 ignores region but the SDK requires one. Region: "auto", Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""), }, func(o *s3.Options) { @@ -111,12 +110,12 @@ func (b *S3PresignBackend) PresignPut(ctx context.Context, oid string, size int6 ChecksumSHA256: aws.String(checksumB64), }, func(o *s3.PresignOptions) { o.Expires = presignURLTTL - // DisableHeaderHoisting keeps x-amz-checksum-sha256 as a signed - // header instead of hoisting it to a query parameter, which is - // what makes S3/R2 enforce the checksum on PUT. - // https://github.com/aws/aws-sdk-go-v2/issues/2610 o.Presigner = v4.NewSigner(func(so *v4.SignerOptions) { so.DisableURIPathEscaping = true + // DisableHeaderHoisting keeps x-amz-checksum-sha256 as a signed + // header instead of hoisting it to a query parameter, which is + // what makes S3/R2 enforce the checksum on PUT. + // https://github.com/aws/aws-sdk-go-v2/issues/2610 so.DisableHeaderHoisting = true }) }) From 5bd9e76d66320d3ebae15b2663d0dfef442f45f7 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Thu, 7 May 2026 11:33:39 -0400 Subject: [PATCH 2/2] lint --- integration-tests/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/main_test.go b/integration-tests/main_test.go index 953a79d..e185223 100644 --- a/integration-tests/main_test.go +++ b/integration-tests/main_test.go @@ -1,6 +1,6 @@ //go:build !windows -package integration_tests +package main import ( "context"